feat: apply cronos store (#207)

* Support object store (#206) generic interface generic btree generic cachekv generic transient store support ObjStore changelog Update CHANGELOG.md Signed-off-by: yihuang <[email protected]> object store key Apply review suggestions fix merge conflict fix snapshot revert dependers * Problem: CacheWrapWithTrace api is not used (#207) Solution: - remove the api changelog * prefix store support object store (#236) * Problem: MultiStore interface is bloated (#240) * Problem: MultiStore interface is bloated Solution: - Split out specialied methods from it, keeping the MultiStore generic * Update store/CHANGELOG.md Signed-off-by: yihuang <[email protected]> --------- Signed-off-by: yihuang <[email protected]> * Problem: nested cache store not efficient (#241) * Problem: nested cache store not efficient Solution: - introduce copy-on-write btree based cache store temp * changelog * rename * Update store/cachekv/store.go Signed-off-by: yihuang <[email protected]> --------- Signed-off-by: yihuang <[email protected]> * Problem: many memory allocs in cache store wrapping (#242) Solution: - init cachestore on cachestore lazily. - cleanup some unused stuff. Update store/CHANGELOG.md Signed-off-by: yihuang <[email protected]> * Problem: no API to use the new CoW branch store (#243) * Support RunAtomic API * add unit test * Problem: store key type assertion is incorrect (#244) fix and add test * Problem: Restore don't work snapshot revert usage (#245) Solution: - fix and add test to support the usage pattern in ethermint add Discard method to CacheWrap better testing * feat: save some memory allocations on un-used cache store (#248) * feat: save some memory allocations on un-used cache store * Update CHANGELOG.md Signed-off-by: yihuang <[email protected]> * Update store/internal/btree/btree.go Signed-off-by: yihuang <[email protected]> --------- Signed-off-by: yihuang <[email protected]> * Problem: no api to create cachemulti store from external cache store (#258) Solution: - add API NewFromParent to cache multistore. Update store/CHANGELOG.md Signed-off-by: yihuang <[email protected]> fix test fix lint * Problem: snapshot for object store is not skipped (cosmos#585) * Problem: versiondb/memiavl don't compatible with upstream sdk (cosmos#588) * Problem: versiondb/memiavl don't compatible with upstream sdk Solution: - avoid api breaking change * fix build * chore: Update test.yml go version 1.21 * chore: fix lint --------- Signed-off-by: yihuang <[email protected]> Co-authored-by: yihuang <[email protected]> Co-authored-by: mmsqe <[email protected]> Co-authored-by: Kyuhyeon Choi <[email protected]>
b-harvest · Dec 17, 2024 · fab9101 · fab9101
1 parent f516a1e
commit fab9101
Show file tree

Hide file tree

Showing 32 changed files with 965 additions and 966 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -291,7 +291,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: core/go.sum
@@ -322,7 +322,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: depinject/go.sum
@@ -351,7 +351,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: errors/go.sum
@@ -382,7 +382,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: math/go.sum
@@ -472,7 +472,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: orm/go.sum
@@ -596,7 +596,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: store/go.sum
@@ -627,7 +627,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
           check-latest: true
           cache: true
           cache-dependency-path: log/go.sum

diff --git a/store/cachekv/internal/btree.go b/store/cachekv/internal/btree.go
diff --git a/store/cachekv/internal/mergeiterator.go b/store/cachekv/internal/mergeiterator.go
@@ -14,21 +14,24 @@ import (
 // cache shadows (overrides) the parent.
 //
 // TODO: Optimize by memoizing.
-type cacheMergeIterator struct {
-	parent    types.Iterator
-	cache     types.Iterator
+type cacheMergeIterator[V any] struct {
+	parent    types.GIterator[V]
+	cache     types.GIterator[V]
 	ascending bool
 
 	valid bool
+
+	isZero func(V) bool
 }
 
-var _ types.Iterator = (*cacheMergeIterator)(nil)
+var _ types.Iterator = (*cacheMergeIterator[[]byte])(nil)
 
-func NewCacheMergeIterator(parent, cache types.Iterator, ascending bool) types.Iterator {
-	iter := &cacheMergeIterator{
+func NewCacheMergeIterator[V any](parent, cache types.GIterator[V], ascending bool, isZero func(V) bool) types.GIterator[V] {
+	iter := &cacheMergeIterator[V]{
 		parent:    parent,
 		cache:     cache,
 		ascending: ascending,
+		isZero:    isZero,
 	}
 
 	iter.valid = iter.skipUntilExistsOrInvalid()
@@ -37,17 +40,17 @@ func NewCacheMergeIterator(parent, cache types.Iterator, ascending bool) types.I
 
 // Domain implements Iterator.
 // Returns parent domain because cache and parent domains are the same.
-func (iter *cacheMergeIterator) Domain() (start, end []byte) {
+func (iter *cacheMergeIterator[V]) Domain() (start, end []byte) {
 	return iter.parent.Domain()
 }
 
 // Valid implements Iterator.
-func (iter *cacheMergeIterator) Valid() bool {
+func (iter *cacheMergeIterator[V]) Valid() bool {
 	return iter.valid
 }
 
 // Next implements Iterator
-func (iter *cacheMergeIterator) Next() {
+func (iter *cacheMergeIterator[V]) Next() {
 	iter.assertValid()
 
 	switch {
@@ -74,7 +77,7 @@ func (iter *cacheMergeIterator) Next() {
 }
 
 // Key implements Iterator
-func (iter *cacheMergeIterator) Key() []byte {
+func (iter *cacheMergeIterator[V]) Key() []byte {
 	iter.assertValid()
 
 	// If parent is invalid, get the cache key.
@@ -104,7 +107,7 @@ func (iter *cacheMergeIterator) Key() []byte {
 }
 
 // Value implements Iterator
-func (iter *cacheMergeIterator) Value() []byte {
+func (iter *cacheMergeIterator[V]) Value() V {
 	iter.assertValid()
 
 	// If parent is invalid, get the cache value.
@@ -134,7 +137,7 @@ func (iter *cacheMergeIterator) Value() []byte {
 }
 
 // Close implements Iterator
-func (iter *cacheMergeIterator) Close() error {
+func (iter *cacheMergeIterator[V]) Close() error {
 	err1 := iter.cache.Close()
 	if err := iter.parent.Close(); err != nil {
 		return err
@@ -145,7 +148,7 @@ func (iter *cacheMergeIterator) Close() error {
 
 // Error returns an error if the cacheMergeIterator is invalid defined by the
 // Valid method.
-func (iter *cacheMergeIterator) Error() error {
+func (iter *cacheMergeIterator[V]) Error() error {
 	if !iter.Valid() {
 		return errors.New("invalid cacheMergeIterator")
 	}
@@ -155,14 +158,14 @@ func (iter *cacheMergeIterator) Error() error {
 
 // If not valid, panics.
 // NOTE: May have side-effect of iterating over cache.
-func (iter *cacheMergeIterator) assertValid() {
+func (iter *cacheMergeIterator[V]) assertValid() {
 	if err := iter.Error(); err != nil {
 		panic(err)
 	}
 }
 
 // Like bytes.Compare but opposite if not ascending.
-func (iter *cacheMergeIterator) compare(a, b []byte) int {
+func (iter *cacheMergeIterator[V]) compare(a, b []byte) int {
 	if iter.ascending {
 		return bytes.Compare(a, b)
 	}
@@ -175,9 +178,9 @@ func (iter *cacheMergeIterator) compare(a, b []byte) int {
 // If the current cache item is not a delete item, does nothing.
 // If `until` is nil, there is no limit, and cache may end up invalid.
 // CONTRACT: cache is valid.
-func (iter *cacheMergeIterator) skipCacheDeletes(until []byte) {
+func (iter *cacheMergeIterator[V]) skipCacheDeletes(until []byte) {
 	for iter.cache.Valid() &&
-		iter.cache.Value() == nil &&
+		iter.isZero(iter.cache.Value()) &&
 		(until == nil || iter.compare(iter.cache.Key(), until) < 0) {
 		iter.cache.Next()
 	}
@@ -186,7 +189,7 @@ func (iter *cacheMergeIterator) skipCacheDeletes(until []byte) {
 // Fast forwards cache (or parent+cache in case of deleted items) until current
 // item exists, or until iterator becomes invalid.
 // Returns whether the iterator is valid.
-func (iter *cacheMergeIterator) skipUntilExistsOrInvalid() bool {
+func (iter *cacheMergeIterator[V]) skipUntilExistsOrInvalid() bool {
 	for {
 		// If parent is invalid, fast-forward cache.
 		if !iter.parent.Valid() {
@@ -211,7 +214,7 @@ func (iter *cacheMergeIterator) skipUntilExistsOrInvalid() bool {
 		case 0: // parent == cache.
 			// Skip over if cache item is a delete.
 			valueC := iter.cache.Value()
-			if valueC == nil {
+			if iter.isZero(valueC) {
 				iter.parent.Next()
 				iter.cache.Next()
 
@@ -223,7 +226,7 @@ func (iter *cacheMergeIterator) skipUntilExistsOrInvalid() bool {
 		case 1: // cache < parent
 			// Skip over if cache item is a delete.
 			valueC := iter.cache.Value()
-			if valueC == nil {
+			if iter.isZero(valueC) {
 				iter.skipCacheDeletes(keyP)
 				continue
 			}

diff --git a/store/cachekv/search_benchmark_test.go b/store/cachekv/search_benchmark_test.go