feat: allow saving version 0 (backport #1002)

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ jobs:
         # integer overflow).
       - name: test & coverage report creation
         run: |
+          cd cmd/legacydump && go build -o legacydump main.go && cd ../..
           go test ./... -mod=readonly -timeout 10m -short -race -coverprofile=coverage.txt -covermode=atomic
           go test ./... -mod=readonly -timeout 15m
           GOARCH=386 go test ./... -mod=readonly -timeout 15m

.gitignore

@@ -1,3 +1,4 @@
+cmd/legacydump/legacydump
 vendor
 .glide
 *.swp

Makefile

@@ -24,9 +24,12 @@ test-short:
 	@go test ./... $(LDFLAGS) -v --race --short
 .PHONY: test-short
 
-test:
+cmd/legacydump/legacydump: 
+	cd cmd/legacydump && go build -o legacydump main.go
+
+test: cmd/legacydump/legacydump
 	@echo "--> Running go test"
-	@go test ./... $(LDFLAGS) -v 
+	@go test ./... $(LDFLAGS) 
 .PHONY: test
 
 format:

docs/node/nodedb.md

@@ -19,7 +19,7 @@ When a version `v` is deleted, all nodes which removed in the current version wi
 ```golang
 // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards.
 func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error {
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}

immutable_tree.go

@@ -303,7 +303,7 @@ func (t *ImmutableTree) IsFastCacheEnabled() (bool, error) {
 }
 
 func (t *ImmutableTree) isLatestTreeVersion() (bool, error) {
-	latestVersion, err := t.ndb.getLatestVersion()
+	_, latestVersion, err := t.ndb.getLatestVersion()
 	if err != nil {
 		return false, err
 	}

iterator_test.go

@@ -182,7 +182,7 @@ func TestIterator_WithDelete_Full_Ascending_Success(t *testing.T) {
 	err = tree.DeleteVersionsTo(1)
 	require.NoError(t, err)
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	immutableTree, err := tree.GetImmutable(latestVersion)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func setupIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (corestore
 	_, _, err := tree.SaveVersion()
 	require.NoError(t, err)
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	immutableTree, err := tree.GetImmutable(latestVersion)
 	require.NoError(t, err)

migrate_test.go

@@ -177,7 +177,7 @@ func TestDeleteVersions(t *testing.T) {
 	// Test LoadVersionForOverwriting for the legacy version
 	err = tree.LoadVersionForOverwriting(int64(targetVersion))
 	require.NoError(t, err)
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, int64(targetVersion), latestVersion)
 	legacyLatestVersion, err := tree.ndb.getLegacyLatestVersion()

mutable_tree.go

@@ -41,6 +41,7 @@ type MutableTree struct {
 	unsavedFastNodeRemovals  *sync.Map      // map[string]interface{} FastNodes that have not yet been removed from disk
 	ndb                      *nodeDB
 	skipFastStorageUpgrade   bool // If true, the tree will work like no fast storage and always not upgrade fast storage
+	initialVersionSet        bool
 
 	mtx sync.Mutex
 }
@@ -63,6 +64,7 @@ func NewMutableTree(db dbm.DB, cacheSize int, skipFastStorageUpgrade bool, lg Lo
 		unsavedFastNodeRemovals:  &sync.Map{},
 		ndb:                      ndb,
 		skipFastStorageUpgrade:   skipFastStorageUpgrade,
+		initialVersionSet:        opts.initialVersionSet,
 	}
 }
 
@@ -74,7 +76,8 @@ func (tree *MutableTree) IsEmpty() bool {
 
 // GetLatestVersion returns the latest version of the tree.
 func (tree *MutableTree) GetLatestVersion() (int64, error) {
-	return tree.ndb.getLatestVersion()
+	_, v, err := tree.ndb.getLatestVersion()
+	return v, err
 }
 
 // VersionExists returns whether or not a version exists.
@@ -91,10 +94,13 @@ func (tree *MutableTree) VersionExists(version int64) bool {
 	if err != nil {
 		return false
 	}
-	latestVersion, err := tree.ndb.getLatestVersion()
+	found, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return false
 	}
+	if !found {
+		return false
+	}
 
 	return firstVersion <= version && version <= latestVersion
 }
@@ -105,7 +111,7 @@ func (tree *MutableTree) AvailableVersions() []int {
 	if err != nil {
 		return nil
 	}
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return nil
 	}
@@ -147,7 +153,7 @@ func (tree *MutableTree) WorkingHash() []byte {
 
 func (tree *MutableTree) WorkingVersion() int64 {
 	version := tree.version + 1
-	if version == 1 && tree.ndb.opts.InitialVersion > 0 {
+	if version == 1 && tree.initialVersionSet {
 		version = int64(tree.ndb.opts.InitialVersion)
 	}
 	return version
@@ -450,21 +456,16 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) {
 			tree.ndb.opts.InitialVersion, firstVersion)
 	}
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	ok, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return 0, err
 	}
 
-	if firstVersion > 0 && firstVersion < int64(tree.ndb.opts.InitialVersion) {
-		return latestVersion, fmt.Errorf("initial version set to %v, but found earlier version %v",
-			tree.ndb.opts.InitialVersion, firstVersion)
-	}
-
 	if latestVersion < targetVersion {
 		return latestVersion, fmt.Errorf("wanted to load target %d but only found up to %d", targetVersion, latestVersion)
 	}
 
-	if firstVersion == 0 {
+	if !ok {
 		if targetVersion <= 0 {
 			if !tree.skipFastStorageUpgrade {
 				tree.mtx.Lock()
@@ -605,7 +606,7 @@ func (tree *MutableTree) enableFastStorageAndCommit() error {
 		return err
 	}
 
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -708,6 +709,7 @@ func (tree *MutableTree) UnsetCommitting() {
 // the tree. Returns the hash and new version number.
 func (tree *MutableTree) SaveVersion() ([]byte, int64, error) {
 	version := tree.WorkingVersion()
+	tree.initialVersionSet = false
 
 	if tree.VersionExists(version) {
 		// If the version already exists, return an error as we're attempting to overwrite.
@@ -872,6 +874,7 @@ func (tree *MutableTree) saveFastNodeRemovals() error {
 // and is otherwise ignored.
 func (tree *MutableTree) SetInitialVersion(version uint64) {
 	tree.ndb.opts.InitialVersion = version
+	tree.initialVersionSet = true
 }
 
 // DeleteVersionsTo removes versions upto the given version from the MutableTree.

mutable_tree_test.go

@@ -770,6 +770,8 @@ func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) {
 	require.False(t, isUpgradeable)
 	require.NoError(t, err)
 
+	_, _, err = tree.SaveVersion()
+	require.NoError(t, err)
 	isFastCacheEnabled, err = tree.IsFastCacheEnabled()
 	require.NoError(t, err)
 	require.True(t, isFastCacheEnabled)
@@ -908,7 +910,7 @@ func TestFastStorageReUpgradeProtection_NoForceUpgrade_Success(t *testing.T) {
 
 	// Pretend that we called Load and have the latest state in the tree
 	tree.version = latestTreeVersion
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, latestVersion, int64(latestTreeVersion))
 
@@ -1002,7 +1004,7 @@ func TestFastStorageReUpgradeProtection_ForceUpgradeFirstTime_NoForceSecondTime_
 
 	// Pretend that we called Load and have the latest state in the tree
 	tree.version = latestTreeVersion
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, latestVersion, int64(latestTreeVersion))
 
@@ -1480,3 +1482,16 @@ func TestMutableTreeClose(t *testing.T) {
 
 	require.NoError(t, tree.Close())
 }
+
+func TestMutableTree_InitialVersionZero(t *testing.T) {
+	db := dbm.NewMemDB()
+
+	tree := NewMutableTree(db, 0, false, NewNopLogger(), InitialVersionOption(0))
+
+	_, err := tree.Set([]byte("hello"), []byte("world"))
+	require.NoError(t, err)
+
+	_, version, err := tree.SaveVersion()
+	require.NoError(t, err)
+	require.Equal(t, int64(0), version)
+}

nodedb.go

@@ -332,7 +332,7 @@ func (ndb *nodeDB) shouldForceFastStorageUpgrade() (bool, error) {
 	versions := strings.Split(ndb.storageVersion, fastStorageVersionDelimiter)
 
 	if len(versions) == 2 {
-		latestVersion, err := ndb.getLatestVersion()
+		_, latestVersion, err := ndb.getLatestVersion()
 		if err != nil {
 			// TODO: should be true or false as default? (removed panic here)
 			return false, err
@@ -526,7 +526,7 @@ func (ndb *nodeDB) deleteLegacyVersions(legacyLatestVersion int64) error {
 
 // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards.
 func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error {
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -643,7 +643,7 @@ func (ndb *nodeDB) deleteVersionsTo(toVersion int64) error {
 		return err
 	}
 
-	latest, err := ndb.getLatestVersion()
+	_, latest, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}
@@ -728,7 +728,7 @@ func (ndb *nodeDB) getFirstVersion() (int64, error) {
 		return version, nil
 	}
 	// Find the first version
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	if err != nil {
 		return 0, err
 	}
@@ -798,21 +798,21 @@ func (ndb *nodeDB) resetLegacyLatestVersion(version int64) {
 	ndb.legacyLatestVersion = version
 }
 
-func (ndb *nodeDB) getLatestVersion() (int64, error) {
+func (ndb *nodeDB) getLatestVersion() (bool, int64, error) {
 	ndb.mtx.Lock()
 	latestVersion := ndb.latestVersion
 	ndb.mtx.Unlock()
 
 	if latestVersion > 0 {
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
 	itr, err := ndb.db.ReverseIterator(
 		nodeKeyPrefixFormat.KeyInt64(int64(1)),
 		nodeKeyPrefixFormat.KeyInt64(int64(math.MaxInt64)),
 	)
 	if err != nil {
-		return 0, err
+		return false, 0, err
 	}
 	defer itr.Close()
 
@@ -822,24 +822,25 @@ func (ndb *nodeDB) getLatestVersion() (int64, error) {
 		nodeKeyFormat.Scan(k, &nk)
 		latestVersion = GetNodeKey(nk).version
 		ndb.resetLatestVersion(latestVersion)
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
 	if err := itr.Error(); err != nil {
-		return 0, err
+		return false, 0, err
 	}
 
 	// If there are no versions, try to get the latest version from the legacy format.
 	latestVersion, err = ndb.getLegacyLatestVersion()
 	if err != nil {
-		return 0, err
+		return false, 0, err
 	}
 	if latestVersion > 0 {
 		ndb.resetLatestVersion(latestVersion)
-		return latestVersion, nil
+		return true, latestVersion, nil
 	}
 
-	return 0, nil
+	return false, 0, nil
+	// return -1, nil
 }
 
 func (ndb *nodeDB) resetLatestVersion(version int64) {
@@ -1247,7 +1248,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(
 	if startVersion < firstVersion {
 		startVersion = firstVersion
 	}
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	if err != nil {
 		return err
 	}

nodedb_test.go

@@ -87,7 +87,7 @@ func TestSetStorageVersion_Success(t *testing.T) {
 	ndb := newNodeDB(db, 0, DefaultOptions(), NewNopLogger())
 	require.Equal(t, defaultStorageVersionValue, ndb.getStorageVersion())
 
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	require.NoError(t, err)
 
 	err = ndb.SetFastStorageVersionToBatch(latestVersion)
@@ -404,7 +404,7 @@ func TestDeleteVersionsFromNoDeadlock(t *testing.T) {
 	err := ndb.SetFastStorageVersionToBatch(ndb.latestVersion)
 	require.NoError(t, err)
 
-	latestVersion, err := ndb.getLatestVersion()
+	_, latestVersion, err := ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, expectedVersion+fastStorageVersionDelimiter+strconv.Itoa(int(latestVersion)), ndb.getStorageVersion())
 	require.NoError(t, ndb.batch.Write())

options.go

@@ -87,6 +87,8 @@ type Options struct {
 
 	// AsyncPruning is a flag to enable async pruning
 	AsyncPruning bool
+
+	initialVersionSet bool
 }
 
 // DefaultOptions returns the default options for IAVL.
@@ -105,6 +107,7 @@ func SyncOption(sync bool) Option {
 func InitialVersionOption(iv uint64) Option {
 	return func(opts *Options) {
 		opts.InitialVersion = iv
+		opts.initialVersionSet = true
 	}
 }
 

tree_random_test.go

@@ -282,7 +282,7 @@ func assertEmptyDatabase(t *testing.T, tree *MutableTree) {
 
 	storageVersionValue, err := tree.ndb.db.Get([]byte(firstKey))
 	require.NoError(t, err)
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	require.Equal(t, fastStorageVersionValue+fastStorageVersionDelimiter+strconv.Itoa(int(latestVersion)), string(storageVersionValue))
 
@@ -346,7 +346,7 @@ func assertMirror(t *testing.T, tree *MutableTree, mirror map[string]string, ver
 
 // Checks that fast node cache matches live state.
 func assertFastNodeCacheIsLive(t *testing.T, tree *MutableTree, mirror map[string]string, version int64) {
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	if latestVersion != version {
 		// The fast node cache check should only be done to the latest version
@@ -363,7 +363,7 @@ func assertFastNodeCacheIsLive(t *testing.T, tree *MutableTree, mirror map[strin
 
 // Checks that fast nodes on disk match live state.
 func assertFastNodeDiskIsLive(t *testing.T, tree *MutableTree, mirror map[string]string, version int64) {
-	latestVersion, err := tree.ndb.getLatestVersion()
+	_, latestVersion, err := tree.ndb.getLatestVersion()
 	require.NoError(t, err)
 	if latestVersion != version {
 		// The fast node disk check should only be done to the latest version