Problem: no command to fix corrupted data in versiondb (backport: #1685)

Closes: #1683 Solution: - add fix command to fix corrupted data in versiondb rename support SkipVersionZero support SkipVersionZero cleanup cleanup cleanup fix test cleanup destroy log store name fix data manually cli Update versiondb/client/fixdata.go Signed-off-by: yihuang <[email protected]> Update versiondb/client/fixdata.go Signed-off-by: yihuang <[email protected]> rnemae Update CHANGELOG.md Signed-off-by: yihuang <[email protected]> fix test don't return nil as empty slice add dryrun mode separete read from iteration add stores flag debug add timestamp api
crypto-org-chain · Nov 12, 2024 · 1970182 · 1970182
1 parent 2cfdca4
commit 1970182
Show file tree

Hide file tree

Showing 9 changed files with 324 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 * [#1670](https://github.com/crypto-org-chain/cronos/pull/1670) Fix state overwrite in debug trace APIs.
 * [#1679](https://github.com/crypto-org-chain/cronos/pull/1679) Include no trace detail on insufficient balance fix.
+* [#1685](https://github.com/crypto-org-chain/cronos/pull/1685) Add command to fix versiondb corrupted data.
 
 *Oct 14, 2024*
 

diff --git a/app/versiondb.go b/app/versiondb.go
@@ -23,6 +23,7 @@ func (app *App) setupVersionDB(
 	if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
 		return nil, err
 	}
+
 	versionDB, err := tsrocksdb.NewStore(dataDir)
 	if err != nil {
 		return nil, err
@@ -34,6 +35,9 @@ func (app *App) setupVersionDB(
 		exposeStoreKeys = append(exposeStoreKeys, storeKey)
 	}
 
+	// see: https://github.com/crypto-org-chain/cronos/issues/1683
+	versionDB.SetSkipVersionZero(true)
+
 	service := versiondb.NewStreamingService(versionDB, exposeStoreKeys)
 	app.SetStreamingService(service)
 

diff --git a/versiondb/client/cmd.go b/versiondb/client/cmd.go
@@ -28,6 +28,7 @@ func ChangeSetGroupCmd(opts Options) *cobra.Command {
 		ChangeSetToVersionDBCmd(),
 		RestoreAppDBCmd(opts),
 		RestoreVersionDBCmd(),
+		FixDataCmd(opts.DefaultStores),
 	)
 	return cmd
 }
diff --git a/versiondb/client/fixdata.go b/versiondb/client/fixdata.go
@@ -0,0 +1,59 @@
+package client
+
+import (
+	"github.com/crypto-org-chain/cronos/versiondb/tsrocksdb"
+	"github.com/linxGnu/grocksdb"
+	"github.com/spf13/cobra"
+)
+
+const (
+	FlagDryRun = "dry-run"
+	FlagStore  = "store"
+)
+
+func FixDataCmd(defaultStores []string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "fixdata <dir>",
+		Args:  cobra.ExactArgs(1),
+		Short: "Fix wrong data in versiondb, see: https://github.com/crypto-org-chain/cronos/issues/1683",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir := args[0]
+			dryRun, err := cmd.Flags().GetBool(FlagDryRun)
+			if err != nil {
+				return err
+			}
+			stores, err := cmd.Flags().GetStringArray(FlagStore)
+			if err != nil {
+				return err
+			}
+			if len(stores) == 0 {
+				stores = defaultStores
+			}
+
+			var (
+				db       *grocksdb.DB
+				cfHandle *grocksdb.ColumnFamilyHandle
+			)
+
+			if dryRun {
+				db, cfHandle, err = tsrocksdb.OpenVersionDBForReadOnly(dir, false)
+			} else {
+				db, cfHandle, err = tsrocksdb.OpenVersionDB(dir)
+			}
+			if err != nil {
+				return err
+			}
+
+			versionDB := tsrocksdb.NewStoreWithDB(db, cfHandle)
+			if err := versionDB.FixData(stores, dryRun); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Bool(FlagDryRun, false, "Dry run, do not write to the database, open the database in read-only mode.")
+	cmd.Flags().StringArray(FlagStore, nil, "Store names to fix, if not specified, all stores will be fixed.")
+	return cmd
+}
diff --git a/versiondb/tsrocksdb/iterator.go b/versiondb/tsrocksdb/iterator.go
@@ -2,8 +2,9 @@ package tsrocksdb
 
 import (
 	"bytes"
+	"encoding/binary"
 
-	"github.com/cosmos/cosmos-sdk/store/types"
+	"github.com/crypto-org-chain/cronos/versiondb"
 	"github.com/linxGnu/grocksdb"
 )
 
@@ -12,11 +13,14 @@ type rocksDBIterator struct {
 	prefix, start, end []byte
 	isReverse          bool
 	isInvalid          bool
+
+	// see: https://github.com/crypto-org-chain/cronos/issues/1683
+	skipVersionZero bool
 }
 
-var _ types.Iterator = (*rocksDBIterator)(nil)
+var _ versiondb.Iterator = (*rocksDBIterator)(nil)
 
-func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, isReverse bool) *rocksDBIterator {
+func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, isReverse bool, skipVersionZero bool) *rocksDBIterator {
 	if isReverse {
 		if end == nil {
 			source.SeekToLast()
@@ -39,14 +43,18 @@ func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, is
 			source.Seek(start)
 		}
 	}
-	return &rocksDBIterator{
-		source:    source,
-		prefix:    prefix,
-		start:     start,
-		end:       end,
-		isReverse: isReverse,
-		isInvalid: false,
+	it := &rocksDBIterator{
+		source:          source,
+		prefix:          prefix,
+		start:           start,
+		end:             end,
+		isReverse:       isReverse,
+		isInvalid:       false,
+		skipVersionZero: skipVersionZero,
 	}
+
+	it.trySkipZeroVersion()
+	return it
 }
 
 // Domain implements Iterator.
@@ -114,6 +122,21 @@ func (itr rocksDBIterator) Next() {
 	} else {
 		itr.source.Next()
 	}
+
+	itr.trySkipZeroVersion()
+}
+
+func (itr rocksDBIterator) Timestamp() []byte {
+	itr.assertIsValid()
+	return moveSliceToBytes(itr.source.Timestamp())
+}
+
+func (itr rocksDBIterator) trySkipZeroVersion() {
+	if itr.skipVersionZero {
+		for itr.Valid() && binary.LittleEndian.Uint64(itr.Timestamp()) == 0 {
+			itr.Next()
+		}
+	}
 }
 
 // Error implements Iterator.

diff --git a/versiondb/tsrocksdb/opts.go b/versiondb/tsrocksdb/opts.go
@@ -64,6 +64,20 @@ func OpenVersionDB(dir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, erro
 	return db, cfHandles[1], nil
 }
 
+// OpenVersionDBForReadOnly open versiondb in readonly mode
+func OpenVersionDBForReadOnly(dir string, errorIfWalFileExists bool) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
+	opts := grocksdb.NewDefaultOptions()
+	db, cfHandles, err := grocksdb.OpenDbForReadOnlyColumnFamilies(
+		opts, dir, []string{"default", VersionDBCFName},
+		[]*grocksdb.Options{opts, NewVersionDBOpts(false)},
+		errorIfWalFileExists,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+	return db, cfHandles[1], nil
+}
+
 // OpenVersionDBAndTrimHistory opens versiondb similar to `OpenVersionDB`,
 // but it also trim the versions newer than target one, can be used for rollback.
 func OpenVersionDBAndTrimHistory(dir string, version int64) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {

diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go
@@ -1,6 +1,7 @@
 package tsrocksdb
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -38,6 +39,9 @@ func init() {
 type Store struct {
 	db       *grocksdb.DB
 	cfHandle *grocksdb.ColumnFamilyHandle
+
+	// see: https://github.com/crypto-org-chain/cronos/issues/1683
+	skipVersionZero bool
 }
 
 func NewStore(dir string) (Store, error) {
@@ -58,6 +62,10 @@ func NewStoreWithDB(db *grocksdb.DB, cfHandle *grocksdb.ColumnFamilyHandle) Stor
 	}
 }
 
+func (s *Store) SetSkipVersionZero(skip bool) {
+	s.skipVersionZero = skip
+}
+
 func (s Store) SetLatestVersion(version int64) error {
 	var ts [TimestampSize]byte
 	binary.LittleEndian.PutUint64(ts[:], uint64(version))
@@ -86,11 +94,23 @@ func (s Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error
 }
 
 func (s Store) GetAtVersionSlice(storeKey string, key []byte, version *int64) (*grocksdb.Slice, error) {
-	return s.db.GetCF(
+	value, ts, err := s.db.GetCFWithTS(
 		newTSReadOptions(version),
 		s.cfHandle,
 		prependStoreKey(storeKey, key),
 	)
+	if err != nil {
+		return nil, err
+	}
+	defer ts.Free()
+
+	if value.Exists() && s.skipVersionZero {
+		if binary.LittleEndian.Uint64(ts.Data()) == 0 {
+			return grocksdb.NewSlice(nil, 0), nil
+		}
+	}
+
+	return value, err
 }
 
 // GetAtVersion implements VersionStore interface
@@ -127,29 +147,25 @@ func (s Store) GetLatestVersion() (int64, error) {
 }
 
 // IteratorAtVersion implements VersionStore interface
-func (s Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) {
-	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
-		return nil, errKeyEmpty
-	}
-
-	prefix := storePrefix(storeKey)
-	start, end = iterateWithPrefix(prefix, start, end)
-
-	itr := s.db.NewIteratorCF(newTSReadOptions(version), s.cfHandle)
-	return newRocksDBIterator(itr, prefix, start, end, false), nil
+func (s Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) (versiondb.Iterator, error) {
+	return s.iteratorAtVersion(storeKey, start, end, version, false)
 }
 
 // ReverseIteratorAtVersion implements VersionStore interface
-func (s Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) {
+func (s Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (versiondb.Iterator, error) {
+	return s.iteratorAtVersion(storeKey, start, end, version, true)
+}
+
+func (s Store) iteratorAtVersion(storeKey string, start, end []byte, version *int64, reverse bool) (versiondb.Iterator, error) {
 	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
 		return nil, errKeyEmpty
 	}
 
 	prefix := storePrefix(storeKey)
-	start, end = iterateWithPrefix(storePrefix(storeKey), start, end)
+	start, end = iterateWithPrefix(prefix, start, end)
 
 	itr := s.db.NewIteratorCF(newTSReadOptions(version), s.cfHandle)
-	return newRocksDBIterator(itr, prefix, start, end, true), nil
+	return newRocksDBIterator(itr, prefix, start, end, reverse, s.skipVersionZero), nil
 }
 
 // FeedChangeSet is used to migrate legacy change sets into versiondb
@@ -216,6 +232,93 @@ func (s Store) Flush() error {
 	)
 }
 
+// FixData fixes wrong data written in versiondb due to rocksdb upgrade, the operation is idempotent.
+// see: https://github.com/crypto-org-chain/cronos/issues/1683
+// call this before `SetSkipVersionZero(true)`.
+func (s Store) FixData(storeNames []string, dryRun bool) error {
+	for _, storeName := range storeNames {
+		if err := s.fixDataStore(storeName, dryRun); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// fixDataStore iterate the wrong data at version 0, parse the timestamp from the key and write it again.
+func (s Store) fixDataStore(storeName string, dryRun bool) error {
+	pairs, err := s.loadWrongData(storeName)
+	if err != nil {
+		return err
+	}
+
+	batch := grocksdb.NewWriteBatch()
+	defer batch.Destroy()
+
+	prefix := storePrefix(storeName)
+	readOpts := grocksdb.NewDefaultReadOptions()
+	defer readOpts.Destroy()
+	for _, pair := range pairs {
+		realKey := cloneAppend(prefix, pair.Key)
+
+		readOpts.SetTimestamp(pair.Timestamp)
+		oldValue, err := s.db.GetCF(readOpts, s.cfHandle, realKey)
+		if err != nil {
+			return err
+		}
+
+		clean := bytes.Equal(oldValue.Data(), pair.Value)
+		oldValue.Free()
+
+		if clean {
+			continue
+		}
+
+		if dryRun {
+			fmt.Printf("fix data: %s, key: %X, ts: %X\n", storeName, pair.Key, pair.Timestamp)
+		} else {
+			batch.PutCFWithTS(s.cfHandle, realKey, pair.Timestamp, pair.Value)
+		}
+	}
+
+	if !dryRun {
+		return s.db.Write(defaultSyncWriteOpts, batch)
+	}
+
+	return nil
+}
+
+type KVPairWithTS struct {
+	Key       []byte
+	Value     []byte
+	Timestamp []byte
+}
+
+func (s Store) loadWrongData(storeName string) ([]KVPairWithTS, error) {
+	var version int64
+	iter, err := s.IteratorAtVersion(storeName, nil, nil, &version)
+	if err != nil {
+		return nil, err
+	}
+	defer iter.Close()
+
+	var pairs []KVPairWithTS
+	for ; iter.Valid(); iter.Next() {
+		key := iter.Key()
+		if len(key) < TimestampSize {
+			return nil, fmt.Errorf("invalid key length: %X, store: %s", key, storeName)
+		}
+
+		pairs = append(pairs, KVPairWithTS{
+			Key:       key[:len(key)-TimestampSize],
+			Timestamp: key[len(key)-TimestampSize:],
+			Value:     iter.Value(),
+		})
+	}
+
+	return pairs, nil
+}
+
 func newTSReadOptions(version *int64) *grocksdb.ReadOptions {
 	var ver uint64
 	if version == nil {