diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f991125..3270486e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased +### Changes +- ([\#74](https://github.com/forbole/juno/pull/74)) Applied `GetMissingHeights()` in `enqueueMissingBlocks()` & in `parse blocks missing` cmd + ## v4.0.0 ### Changes - Updated cosmos/cosmos-sdk to `v0.45.8` diff --git a/cmd/parse/blocks/blocks.go b/cmd/parse/blocks/blocks.go index ea8e4c7a..b8f0a0fb 100644 --- a/cmd/parse/blocks/blocks.go +++ b/cmd/parse/blocks/blocks.go @@ -24,7 +24,7 @@ const ( func newAllCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { cmd := &cobra.Command{ Use: "all", - Short: "Fix missing blocks and transactions in database", + Short: "Reparse blocks and transactions ranged from the given start height to the given end height", Long: fmt.Sprintf(`Refetch all the blocks in the specified range and stores them inside the database. You can specify a custom blocks range by using the %s and %s flags. By default, all the blocks fetched from the node will not be stored inside the database if they are already present. diff --git a/cmd/parse/blocks/cmd.go b/cmd/parse/blocks/cmd.go index e2789d58..f7b4a398 100644 --- a/cmd/parse/blocks/cmd.go +++ b/cmd/parse/blocks/cmd.go @@ -15,6 +15,7 @@ func NewBlocksCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { cmd.AddCommand( newAllCmd(parseConfig), + newMissingCmd(parseConfig), ) return cmd diff --git a/cmd/parse/blocks/missing.go b/cmd/parse/blocks/missing.go new file mode 100644 index 00000000..ee2d84d4 --- /dev/null +++ b/cmd/parse/blocks/missing.go @@ -0,0 +1,52 @@ +package blocks + +import ( + "fmt" + "strconv" + + parsecmdtypes "github.com/forbole/juno/v4/cmd/parse/types" + + "github.com/spf13/cobra" + + "github.com/forbole/juno/v4/parser" + "github.com/forbole/juno/v4/types/config" +) + +// newMissingCmd returns a Cobra command that allows to fix missing blocks in database +func newMissingCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "missing [start height]", + Short: "Refetch all the missing heights in the database starting from the given start height", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + startHeight, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return fmt.Errorf("make sure the given start height is a positive integer") + } + + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parseConfig) + if err != nil { + return err + } + + workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) + worker := parser.NewWorker(workerCtx, nil, 0) + + dbLastHeight, err := parseCtx.Database.GetLastBlockHeight() + if err != nil { + return fmt.Errorf("error while getting DB last block height: %s", err) + } + + for _, k := range parseCtx.Database.GetMissingHeights(startHeight, dbLastHeight) { + err = worker.Process(k) + if err != nil { + return fmt.Errorf("error while re-fetching block %d: %s", k, err) + } + } + + return nil + }, + } + + return cmd +} diff --git a/cmd/start/cmd.go b/cmd/start/cmd.go index c016747c..2c0674a4 100644 --- a/cmd/start/cmd.go +++ b/cmd/start/cmd.go @@ -157,7 +157,7 @@ func enqueueMissingBlocks(exportQueue types.HeightQueue, ctx *parser.Context) { } } else { ctx.Logger.Info("syncing missing blocks...", "latest_block_height", latestBlockHeight) - for i := startHeight; i <= latestBlockHeight; i++ { + for _, i := range ctx.Database.GetMissingHeights(startHeight, latestBlockHeight) { ctx.Logger.Debug("enqueueing missing block", "height", i) exportQueue <- i } diff --git a/database/database.go b/database/database.go index c1d89738..65d09c98 100644 --- a/database/database.go +++ b/database/database.go @@ -20,6 +20,9 @@ type Database interface { // An error is returned if the operation fails. GetLastBlockHeight() (int64, error) + // GetMissingHeights returns a slice of missing block heights between startHeight and endHeight + GetMissingHeights(startHeight, endHeight int64) []int64 + // SaveBlock will be called when a new block is parsed, passing the block itself // and the transactions contained inside that block. // An error is returned if the operation fails. diff --git a/database/postgresql/postgresql.go b/database/postgresql/postgresql.go index 54e9f002..8e6df0c5 100644 --- a/database/postgresql/postgresql.go +++ b/database/postgresql/postgresql.go @@ -96,6 +96,22 @@ func (db *Database) GetLastBlockHeight() (int64, error) { return height, nil } +// GetMissingHeights returns a slice of missing block heights between startHeight and endHeight +func (db *Database) GetMissingHeights(startHeight, endHeight int64) []int64 { + var result []int64 + stmt := `SELECT generate_series($1::int,$2::int) EXCEPT SELECT height FROM block ORDER BY 1;` + err := db.SQL.Select(&result, stmt, startHeight, endHeight) + if err != nil { + return nil + } + + if len(result) == 0 { + return nil + } + + return result +} + // SaveBlock implements database.Database func (db *Database) SaveBlock(block *types.Block) error { sqlStatement := `