Sync from L1

cmd/juno/juno.go

@@ -11,12 +11,17 @@
 	"syscall"
 	"time"
 
+	"github.com/NethermindEth/juno/db/pebble"
+	"github.com/NethermindEth/juno/l1data"
 	"github.com/NethermindEth/juno/node"
+	"github.com/NethermindEth/juno/syncl1"
 	"github.com/NethermindEth/juno/utils"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	_ "go.uber.org/automaxprocs"
+	"tailscale.com/logtail/backoff"
 )
 
 const greeting = `
@@ -142,6 +147,43 @@
 		n.Run(cmd.Context())
 		return nil
 	})
+	endBlock := new(uint64)
+	cmd.AddCommand(newSyncL1Cmd(endBlock, func(cmd *cobra.Command, _ []string) error {
+		if config.Network != utils.Mainnet {
+			return fmt.Errorf("syncing from L1 is only supported on mainnet right now")
+		}
+
+		log, err := utils.NewZapLogger(config.LogLevel, config.Colour)
+		if err != nil {
+			return fmt.Errorf("create logger: %v", err)
+		}
+		database, err := pebble.New(config.DatabasePath, defaultCacheSizeMb, log)
+		if err != nil {
+			return fmt.Errorf("open DB: %v", err)
+		}
+		ethClient, err := ethclient.Dial(config.EthNode)
+		if err != nil {
+			return fmt.Errorf("dial %s: %v", config.EthNode, err)
+		}
+		l1Data, err := l1data.New(ethClient)
+		if err != nil {
+			return fmt.Errorf("create l1 client: %v", err)
+		}
+		l1Data.WithBackoff(backoff.NewBackoff("syncl1", func(format string, a ...any) {
+			log.Warnw(fmt.Sprintf(format, a...))
+		}, time.Minute))
+		fetcher := l1data.NewStateDiffFetcher(l1Data)
+		s, err := syncl1.New(database, l1Data, fetcher, syncl1.MainnetConfig, *endBlock, log)
+		if err != nil {
+			return fmt.Errorf("new l1 synchronizer: %v", err)
+		}
+
+		if err := s.Run(cmd.Context()); err != nil {
+			return fmt.Errorf("sync l1: %v", err)
+		}
+
+		return nil
+	}))
 
 	if err := cmd.ExecuteContext(ctx); err != nil {
 		os.Exit(1)
@@ -168,9 +210,9 @@
 	var cfgFile string
 	var cwdErr error
 
-	// PreRunE populates the configuration struct from the Cobra flags and Viper configuration.
+	// PersistentPreRunE populates the configuration struct from the Cobra flags and Viper configuration.
 	// This is called in step 3 of the process described above.
-	junoCmd.PreRunE = func(cmd *cobra.Command, _ []string) error {
+	junoCmd.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error {
 		// If we couldn't find the current working directory and the database path is empty,
 		// return the error.
 		if cwdErr != nil && config.DatabasePath == "" {
@@ -212,20 +254,20 @@
 	defaultMaxVMs := 3 * runtime.GOMAXPROCS(0)
 
 	junoCmd.Flags().StringVar(&cfgFile, configF, defaultConfig, configFlagUsage)
-	junoCmd.Flags().Var(&defaultLogLevel, logLevelF, logLevelFlagUsage)
+	junoCmd.PersistentFlags().Var(&defaultLogLevel, logLevelF, logLevelFlagUsage)
 	junoCmd.Flags().Bool(httpF, defaultHTTP, httpUsage)
 	junoCmd.Flags().String(httpHostF, defaulHost, httpHostUsage)
 	junoCmd.Flags().Uint16(httpPortF, defaultHTTPPort, httpPortUsage)
 	junoCmd.Flags().Bool(wsF, defaultWS, wsUsage)
 	junoCmd.Flags().String(wsHostF, defaulHost, wsHostUsage)
 	junoCmd.Flags().Uint16(wsPortF, defaultWSPort, wsPortUsage)
-	junoCmd.Flags().String(dbPathF, defaultDBPath, dbPathUsage)
-	junoCmd.Flags().Var(&defaultNetwork, networkF, networkUsage)
-	junoCmd.Flags().String(ethNodeF, defaultEthNode, ethNodeUsage)
+	junoCmd.PersistentFlags().String(dbPathF, defaultDBPath, dbPathUsage)
+	junoCmd.PersistentFlags().Var(&defaultNetwork, networkF, networkUsage)
+	junoCmd.PersistentFlags().String(ethNodeF, defaultEthNode, ethNodeUsage)
 	junoCmd.Flags().Bool(pprofF, defaultPprof, pprofUsage)
 	junoCmd.Flags().String(pprofHostF, defaulHost, pprofHostUsage)
 	junoCmd.Flags().Uint16(pprofPortF, defaultPprofPort, pprofPortUsage)
-	junoCmd.Flags().Bool(colourF, defaultColour, colourUsage)
+	junoCmd.PersistentFlags().Bool(colourF, defaultColour, colourUsage)
 	junoCmd.Flags().Duration(pendingPollIntervalF, defaultPendingPollInterval, pendingPollIntervalUsage)
 	junoCmd.Flags().Bool(p2pF, defaultP2p, p2pUsage)
 	junoCmd.Flags().String(p2pAddrF, defaultP2pAddr, p2PAddrUsage)

cmd/juno/syncl1.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/spf13/cobra"
+)
+
+func newSyncL1Cmd(endBlock *uint64, run func(*cobra.Command, []string) error) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "syncl1 <end-block-number> [flags]",
+		Short: "sync the state from L1 using Juno's database format.",
+		Args:  cobra.ExactArgs(1),
+		PreRunE: func(_ *cobra.Command, args []string) error {
+			var err error
+			*endBlock, err = strconv.ParseUint(args[0], 10, 64)
+			if err != nil {
+				return fmt.Errorf("parse end block number: %v", err)
+			}
+			return nil
+		},
+		RunE: run,
+	}
+	return cmd
+}

cmd/juno/syncl1_pkg_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewSyncL1Cmd(t *testing.T) {
+	tests := map[string]struct {
+		args []string
+		want int // if negative, expect error
+	}{
+		"no args": {
+			args: []string{},
+			want: -1,
+		},
+		"single empty arg": {
+			args: []string{""},
+			want: -1,
+		},
+		"single negative arg": {
+			args: []string{"-1"},
+			want: -1,
+		},
+		"two args": {
+			args: []string{"1", "1"},
+			want: -1,
+		},
+		"zero": {
+			args: []string{"0"},
+			want: 0,
+		},
+		"large number": {
+			args: []string{"1232882"},
+			want: 1232882,
+		},
+	}
+
+	for description, test := range tests {
+		t.Run(description, func(t *testing.T) {
+			var endBlock uint64
+			cmd := newSyncL1Cmd(&endBlock, func(_ *cobra.Command, _ []string) error { return nil })
+			cmd.SetArgs(test.args)
+			err := cmd.Execute()
+			if test.want < 0 {
+				require.Error(t, err)
+				return
+			}
+			require.Equal(t, uint64(test.want), endBlock)
+		})
+	}
+}

db/buckets.go

@@ -32,6 +32,7 @@ const (
 	BlockCommitments
 	Temporary // used temporarily for migrations
 	SchemaIntermediateState
+	EthereumHeight
 )
 
 // Key flattens a prefix and series of byte arrays into a single []byte.

go.mod

@@ -3,14 +3,14 @@ module github.com/NethermindEth/juno
 go 1.21
 
 require (
-	github.com/Masterminds/semver/v3 v3.2.0
+	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/bits-and-blooms/bitset v1.7.0
 	github.com/bits-and-blooms/bloom/v3 v3.4.0
 	github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/ethereum/go-ethereum v1.12.0
-	github.com/fxamacker/cbor/v2 v2.4.0
+	github.com/fxamacker/cbor/v2 v2.5.0
 	github.com/go-playground/validator/v10 v10.11.1
 	github.com/jinzhu/copier v0.3.5
 	github.com/libp2p/go-libp2p v0.31.0
@@ -19,25 +19,24 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/multiformats/go-multiaddr v0.11.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.17.0
 	github.com/rs/cors v1.10.1
 	github.com/sourcegraph/conc v0.2.0
-	github.com/spf13/cobra v1.5.0
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.12.0
+	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/mock v0.3.0
-	go.uber.org/zap v1.25.0
+	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.14.0
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 	nhooyr.io/websocket v1.8.7
+	tailscale.com v1.54.1
 )
 
-require gopkg.in/yaml.v2 v2.4.0 // indirect
-
 require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
@@ -62,12 +61,12 @@ require (
 	github.com/getsentry/sentry-go v0.24.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -83,7 +82,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.3 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/boxo v0.10.0 // indirect
 	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect
@@ -93,7 +92,7 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -110,11 +109,11 @@ require (
 	github.com/libp2p/go-netroute v0.2.1 // indirect
 	github.com/libp2p/go-reuseport v0.4.0 // indirect
 	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/miekg/dns v1.1.55 // indirect
+	github.com/miekg/dns v1.1.56 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
@@ -133,13 +132,12 @@ require (
 	github.com/opencontainers/runtime-spec v1.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
 	github.com/quic-go/quic-go v0.38.1 // indirect
@@ -149,10 +147,10 @@ require (
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/sourcegraph/sourcegraph/lib v0.0.0-20221216004406-749998a2ac74 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.8.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.3.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
@@ -164,6 +162,7 @@ require (
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/fx v1.20.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect