From 661e30ba0f0ee1ce0c21e804064bcbb254f12be4 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Thu, 7 Mar 2024 16:14:22 -0500 Subject: [PATCH 1/9] wip DAB interface improvement --- dataavailability/dataavailability.go | 118 +++++++++++++----- .../datacommittee/datacommittee.go | 20 ++- dataavailability/interfaces.go | 17 ++- etherman/etherman.go | 29 +++-- etherman/interfaces.go | 2 +- etherman/mock_da.go | 22 ++-- jsonrpc/client/zkevm.go | 32 +++++ jsonrpc/endpoints_zkevm.go | 36 ++++++ jsonrpc/endpoints_zkevm.openrpc.json | 35 +++++- jsonrpc/mocks/mock_state.go | 30 +++++ jsonrpc/types/interfaces.go | 1 + jsonrpc/types/types.go | 17 +++ state/interfaces.go | 1 + state/mocks/mock_storage.go | 60 +++++++++ state/pgstatestorage/pgstatestorage.go | 23 ++++ state/pgstatestorage/pgstatestorage_test.go | 33 +++++ test/docker-compose.yml | 9 +- 17 files changed, 419 insertions(+), 66 deletions(-) diff --git a/dataavailability/dataavailability.go b/dataavailability/dataavailability.go index 1ca09a39ef..b3c9f65841 100644 --- a/dataavailability/dataavailability.go +++ b/dataavailability/dataavailability.go @@ -7,12 +7,11 @@ import ( "github.com/0xPolygonHermez/zkevm-node/etherman/types" "github.com/0xPolygonHermez/zkevm-node/log" - "github.com/0xPolygonHermez/zkevm-node/state" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) -const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s" +const unexpectedHashTemplate = "mismatch on transaction data for batch num %d. Expected hash %s, actual hash: %s" // DataAvailability implements an abstract data availability integration type DataAvailability struct { @@ -60,45 +59,69 @@ func (d *DataAvailability) PostSequence(ctx context.Context, sequences []types.S // 1. From local DB // 2. From Sequencer // 3. From DA backend -func (d *DataAvailability) GetBatchL2Data(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) { - found := true - transactionsData, err := d.state.GetBatchL2DataByNumber(d.ctx, batchNum, nil) +func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { + if len(batchNums) != len(batchHashes) { + return nil, fmt.Errorf("invalid L2 batch data retrieval arguments, %d != %d", len(batchNums), len(batchHashes)) + } + + data, err := d.localData(batchNums, batchHashes) if err != nil { - if err == state.ErrNotFound { - found = false + log.Warnf("error retrieving local data for batches %v: %s", batchNums, err.Error()) + } else { + return data, nil + } + + if !d.isTrustedSequencer { + data, err = d.trustedSeqData(batchNums, batchHashes) + if err != nil { + log.Warnf("error retrieving trusted sequencer data for batches %v: %s", batchNums, err.Error()) } else { - return nil, fmt.Errorf("failed to get batch data from state for batch num %d: %w", batchNum, err) + return data, nil } } - actualTransactionsHash := crypto.Keccak256Hash(transactionsData) - if !found || expectedTransactionsHash != actualTransactionsHash { - if found { - log.Warnf(unexpectedHashTemplate, batchNum, expectedTransactionsHash, actualTransactionsHash) - } - if !d.isTrustedSequencer { - log.Info("trying to get data from trusted sequencer") - data, err := d.getDataFromTrustedSequencer(batchNum, expectedTransactionsHash) - if err != nil { - log.Warn("failed to get data from trusted sequencer: %w", err) - } else { - return data, nil - } + return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage) +} + +func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([][]byte, error) { + data, err := d.state.GetBatchL2DataByNumbers(d.ctx, numbers, nil) + if err != nil { + return nil, err + } + var batches [][]byte + for i := 0; i < len(numbers); i++ { + batchNumber := numbers[i] + expectedHash := hashes[i] + batchData, ok := data[batchNumber] + if !ok { + return nil, fmt.Errorf("could not get data locally for batch numbers %v", numbers) } + actualHash := crypto.Keccak256Hash(batchData) + if actualHash != expectedHash { + log.Warnf(unexpectedHashTemplate, batchNumber, expectedHash, actualHash) + } else { + batches = append(batches, batchData) + } + } + return batches, nil +} - log.Info("trying to get data from the data availability backend") - data, err := d.backend.GetBatchL2Data(batchNum, expectedTransactionsHash) - if err != nil { - log.Error("failed to get data from the data availability backend: %w", err) - if d.isTrustedSequencer { - return nil, fmt.Errorf("data not found on the local DB nor on any data committee member") - } else { - return nil, fmt.Errorf("data not found on the local DB, nor from the trusted sequencer nor on any data committee member") - } +func (d *DataAvailability) trustedSeqData(numbers []uint64, hashes []common.Hash) ([][]byte, error) { + data, err := d.getBatchesDataFromTrustedSequencer(numbers, hashes) + if err != nil { + return nil, err + } + var batches [][]byte + for i := 0; i < len(numbers); i++ { + batchNumber := numbers[i] + // hash has already been checked + batchData, ok := data[batchNumber] + if !ok { + continue } - return data, nil + batches[i] = batchData } - return transactionsData, nil + return batches, nil } func (d *DataAvailability) getDataFromTrustedSequencer(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) { @@ -114,3 +137,34 @@ func (d *DataAvailability) getDataFromTrustedSequencer(batchNum uint64, expected } return b.BatchL2Data, nil } + +func (d *DataAvailability) getBatchesDataFromTrustedSequencer(batchNums []uint64, expectedHashes []common.Hash) (map[uint64][]byte, error) { + if len(batchNums) != len(expectedHashes) { + return nil, fmt.Errorf("invalid arguments, len of batch numbers does not equal length of expected hashes: %d != %d", + len(batchNums), len(expectedHashes)) + } + var nums []*big.Int + for _, n := range batchNums { + nums = append(nums, new(big.Int).SetUint64(n)) + } + batchData, err := d.zkEVMClient.BatchesByNumbers(d.ctx, nums) + if err != nil { + return nil, fmt.Errorf("failed to get batches %v data from trusted sequencer: %w", batchNums, err) + } + result := make(map[uint64][]byte) + for i := 0; i < len(batchNums); i++ { + number := batchNums[i] + batch := batchData[i] + if batch.Empty { + continue + } + expectedTransactionsHash := expectedHashes[i] + actualTransactionsHash := crypto.Keccak256Hash(batch.BatchL2Data) + if expectedTransactionsHash != actualTransactionsHash { + log.Warnf(unexpectedHashTemplate, number, expectedTransactionsHash, actualTransactionsHash) + continue + } + result[number] = batch.BatchL2Data + } + return result, nil +} diff --git a/dataavailability/datacommittee/datacommittee.go b/dataavailability/datacommittee/datacommittee.go index 1d638e03e5..363dab301b 100644 --- a/dataavailability/datacommittee/datacommittee.go +++ b/dataavailability/datacommittee/datacommittee.go @@ -20,7 +20,7 @@ import ( "golang.org/x/net/context" ) -const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s" +const unexpectedHashTemplate = "missmatch on transaction data. Expected hash %s, actual hash: %s" // DataCommitteeMember represents a member of the Data Committee type DataCommitteeMember struct { @@ -87,8 +87,22 @@ func (d *DataCommitteeBackend) Init() error { return nil } +func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { + // TODO: optimize this on the DAC side by implementing a multi batch retrieve api + // FIXME: how to use dataAvailabilityMessage ? + var batchData [][]byte + for _, h := range hashes { + data, err := d.GetBatchL2Data(h) + if err != nil { + return nil, err + } + batchData = append(batchData, data) + } + return batchData, nil +} + // GetBatchL2Data returns the data from the DAC. It checks that it matches with the expected hash -func (d *DataCommitteeBackend) GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) { +func (d *DataCommitteeBackend) GetBatchL2Data(hash common.Hash) ([]byte, error) { intialMember := d.selectedCommitteeMember found := false for !found && intialMember != -1 { @@ -110,7 +124,7 @@ func (d *DataCommitteeBackend) GetBatchL2Data(batchNum uint64, hash common.Hash) actualTransactionsHash := crypto.Keccak256Hash(data) if actualTransactionsHash != hash { unexpectedHash := fmt.Errorf( - unexpectedHashTemplate, batchNum, hash, actualTransactionsHash, + unexpectedHashTemplate, hash, actualTransactionsHash, ) log.Warnf( "error getting data from DAC node %s at %s: %s", diff --git a/dataavailability/interfaces.go b/dataavailability/interfaces.go index 441829fafb..efe1dde338 100644 --- a/dataavailability/interfaces.go +++ b/dataavailability/interfaces.go @@ -12,13 +12,14 @@ import ( type stateInterface interface { GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) + GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error) } // BatchDataProvider is used to retrieve batch data type BatchDataProvider interface { // GetBatchL2Data retrieve the data of a batch from the DA backend. The returned data must be the pre-image of the hash - GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) + GetBatchL2Data(batchNum []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) } // SequenceSender is used to send provided sequence of batches @@ -28,11 +29,18 @@ type SequenceSender interface { PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) } -// DABackender is the interface needed to implement in order to -// integrate a DA service -type DABackender interface { +type SequenceRetriever interface { + GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) +} + +type DataManager interface { BatchDataProvider SequenceSender +} + +type DABackender interface { + SequenceRetriever + SequenceSender // Init initializes the DABackend Init() error } @@ -40,4 +48,5 @@ type DABackender interface { // ZKEVMClientTrustedBatchesGetter contains the methods required to interact with zkEVM-RPC type ZKEVMClientTrustedBatchesGetter interface { BatchByNumber(ctx context.Context, number *big.Int) (*types.Batch, error) + BatchesByNumbers(ctx context.Context, numbers []*big.Int) ([]*types.BatchData, error) } diff --git a/etherman/etherman.go b/etherman/etherman.go index a75fb8fc23..3a1cba9abd 100644 --- a/etherman/etherman.go +++ b/etherman/etherman.go @@ -1264,9 +1264,10 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add if err != nil { return nil, err } - var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData + switch method.Name { case "rollup": // TODO: put correct value + var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData err = json.Unmarshal(bytedata, &sequences) if err != nil { return nil, err @@ -1295,18 +1296,25 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add return nil, err } coinbase := (data[1]).(common.Address) + dataAvailabilityMessage := (data[2]).([]byte) // TODO: is this right??? sequencedBatches := make([]SequencedBatch, len(sequencesValidium)) - for i, seq := range sequencesValidium { + var batchNums []uint64 + var hashes []common.Hash + for i, _ := range sequencesValidium { bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1)) - batchL2Data, err := da.GetBatchL2Data(bn, sequencesValidium[i].TransactionsHash) - if err != nil { - return nil, err - } + batchNums = append(batchNums, bn) + hashes = append(hashes, sequencesValidium[i].TransactionsHash) + } + batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage) + if err != nil { + return nil, err + } + for i, bn := range batchNums { s := polygonzkevm.PolygonRollupBaseEtrogBatchData{ - Transactions: batchL2Data, // TODO: get data from DA - ForcedGlobalExitRoot: seq.ForcedGlobalExitRoot, - ForcedTimestamp: seq.ForcedTimestamp, - ForcedBlockHashL1: seq.ForcedBlockHashL1, + Transactions: batchL2Data[i], + ForcedGlobalExitRoot: sequencesValidium[i].ForcedGlobalExitRoot, + ForcedTimestamp: sequencesValidium[i].ForcedTimestamp, + ForcedBlockHashL1: sequencesValidium[i].ForcedBlockHashL1, } sequencedBatches[i] = SequencedBatch{ BatchNumber: bn, @@ -1318,7 +1326,6 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add PolygonRollupBaseEtrogBatchData: &s, } } - return sequencedBatches, nil default: return nil, fmt.Errorf("unexpected method called in sequence batches transaction: %s", method.RawName) diff --git a/etherman/interfaces.go b/etherman/interfaces.go index 07d86de3d7..24b97ee574 100644 --- a/etherman/interfaces.go +++ b/etherman/interfaces.go @@ -3,5 +3,5 @@ package etherman import "github.com/ethereum/go-ethereum/common" type dataAvailabilityProvider interface { - GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) + GetBatchL2Data(batchNum []uint64, hash []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) } diff --git a/etherman/mock_da.go b/etherman/mock_da.go index cb19f298d2..b6d44d6414 100644 --- a/etherman/mock_da.go +++ b/etherman/mock_da.go @@ -13,29 +13,29 @@ type daMock struct { mock.Mock } -// GetBatchL2Data provides a mock function with given fields: batchNum, hash -func (_m *daMock) GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) { - ret := _m.Called(batchNum, hash) +// GetBatchL2Data provides a mock function with given fields: batchNum, hash, dataAvailabilityMessage +func (_m *daMock) GetBatchL2Data(batchNum []uint64, hash []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { + ret := _m.Called(batchNum, hash, dataAvailabilityMessage) if len(ret) == 0 { panic("no return value specified for GetBatchL2Data") } - var r0 []byte + var r0 [][]byte var r1 error - if rf, ok := ret.Get(0).(func(uint64, common.Hash) ([]byte, error)); ok { - return rf(batchNum, hash) + if rf, ok := ret.Get(0).(func([]uint64, []common.Hash, []byte) ([][]byte, error)); ok { + return rf(batchNum, hash, dataAvailabilityMessage) } - if rf, ok := ret.Get(0).(func(uint64, common.Hash) []byte); ok { - r0 = rf(batchNum, hash) + if rf, ok := ret.Get(0).(func([]uint64, []common.Hash, []byte) [][]byte); ok { + r0 = rf(batchNum, hash, dataAvailabilityMessage) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) + r0 = ret.Get(0).([][]byte) } } - if rf, ok := ret.Get(1).(func(uint64, common.Hash) error); ok { - r1 = rf(batchNum, hash) + if rf, ok := ret.Get(1).(func([]uint64, []common.Hash, []byte) error); ok { + r1 = rf(batchNum, hash, dataAvailabilityMessage) } else { r1 = ret.Error(1) } diff --git a/jsonrpc/client/zkevm.go b/jsonrpc/client/zkevm.go index 7bd6be3332..d87ffdc0df 100644 --- a/jsonrpc/client/zkevm.go +++ b/jsonrpc/client/zkevm.go @@ -58,6 +58,38 @@ func (c *Client) BatchByNumber(ctx context.Context, number *big.Int) (*types.Bat return result, nil } +func (c *Client) BatchesByNumbers(_ context.Context, numbers []*big.Int) ([]*types.BatchData, error) { + var list []types.BatchNumber + for _, n := range numbers { + list = append(list, types.BatchNumber(n.Int64())) + } + if len(list) == 0 { + list = append(list, types.LatestBatchNumber) + } + + var batchNumbers []string + for _, n := range list { + batchNumbers = append(batchNumbers, n.StringOrHex()) + } + + response, err := JSONRPCCall(c.url, "zkevm_getBatchDataByNumbers", batchNumbers, true) + if err != nil { + return nil, err + } + + if response.Error != nil { + return nil, response.Error.RPCError() + } + + var result *types.BatchDataResult + err = json.Unmarshal(response.Result, &result) + if err != nil { + return nil, err + } + + return result.Data, nil +} + // ExitRootsByGER returns the exit roots accordingly to the provided Global Exit Root func (c *Client) ExitRootsByGER(ctx context.Context, globalExitRoot common.Hash) (*types.ExitRoots, error) { response, err := JSONRPCCall(c.url, "zkevm_getExitRootsByGER", globalExitRoot.String()) diff --git a/jsonrpc/endpoints_zkevm.go b/jsonrpc/endpoints_zkevm.go index f4c6020ba8..7c3e17a555 100644 --- a/jsonrpc/endpoints_zkevm.go +++ b/jsonrpc/endpoints_zkevm.go @@ -204,6 +204,42 @@ func (z *ZKEVMEndpoints) GetBatchByNumber(batchNumber types.BatchNumber, fullTx }) } +// GetBatchDataByNumbers returns the batch data for batches by numbers +func (z *ZKEVMEndpoints) GetBatchDataByNumbers(filter types.BatchFilter) (interface{}, types.Error) { + return z.txMan.NewDbTxScope(z.state, func(ctx context.Context, dbTx pgx.Tx) (interface{}, types.Error) { + var batchNumbers []uint64 + for _, bn := range filter.Numbers { + n, rpcErr := bn.GetNumericBatchNumber(ctx, z.state, z.etherman, dbTx) + if rpcErr != nil { + return nil, rpcErr + } + batchNumbers = append(batchNumbers, n) + } + + batchesData, err := z.state.GetBatchL2DataByNumbers(ctx, batchNumbers, dbTx) + if errors.Is(err, state.ErrNotFound) { + return nil, nil + } else if err != nil { + return RPCErrorResponse(types.DefaultErrorCode, + fmt.Sprintf("couldn't load batch data from state by numbers %v", filter.Numbers), err, true) + } + + var ret []*types.BatchData + for _, n := range batchNumbers { + data := &types.BatchData{Number: types.ArgUint64(n)} + if b, ok := batchesData[n]; ok { + data.BatchL2Data = b + data.Empty = false + } else { + data.Empty = true + } + ret = append(ret, data) + } + + return types.BatchDataResult{Data: ret}, nil + }) +} + // GetFullBlockByNumber returns information about a block by block number func (z *ZKEVMEndpoints) GetFullBlockByNumber(number types.BlockNumber, fullTx bool) (interface{}, types.Error) { return z.txMan.NewDbTxScope(z.state, func(ctx context.Context, dbTx pgx.Tx) (interface{}, types.Error) { diff --git a/jsonrpc/endpoints_zkevm.openrpc.json b/jsonrpc/endpoints_zkevm.openrpc.json index d795e0f1cb..5875d2fc76 100644 --- a/jsonrpc/endpoints_zkevm.openrpc.json +++ b/jsonrpc/endpoints_zkevm.openrpc.json @@ -166,6 +166,18 @@ } ] }, + { + "name": "zkevm_getBatchDataByNumbers", + "summary": "Gets batch data for a given list of batch numbers", + "params": [ + { + "$ref": "#/components/contentDescriptors/BatchFilter" + } + ], + "result": { + "$ref": "#/components/contentDescriptors/BatchDataResult" + } + }, { "name": "zkevm_getBatchByNumber", "summary": "Gets a batch for a given number", @@ -512,6 +524,27 @@ "$ref": "#/components/schemas/Batch" } }, + "BatchFilter": { + "name": "filter", + "description": "batch filter", + "schema": { + "$ref": "#/components/schemas/BatchFilter" + } + }, + "BatchData": { + "name": "batchData", + "description": "batch data", + "schema": { + "$ref": "#/components/schemas/BatchData" + } + }, + "BatchDataResult": { + "name": "batchDataResult", + "description": "batch data result", + "schema": { + "$ref": "#/components/schemas/BatchDataResult" + } + }, "Block": { "name": "block", "summary": "A block", @@ -1440,4 +1473,4 @@ } } } -} \ No newline at end of file +} diff --git a/jsonrpc/mocks/mock_state.go b/jsonrpc/mocks/mock_state.go index 36f552fe65..05e596d513 100644 --- a/jsonrpc/mocks/mock_state.go +++ b/jsonrpc/mocks/mock_state.go @@ -211,6 +211,36 @@ func (_m *StateMock) GetBatchByNumber(ctx context.Context, batchNumber uint64, d return r0, r1 } +// GetBatchL2DataByNumbers provides a mock function with given fields: ctx, batchNumbers, dbTx +func (_m *StateMock) GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) { + ret := _m.Called(ctx, batchNumbers, dbTx) + + if len(ret) == 0 { + panic("no return value specified for GetBatchL2DataByNumbers") + } + + var r0 map[uint64][]byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []uint64, pgx.Tx) (map[uint64][]byte, error)); ok { + return rf(ctx, batchNumbers, dbTx) + } + if rf, ok := ret.Get(0).(func(context.Context, []uint64, pgx.Tx) map[uint64][]byte); ok { + r0 = rf(ctx, batchNumbers, dbTx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[uint64][]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []uint64, pgx.Tx) error); ok { + r1 = rf(ctx, batchNumbers, dbTx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetBatchTimestamp provides a mock function with given fields: ctx, batchNumber, forcedForkId, dbTx func (_m *StateMock) GetBatchTimestamp(ctx context.Context, batchNumber uint64, forcedForkId *uint64, dbTx pgx.Tx) (*time.Time, error) { ret := _m.Called(ctx, batchNumber, forcedForkId, dbTx) diff --git a/jsonrpc/types/interfaces.go b/jsonrpc/types/interfaces.go index 5c3322a637..bb573744fd 100644 --- a/jsonrpc/types/interfaces.go +++ b/jsonrpc/types/interfaces.go @@ -65,6 +65,7 @@ type StateInterface interface { GetLastVerifiedBatch(ctx context.Context, dbTx pgx.Tx) (*state.VerifiedBatch, error) GetLastBatchNumber(ctx context.Context, dbTx pgx.Tx) (uint64, error) GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error) + GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) GetTransactionsByBatchNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (txs []types.Transaction, effectivePercentages []uint8, err error) GetVirtualBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.VirtualBatch, error) GetVerifiedBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.VerifiedBatch, error) diff --git a/jsonrpc/types/types.go b/jsonrpc/types/types.go index 55a6a900f7..b636d4db73 100644 --- a/jsonrpc/types/types.go +++ b/jsonrpc/types/types.go @@ -446,6 +446,23 @@ func NewBatch(ctx context.Context, st StateInterface, batch *state.Batch, virtua return res, nil } +// BatchFilter is a list of batch numbers to retrieve +type BatchFilter struct { + Numbers []BatchNumber `json:"numbers"` +} + +// BatchData is an abbreviated structure that only contains the number and L2 batch data +type BatchData struct { + Number ArgUint64 `json:"number"` + BatchL2Data ArgBytes `json:"batchL2Data,omitempty"` + Empty bool `json:"empty"` +} + +// BatchDataResult is a list of BatchData for a BatchFilter +type BatchDataResult struct { + Data []*BatchData `json:"data"` +} + // TransactionOrHash for union type of transaction and types.Hash type TransactionOrHash struct { Hash *common.Hash diff --git a/state/interfaces.go b/state/interfaces.go index 353ed9c7c1..697e2236a0 100644 --- a/state/interfaces.go +++ b/state/interfaces.go @@ -150,6 +150,7 @@ type storage interface { GetVirtualBatchParentHash(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (common.Hash, error) GetForcedBatchParentHash(ctx context.Context, forcedBatchNumber uint64, dbTx pgx.Tx) (common.Hash, error) GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) + GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) GetLatestBatchGlobalExitRoot(ctx context.Context, dbTx pgx.Tx) (common.Hash, error) GetL2TxHashByTxHash(ctx context.Context, hash common.Hash, dbTx pgx.Tx) (*common.Hash, error) GetSyncInfoData(ctx context.Context, dbTx pgx.Tx) (SyncInfoDataOnStorage, error) diff --git a/state/mocks/mock_storage.go b/state/mocks/mock_storage.go index 538c34e1c9..3c5731175a 100644 --- a/state/mocks/mock_storage.go +++ b/state/mocks/mock_storage.go @@ -1765,6 +1765,66 @@ func (_c *StorageMock_GetBatchL2DataByNumber_Call) RunAndReturn(run func(context return _c } +// GetBatchL2DataByNumbers provides a mock function with given fields: ctx, batchNumbers, dbTx +func (_m *StorageMock) GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) { + ret := _m.Called(ctx, batchNumbers, dbTx) + + if len(ret) == 0 { + panic("no return value specified for GetBatchL2DataByNumbers") + } + + var r0 map[uint64][]byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []uint64, pgx.Tx) (map[uint64][]byte, error)); ok { + return rf(ctx, batchNumbers, dbTx) + } + if rf, ok := ret.Get(0).(func(context.Context, []uint64, pgx.Tx) map[uint64][]byte); ok { + r0 = rf(ctx, batchNumbers, dbTx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[uint64][]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []uint64, pgx.Tx) error); ok { + r1 = rf(ctx, batchNumbers, dbTx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// StorageMock_GetBatchL2DataByNumbers_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBatchL2DataByNumbers' +type StorageMock_GetBatchL2DataByNumbers_Call struct { + *mock.Call +} + +// GetBatchL2DataByNumbers is a helper method to define mock.On call +// - ctx context.Context +// - batchNumbers []uint64 +// - dbTx pgx.Tx +func (_e *StorageMock_Expecter) GetBatchL2DataByNumbers(ctx interface{}, batchNumbers interface{}, dbTx interface{}) *StorageMock_GetBatchL2DataByNumbers_Call { + return &StorageMock_GetBatchL2DataByNumbers_Call{Call: _e.mock.On("GetBatchL2DataByNumbers", ctx, batchNumbers, dbTx)} +} + +func (_c *StorageMock_GetBatchL2DataByNumbers_Call) Run(run func(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx)) *StorageMock_GetBatchL2DataByNumbers_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]uint64), args[2].(pgx.Tx)) + }) + return _c +} + +func (_c *StorageMock_GetBatchL2DataByNumbers_Call) Return(_a0 map[uint64][]byte, _a1 error) *StorageMock_GetBatchL2DataByNumbers_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *StorageMock_GetBatchL2DataByNumbers_Call) RunAndReturn(run func(context.Context, []uint64, pgx.Tx) (map[uint64][]byte, error)) *StorageMock_GetBatchL2DataByNumbers_Call { + _c.Call.Return(run) + return _c +} + // GetBatchNumberOfL2Block provides a mock function with given fields: ctx, blockNumber, dbTx func (_m *StorageMock) GetBatchNumberOfL2Block(ctx context.Context, blockNumber uint64, dbTx pgx.Tx) (uint64, error) { ret := _m.Called(ctx, blockNumber, dbTx) diff --git a/state/pgstatestorage/pgstatestorage.go b/state/pgstatestorage/pgstatestorage.go index 090aee86c7..891ae7546f 100644 --- a/state/pgstatestorage/pgstatestorage.go +++ b/state/pgstatestorage/pgstatestorage.go @@ -370,3 +370,26 @@ func (p *PostgresStorage) GetBatchL2DataByNumber(ctx context.Context, batchNumbe } return batchL2Data, nil } + +// GetBatchL2DataByNumbers returns the batch L2 data of the given batch numbers. +func (p *PostgresStorage) GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) { + const getBatchL2DataByBatchNumber = "SELECT batch_num, raw_txs_data FROM state.batch WHERE batch_num = ANY($1)" + q := p.getExecQuerier(dbTx) + rows, err := q.Query(ctx, getBatchL2DataByBatchNumber, batchNumbers) + if errors.Is(err, pgx.ErrNoRows) { + return nil, state.ErrNotFound + } else if err != nil { + return nil, err + } + batchL2DataMap := make(map[uint64][]byte) + for rows.Next() { + var batchNum uint64 + var batchL2Data []byte + err := rows.Scan(&batchNum, &batchL2Data) + if err != nil { + return nil, err + } + batchL2DataMap[batchNum] = batchL2Data + } + return batchL2DataMap, nil +} diff --git a/state/pgstatestorage/pgstatestorage_test.go b/state/pgstatestorage/pgstatestorage_test.go index d40f6c2771..65a03d9603 100644 --- a/state/pgstatestorage/pgstatestorage_test.go +++ b/state/pgstatestorage/pgstatestorage_test.go @@ -1154,6 +1154,39 @@ func TestGetBatchL2DataByNumber(t *testing.T) { actualData, err := testState.GetBatchL2DataByNumber(ctx, batchNum, tx) require.NoError(t, err) assert.Equal(t, expectedData, actualData) + + multiGet := []uint64{uint64(4), uint64(5), uint64(6)} + allData, err := testState.GetBatchL2DataByNumbers(ctx, multiGet, tx) + require.NoError(t, err) + require.Equal(t, expectedData, allData[uint64(5)]) +} + +func TestGetBatchL2DataByNumbers(t *testing.T) { + initOrResetDB() + ctx := context.Background() + tx, err := testState.BeginStateTransaction(ctx) + require.NoError(t, err) + defer func() { require.NoError(t, tx.Commit(ctx)) }() + + var i1, i2, i3, i4 = uint64(1), uint64(2), uint64(3), uint64(4) + var d1, d2 = []byte("foobar"), []byte("dingbat") + + const insertBatch = "INSERT INTO state.batch (batch_num, raw_txs_data) VALUES ($1, $2)" + _, err = tx.Exec(ctx, insertBatch, i1, d1) + require.NoError(t, err) + _, err = tx.Exec(ctx, insertBatch, i2, d2) + require.NoError(t, err) + _, err = tx.Exec(ctx, insertBatch, i3, nil) + require.NoError(t, err) + + allData, err := testState.GetBatchL2DataByNumbers(ctx, []uint64{i1, i2, i3, i4}, tx) + require.NoError(t, err) + assert.Equal(t, d1, allData[i1]) + assert.Equal(t, d2, allData[i2]) + assert.Nil(t, allData[i3]) + + _, ok := allData[i4] + assert.False(t, ok) } func createL1InfoTreeExitRootStorageEntryForTest(blockNumber uint64, index uint32) *state.L1InfoTreeExitRootStorageEntry { diff --git a/test/docker-compose.yml b/test/docker-compose.yml index ff46d08050..b012e8c1d9 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.5" networks: default: name: zkevm - + services: grafana: container_name: grafana @@ -513,12 +513,15 @@ services: zkevm-prover: container_name: zkevm-prover + platform: linux/amd64 image: hermeznetwork/zkevm-prover:v4.0.14 ports: - 50061:50061 # MT - 50071:50071 # Executor volumes: - ./config/test.prover.config.json:/usr/src/app/config.json + environment: + - EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1 command: > zkProver -c /usr/src/app/config.json @@ -628,7 +631,7 @@ services: zkevm-sh: container_name: zkevm-sh image: zkevm-node - stdin_open: true + stdin_open: true tty: true environment: - ZKEVM_NODE_STATE_DB_HOST=zkevm-state-db @@ -685,4 +688,4 @@ services: command: - "postgres" - "-N" - - "500" \ No newline at end of file + - "500" From 2f3a578383450a991f2eed9e246e45ec424303d7 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Fri, 8 Mar 2024 12:00:07 -0500 Subject: [PATCH 2/9] apply comments & lint --- dataavailability/dataavailability.go | 61 +++++-------------- .../datacommittee/datacommittee.go | 1 + dataavailability/interfaces.go | 4 ++ etherman/etherman.go | 10 ++- jsonrpc/client/zkevm.go | 2 + 5 files changed, 28 insertions(+), 50 deletions(-) diff --git a/dataavailability/dataavailability.go b/dataavailability/dataavailability.go index b3c9f65841..7a52e84823 100644 --- a/dataavailability/dataavailability.go +++ b/dataavailability/dataavailability.go @@ -65,16 +65,14 @@ func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []comm } data, err := d.localData(batchNums, batchHashes) - if err != nil { - log.Warnf("error retrieving local data for batches %v: %s", batchNums, err.Error()) - } else { + if err == nil { return data, nil } if !d.isTrustedSequencer { - data, err = d.trustedSeqData(batchNums, batchHashes) + data, err = d.trustedSequencerData(batchNums, batchHashes) if err != nil { - log.Warnf("error retrieving trusted sequencer data for batches %v: %s", batchNums, err.Error()) + log.Warnf("trusted sequencer failed to return data for batches %v: %s", batchNums, err.Error()) } else { return data, nil } @@ -83,6 +81,7 @@ func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []comm return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage) } +// localData retrieves batches from local database and returns an error unless all are found func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([][]byte, error) { data, err := d.state.GetBatchL2DataByNumbers(d.ctx, numbers, nil) if err != nil { @@ -94,7 +93,7 @@ func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([] expectedHash := hashes[i] batchData, ok := data[batchNumber] if !ok { - return nil, fmt.Errorf("could not get data locally for batch numbers %v", numbers) + return nil, fmt.Errorf("missing batch %v", batchNumber) } actualHash := crypto.Keccak256Hash(batchData) if actualHash != expectedHash { @@ -106,39 +105,8 @@ func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([] return batches, nil } -func (d *DataAvailability) trustedSeqData(numbers []uint64, hashes []common.Hash) ([][]byte, error) { - data, err := d.getBatchesDataFromTrustedSequencer(numbers, hashes) - if err != nil { - return nil, err - } - var batches [][]byte - for i := 0; i < len(numbers); i++ { - batchNumber := numbers[i] - // hash has already been checked - batchData, ok := data[batchNumber] - if !ok { - continue - } - batches[i] = batchData - } - return batches, nil -} - -func (d *DataAvailability) getDataFromTrustedSequencer(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) { - b, err := d.zkEVMClient.BatchByNumber(d.ctx, new(big.Int).SetUint64(batchNum)) - if err != nil { - return nil, fmt.Errorf("failed to get batch num %d from trusted sequencer: %w", batchNum, err) - } - actualTransactionsHash := crypto.Keccak256Hash(b.BatchL2Data) - if expectedTransactionsHash != actualTransactionsHash { - return nil, fmt.Errorf( - unexpectedHashTemplate, batchNum, expectedTransactionsHash, actualTransactionsHash, - ) - } - return b.BatchL2Data, nil -} - -func (d *DataAvailability) getBatchesDataFromTrustedSequencer(batchNums []uint64, expectedHashes []common.Hash) (map[uint64][]byte, error) { +// trustedSequencerData retrieved batch data from the trusted sequencer and returns an error unless all are found +func (d *DataAvailability) trustedSequencerData(batchNums []uint64, expectedHashes []common.Hash) ([][]byte, error) { if len(batchNums) != len(expectedHashes) { return nil, fmt.Errorf("invalid arguments, len of batch numbers does not equal length of expected hashes: %d != %d", len(batchNums), len(expectedHashes)) @@ -149,22 +117,21 @@ func (d *DataAvailability) getBatchesDataFromTrustedSequencer(batchNums []uint64 } batchData, err := d.zkEVMClient.BatchesByNumbers(d.ctx, nums) if err != nil { - return nil, fmt.Errorf("failed to get batches %v data from trusted sequencer: %w", batchNums, err) + return nil, err } - result := make(map[uint64][]byte) + if len(batchData) != len(batchNums) { + return nil, fmt.Errorf("missing batch data, expected %d, got %d", len(batchNums), len(batchData)) + } + var result [][]byte for i := 0; i < len(batchNums); i++ { number := batchNums[i] batch := batchData[i] - if batch.Empty { - continue - } expectedTransactionsHash := expectedHashes[i] actualTransactionsHash := crypto.Keccak256Hash(batch.BatchL2Data) if expectedTransactionsHash != actualTransactionsHash { - log.Warnf(unexpectedHashTemplate, number, expectedTransactionsHash, actualTransactionsHash) - continue + return nil, fmt.Errorf(unexpectedHashTemplate, number, expectedTransactionsHash, actualTransactionsHash) } - result[number] = batch.BatchL2Data + result = append(result, batch.BatchL2Data) } return result, nil } diff --git a/dataavailability/datacommittee/datacommittee.go b/dataavailability/datacommittee/datacommittee.go index 363dab301b..55cdce3c51 100644 --- a/dataavailability/datacommittee/datacommittee.go +++ b/dataavailability/datacommittee/datacommittee.go @@ -87,6 +87,7 @@ func (d *DataCommitteeBackend) Init() error { return nil } +// GetSequence gets backend data one hash at a time. This should be optimized on the DAC side to get them all at once. func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { // TODO: optimize this on the DAC side by implementing a multi batch retrieve api // FIXME: how to use dataAvailabilityMessage ? diff --git a/dataavailability/interfaces.go b/dataavailability/interfaces.go index efe1dde338..16e67a22a2 100644 --- a/dataavailability/interfaces.go +++ b/dataavailability/interfaces.go @@ -29,15 +29,19 @@ type SequenceSender interface { PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) } +// SequenceRetriever is used to retrieve batch data type SequenceRetriever interface { + // GetSequence retrieves the sequence data from the data availability backend GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) } +// DataManager is an interface for components that send and retrieve batch data type DataManager interface { BatchDataProvider SequenceSender } +// DABackender is an interface for data committee components that store and retrieve batch data type DABackender interface { SequenceRetriever SequenceSender diff --git a/etherman/etherman.go b/etherman/etherman.go index 3a1cba9abd..0e80820510 100644 --- a/etherman/etherman.go +++ b/etherman/etherman.go @@ -1300,10 +1300,14 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add sequencedBatches := make([]SequencedBatch, len(sequencesValidium)) var batchNums []uint64 var hashes []common.Hash - for i, _ := range sequencesValidium { + for i := range sequencesValidium { bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1)) - batchNums = append(batchNums, bn) - hashes = append(hashes, sequencesValidium[i].TransactionsHash) + // ForcedBatches are skipped here. They are not currently enabled for validium, but once they are supported, + // their data must come from the state.forced_batch table + if sequencesValidium[i].ForcedTimestamp != 0 { + batchNums = append(batchNums, bn) + hashes = append(hashes, sequencesValidium[i].TransactionsHash) + } } batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage) if err != nil { diff --git a/jsonrpc/client/zkevm.go b/jsonrpc/client/zkevm.go index d87ffdc0df..a1937d5edb 100644 --- a/jsonrpc/client/zkevm.go +++ b/jsonrpc/client/zkevm.go @@ -58,6 +58,8 @@ func (c *Client) BatchByNumber(ctx context.Context, number *big.Int) (*types.Bat return result, nil } +// BatchesByNumbers returns batches from the current canonical chain by batch numbers. If the list is empty, the last +// known batch is returned as a list. func (c *Client) BatchesByNumbers(_ context.Context, numbers []*big.Int) ([]*types.BatchData, error) { var list []types.BatchNumber for _, n := range numbers { From 1ee11e2a0a7276e0d8f474e23477b4add2e3a96b Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Fri, 8 Mar 2024 12:07:09 -0500 Subject: [PATCH 3/9] logging --- dataavailability/dataavailability.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dataavailability/dataavailability.go b/dataavailability/dataavailability.go index 7a52e84823..489c4c4a3d 100644 --- a/dataavailability/dataavailability.go +++ b/dataavailability/dataavailability.go @@ -97,7 +97,9 @@ func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([] } actualHash := crypto.Keccak256Hash(batchData) if actualHash != expectedHash { - log.Warnf(unexpectedHashTemplate, batchNumber, expectedHash, actualHash) + err = fmt.Errorf(unexpectedHashTemplate, batchNumber, expectedHash, actualHash) + log.Warnf("wrong local data for hash: %s", err.Error()) + return nil, err } else { batches = append(batches, batchData) } From ef79ae08b1b709da1e8b87f2125a7c7c0ce25307 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Mon, 11 Mar 2024 12:45:24 -0400 Subject: [PATCH 4/9] fix tests --- etherman/etherman.go | 2 +- etherman/etherman_test.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/etherman/etherman.go b/etherman/etherman.go index 0e80820510..b832c1a030 100644 --- a/etherman/etherman.go +++ b/etherman/etherman.go @@ -1304,7 +1304,7 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1)) // ForcedBatches are skipped here. They are not currently enabled for validium, but once they are supported, // their data must come from the state.forced_batch table - if sequencesValidium[i].ForcedTimestamp != 0 { + if sequencesValidium[i].ForcedTimestamp == 0 { batchNums = append(batchNums, bn) hashes = append(hashes, sequencesValidium[i].TransactionsHash) } diff --git a/etherman/etherman_test.go b/etherman/etherman_test.go index 8aa73ad96d..2ce43cd88f 100644 --- a/etherman/etherman_test.go +++ b/etherman/etherman_test.go @@ -164,8 +164,10 @@ func TestSequencedBatchesEvent(t *testing.T) { }, polygonzkevm.PolygonValidiumEtrogValidiumBatchData{ TransactionsHash: txsHash, }) - da.Mock.On("GetBatchL2Data", uint64(2), txsHash).Return(data, nil) - da.Mock.On("GetBatchL2Data", uint64(3), txsHash).Return(data, nil) + batchNums := []uint64{2, 3} + batchHashes := []common.Hash{txsHash, txsHash} + batchData := [][]byte{data, data} + da.Mock.On("GetBatchL2Data", batchNums, batchHashes, []byte{}).Return(batchData, nil) _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, sequences, auth.From, []byte{}) require.NoError(t, err) @@ -206,7 +208,7 @@ func TestVerifyBatchEvent(t *testing.T) { } _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, []polygonzkevm.PolygonValidiumEtrogValidiumBatchData{tx}, auth.From, nil) require.NoError(t, err) - da.Mock.On("GetBatchL2Data", uint64(2), crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))).Return(common.Hex2Bytes(rawTxs), nil) + da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))}, []byte{}).Return([][]byte{common.Hex2Bytes(rawTxs)}, nil) // Mine the tx in a block ethBackend.Commit() @@ -319,7 +321,7 @@ func TestSendSequences(t *testing.T) { } tx, err := etherman.sequenceBatches(*auth, []ethmanTypes.Sequence{sequence}, auth.From, []byte{}) require.NoError(t, err) - da.Mock.On("GetBatchL2Data", uint64(2), crypto.Keccak256Hash(batchL2Data)).Return(batchL2Data, nil) + da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(batchL2Data)}, []byte{}).Return([][]byte{batchL2Data}, nil) log.Debug("TX: ", tx.Hash()) ethBackend.Commit() From eef8033178d51a89c05333f3f2b6c971c878ae7d Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Tue, 12 Mar 2024 08:57:58 -0400 Subject: [PATCH 5/9] remove fixme comment --- dataavailability/datacommittee/datacommittee.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dataavailability/datacommittee/datacommittee.go b/dataavailability/datacommittee/datacommittee.go index 55cdce3c51..c79dfb645c 100644 --- a/dataavailability/datacommittee/datacommittee.go +++ b/dataavailability/datacommittee/datacommittee.go @@ -90,7 +90,6 @@ func (d *DataCommitteeBackend) Init() error { // GetSequence gets backend data one hash at a time. This should be optimized on the DAC side to get them all at once. func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { // TODO: optimize this on the DAC side by implementing a multi batch retrieve api - // FIXME: how to use dataAvailabilityMessage ? var batchData [][]byte for _, h := range hashes { data, err := d.GetBatchL2Data(h) From f3ec0fd38c8803ece9138e77a02943455bbf28ac Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Tue, 12 Mar 2024 09:08:01 -0400 Subject: [PATCH 6/9] dataavailability message parameter for tests --- etherman/etherman_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/etherman/etherman_test.go b/etherman/etherman_test.go index 2ce43cd88f..8c8a0a02f5 100644 --- a/etherman/etherman_test.go +++ b/etherman/etherman_test.go @@ -167,8 +167,9 @@ func TestSequencedBatchesEvent(t *testing.T) { batchNums := []uint64{2, 3} batchHashes := []common.Hash{txsHash, txsHash} batchData := [][]byte{data, data} - da.Mock.On("GetBatchL2Data", batchNums, batchHashes, []byte{}).Return(batchData, nil) - _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, sequences, auth.From, []byte{}) + daMessage, _ := hex.DecodeString("0x123456789123456789") + da.Mock.On("GetBatchL2Data", batchNums, batchHashes, daMessage).Return(batchData, nil) + _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, sequences, auth.From, daMessage) require.NoError(t, err) // Mine the tx in a block @@ -206,9 +207,10 @@ func TestVerifyBatchEvent(t *testing.T) { tx := polygonzkevm.PolygonValidiumEtrogValidiumBatchData{ TransactionsHash: crypto.Keccak256Hash(common.Hex2Bytes(rawTxs)), } - _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, []polygonzkevm.PolygonValidiumEtrogValidiumBatchData{tx}, auth.From, nil) + daMessage, _ := hex.DecodeString("0x1234") + _, err = etherman.ZkEVM.SequenceBatchesValidium(auth, []polygonzkevm.PolygonValidiumEtrogValidiumBatchData{tx}, auth.From, daMessage) require.NoError(t, err) - da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))}, []byte{}).Return([][]byte{common.Hex2Bytes(rawTxs)}, nil) + da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))}, daMessage).Return([][]byte{common.Hex2Bytes(rawTxs)}, nil) // Mine the tx in a block ethBackend.Commit() @@ -319,9 +321,10 @@ func TestSendSequences(t *testing.T) { sequence := ethmanTypes.Sequence{ BatchL2Data: batchL2Data, } - tx, err := etherman.sequenceBatches(*auth, []ethmanTypes.Sequence{sequence}, auth.From, []byte{}) + daMessage, _ := hex.DecodeString("0x1234") + tx, err := etherman.sequenceBatches(*auth, []ethmanTypes.Sequence{sequence}, auth.From, daMessage) require.NoError(t, err) - da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(batchL2Data)}, []byte{}).Return([][]byte{batchL2Data}, nil) + da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(batchL2Data)}, daMessage).Return([][]byte{batchL2Data}, nil) log.Debug("TX: ", tx.Hash()) ethBackend.Commit() From 026fba50b8a4fe91e4861abab4815bccf980987c Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Tue, 12 Mar 2024 13:13:19 -0400 Subject: [PATCH 7/9] apply review comments --- dataavailability/interfaces.go | 38 ++++++++++++++------------ etherman/etherman.go | 8 +++--- state/pgstatestorage/pgstatestorage.go | 22 +++++++-------- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/dataavailability/interfaces.go b/dataavailability/interfaces.go index 16e67a22a2..d45b41b358 100644 --- a/dataavailability/interfaces.go +++ b/dataavailability/interfaces.go @@ -10,16 +10,12 @@ import ( "github.com/jackc/pgx/v4" ) -type stateInterface interface { - GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) - GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) - GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error) -} - -// BatchDataProvider is used to retrieve batch data -type BatchDataProvider interface { - // GetBatchL2Data retrieve the data of a batch from the DA backend. The returned data must be the pre-image of the hash - GetBatchL2Data(batchNum []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) +// DABackender is an interface for components that store and retrieve batch data +type DABackender interface { + SequenceRetriever + SequenceSender + // Init initializes the DABackend + Init() error } // SequenceSender is used to send provided sequence of batches @@ -35,20 +31,26 @@ type SequenceRetriever interface { GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) } +// === Internal interfaces === + +type stateInterface interface { + GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) + GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) + GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error) +} + +// BatchDataProvider is used to retrieve batch data +type BatchDataProvider interface { + // GetBatchL2Data retrieve the data of a batch from the DA backend. The returned data must be the pre-image of the hash + GetBatchL2Data(batchNum []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) +} + // DataManager is an interface for components that send and retrieve batch data type DataManager interface { BatchDataProvider SequenceSender } -// DABackender is an interface for data committee components that store and retrieve batch data -type DABackender interface { - SequenceRetriever - SequenceSender - // Init initializes the DABackend - Init() error -} - // ZKEVMClientTrustedBatchesGetter contains the methods required to interact with zkEVM-RPC type ZKEVMClientTrustedBatchesGetter interface { BatchByNumber(ctx context.Context, number *big.Int) (*types.Batch, error) diff --git a/etherman/etherman.go b/etherman/etherman.go index b832c1a030..eafcb86e19 100644 --- a/etherman/etherman.go +++ b/etherman/etherman.go @@ -1296,17 +1296,17 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add return nil, err } coinbase := (data[1]).(common.Address) - dataAvailabilityMessage := (data[2]).([]byte) // TODO: is this right??? + dataAvailabilityMessage := (data[2]).([]byte) sequencedBatches := make([]SequencedBatch, len(sequencesValidium)) var batchNums []uint64 var hashes []common.Hash - for i := range sequencesValidium { + for i, validiumData := range sequencesValidium { bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1)) // ForcedBatches are skipped here. They are not currently enabled for validium, but once they are supported, // their data must come from the state.forced_batch table - if sequencesValidium[i].ForcedTimestamp == 0 { + if validiumData.ForcedTimestamp == 0 { batchNums = append(batchNums, bn) - hashes = append(hashes, sequencesValidium[i].TransactionsHash) + hashes = append(hashes, validiumData.TransactionsHash) } } batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage) diff --git a/state/pgstatestorage/pgstatestorage.go b/state/pgstatestorage/pgstatestorage.go index 891ae7546f..68ce6c43f0 100644 --- a/state/pgstatestorage/pgstatestorage.go +++ b/state/pgstatestorage/pgstatestorage.go @@ -358,17 +358,11 @@ func (p *PostgresStorage) GetNativeBlockHashesInRange(ctx context.Context, fromB // GetBatchL2DataByNumber returns the batch L2 data of the given batch number. func (p *PostgresStorage) GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) { - const getBatchL2DataByBatchNumber = "SELECT raw_txs_data FROM state.batch WHERE batch_num = $1" - q := p.getExecQuerier(dbTx) - var batchL2Data []byte - err := q.QueryRow(ctx, getBatchL2DataByBatchNumber, batchNumber).Scan(&batchL2Data) - - if errors.Is(err, pgx.ErrNoRows) { - return nil, state.ErrNotFound - } else if err != nil { + rows, err := p.GetBatchL2DataByNumbers(ctx, []uint64{batchNumber}, dbTx) + if err != nil { return nil, err } - return batchL2Data, nil + return rows[0], nil } // GetBatchL2DataByNumbers returns the batch L2 data of the given batch numbers. @@ -381,15 +375,21 @@ func (p *PostgresStorage) GetBatchL2DataByNumbers(ctx context.Context, batchNumb } else if err != nil { return nil, err } + defer rows.Close() batchL2DataMap := make(map[uint64][]byte) for rows.Next() { - var batchNum uint64 - var batchL2Data []byte + var ( + batchNum uint64 + batchL2Data []byte + ) err := rows.Scan(&batchNum, &batchL2Data) if err != nil { return nil, err } batchL2DataMap[batchNum] = batchL2Data } + if len(batchNumbers) == 0 { + return nil, state.ErrNotFound + } return batchL2DataMap, nil } From 0a5de1b99a73e9ef6d8cd6993886b3795146c160 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Tue, 12 Mar 2024 13:36:41 -0400 Subject: [PATCH 8/9] fix single row retrieval --- state/pgstatestorage/pgstatestorage.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/state/pgstatestorage/pgstatestorage.go b/state/pgstatestorage/pgstatestorage.go index 68ce6c43f0..5065116753 100644 --- a/state/pgstatestorage/pgstatestorage.go +++ b/state/pgstatestorage/pgstatestorage.go @@ -358,11 +358,15 @@ func (p *PostgresStorage) GetNativeBlockHashesInRange(ctx context.Context, fromB // GetBatchL2DataByNumber returns the batch L2 data of the given batch number. func (p *PostgresStorage) GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error) { - rows, err := p.GetBatchL2DataByNumbers(ctx, []uint64{batchNumber}, dbTx) + batchData, err := p.GetBatchL2DataByNumbers(ctx, []uint64{batchNumber}, dbTx) if err != nil { return nil, err } - return rows[0], nil + data, ok := batchData[batchNumber] + if !ok { + return nil, state.ErrNotFound + } + return data, nil } // GetBatchL2DataByNumbers returns the batch L2 data of the given batch numbers. From 980812c6d617d9a273002c63e4ba010a8984c8db Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Tue, 12 Mar 2024 16:10:05 -0400 Subject: [PATCH 9/9] unified batch and force batch data retrieval --- etherman/etherman.go | 8 ++---- state/pgstatestorage/pgstatestorage.go | 12 ++++++--- state/pgstatestorage/pgstatestorage_test.go | 28 ++++++++++++++++++--- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/etherman/etherman.go b/etherman/etherman.go index eafcb86e19..4bb303024f 100644 --- a/etherman/etherman.go +++ b/etherman/etherman.go @@ -1302,12 +1302,8 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add var hashes []common.Hash for i, validiumData := range sequencesValidium { bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1)) - // ForcedBatches are skipped here. They are not currently enabled for validium, but once they are supported, - // their data must come from the state.forced_batch table - if validiumData.ForcedTimestamp == 0 { - batchNums = append(batchNums, bn) - hashes = append(hashes, validiumData.TransactionsHash) - } + batchNums = append(batchNums, bn) + hashes = append(hashes, validiumData.TransactionsHash) } batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage) if err != nil { diff --git a/state/pgstatestorage/pgstatestorage.go b/state/pgstatestorage/pgstatestorage.go index 5065116753..613bfb2d53 100644 --- a/state/pgstatestorage/pgstatestorage.go +++ b/state/pgstatestorage/pgstatestorage.go @@ -369,11 +369,15 @@ func (p *PostgresStorage) GetBatchL2DataByNumber(ctx context.Context, batchNumbe return data, nil } -// GetBatchL2DataByNumbers returns the batch L2 data of the given batch numbers. +// GetBatchL2DataByNumbers returns the batch L2 data of the given batch numbers. The data is a union of state.batch and state.forced_batch tables. func (p *PostgresStorage) GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error) { - const getBatchL2DataByBatchNumber = "SELECT batch_num, raw_txs_data FROM state.batch WHERE batch_num = ANY($1)" + const getBatchL2DataByBatchNumber = ` + SELECT batch_num, raw_txs_data FROM state.batch WHERE batch_num = ANY($1) + UNION + SELECT forced_batch_num, convert_from(decode(raw_txs_data, 'hex'), 'UTF8')::bytea FROM state.forced_batch WHERE forced_batch_num = ANY($2) +` q := p.getExecQuerier(dbTx) - rows, err := q.Query(ctx, getBatchL2DataByBatchNumber, batchNumbers) + rows, err := q.Query(ctx, getBatchL2DataByBatchNumber, batchNumbers, batchNumbers) if errors.Is(err, pgx.ErrNoRows) { return nil, state.ErrNotFound } else if err != nil { @@ -392,7 +396,7 @@ func (p *PostgresStorage) GetBatchL2DataByNumbers(ctx context.Context, batchNumb } batchL2DataMap[batchNum] = batchL2Data } - if len(batchNumbers) == 0 { + if len(batchL2DataMap) == 0 { return nil, state.ErrNotFound } return batchL2DataMap, nil diff --git a/state/pgstatestorage/pgstatestorage_test.go b/state/pgstatestorage/pgstatestorage_test.go index 65a03d9603..d756aa05d5 100644 --- a/state/pgstatestorage/pgstatestorage_test.go +++ b/state/pgstatestorage/pgstatestorage_test.go @@ -1168,8 +1168,8 @@ func TestGetBatchL2DataByNumbers(t *testing.T) { require.NoError(t, err) defer func() { require.NoError(t, tx.Commit(ctx)) }() - var i1, i2, i3, i4 = uint64(1), uint64(2), uint64(3), uint64(4) - var d1, d2 = []byte("foobar"), []byte("dingbat") + var i1, i2, i3, i4, i5 = uint64(1), uint64(2), uint64(3), uint64(4), uint64(5) + var d1, d2, d4 = []byte("foobar"), []byte("dingbat"), []byte{0xb} const insertBatch = "INSERT INTO state.batch (batch_num, raw_txs_data) VALUES ($1, $2)" _, err = tx.Exec(ctx, insertBatch, i1, d1) @@ -1179,13 +1179,29 @@ func TestGetBatchL2DataByNumbers(t *testing.T) { _, err = tx.Exec(ctx, insertBatch, i3, nil) require.NoError(t, err) - allData, err := testState.GetBatchL2DataByNumbers(ctx, []uint64{i1, i2, i3, i4}, tx) + // Add a forced batch too, needs a block + block1 := *block + block1.BlockNumber = 1000 + err = testState.AddBlock(ctx, &block1, tx) + require.NoError(t, err) + err = tx.Commit(ctx) + require.NoError(t, err) + + tx, err = testState.BeginStateTransaction(ctx) + require.NoError(t, err) + + const insertForcedBatch = "INSERT INTO state.forced_batch (forced_batch_num, timestamp, raw_txs_data, block_num) VALUES (4, now(),'0b', 1000)" + _, err = testState.Exec(ctx, insertForcedBatch) + require.NoError(t, err) + + allData, err := testState.GetBatchL2DataByNumbers(ctx, []uint64{i1, i2, i3, i4, i5}, tx) require.NoError(t, err) assert.Equal(t, d1, allData[i1]) assert.Equal(t, d2, allData[i2]) assert.Nil(t, allData[i3]) + assert.Equal(t, d4, allData[i4]) - _, ok := allData[i4] + _, ok := allData[i5] assert.False(t, ok) } @@ -1390,6 +1406,10 @@ func TestGetForcedBatch(t *testing.T) { require.Equal(t, uint64(2002), fb.BlockNumber) require.Equal(t, "0x717e05de47a87a7d1679e183f1c224150675f6302b7da4eaab526b2b91ae0761", fb.GlobalExitRoot.String()) require.Equal(t, []byte{0xb}, fb.RawTxsData) + + fbData, err := testState.GetBatchL2DataByNumber(ctx, 1, dbTx) + require.NoError(t, err) + require.Equal(t, []byte{0xb}, fbData) } func TestGetLastGER(t *testing.T) {