From 5213145b54f497715cc27011a257aceed7775820 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 20 Oct 2023 12:01:55 -0400 Subject: [PATCH 01/70] add isClusterPrefixed field to invalid control message notification --- network/p2p/consumers.go | 11 ++-- .../control_message_validation_inspector.go | 65 ++++++++++--------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index 571abeabf84..2e65e38c6b1 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -33,14 +33,17 @@ type InvCtrlMsgNotif struct { Error error // MsgType the control message type. MsgType p2pmsg.ControlMessageType + // IsClusterPrefixed indicates if the error occurred on a cluster prefixed topic of the control message. + IsClusterPrefixed bool } // NewInvalidControlMessageNotification returns a new *InvCtrlMsgNotif -func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.ControlMessageType, err error) *InvCtrlMsgNotif { +func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.ControlMessageType, err error, isClusterPrefixed bool) *InvCtrlMsgNotif { return &InvCtrlMsgNotif{ - PeerID: peerID, - Error: err, - MsgType: ctlMsgType, + PeerID: peerID, + Error: err, + MsgType: ctlMsgType, + IsClusterPrefixed: isClusterPrefixed, } } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 9053aec4bb1..5686a8cf2c7 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -152,27 +152,27 @@ func (c *ControlMsgValidationInspector) processInspectRPCReq(req *InspectRPCRequ // iWant validation uses new sample size validation. This will be updated for all other control message types. switch ctrlMsgType { case p2pmsg.CtrlMsgGraft: - err := c.inspectGraftMessages(req.Peer, req.rpc.GetControl().GetGraft(), activeClusterIDS) + err, isClusterPrefixed := c.inspectGraftMessages(req.Peer, req.rpc.GetControl().GetGraft(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgGraft, err) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgGraft, err, isClusterPrefixed) return nil } case p2pmsg.CtrlMsgPrune: - err := c.inspectPruneMessages(req.Peer, req.rpc.GetControl().GetPrune(), activeClusterIDS) + err, isClusterPrefixed := c.inspectPruneMessages(req.Peer, req.rpc.GetControl().GetPrune(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgPrune, err) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgPrune, err, isClusterPrefixed) return nil } case p2pmsg.CtrlMsgIWant: err := c.inspectIWantMessages(req.Peer, req.rpc.GetControl().GetIwant()) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIWant, err) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIWant, err, false) return nil } case p2pmsg.CtrlMsgIHave: - err := c.inspectIHaveMessages(req.Peer, req.rpc.GetControl().GetIhave(), activeClusterIDS) + err, isClusterPrefixed := c.inspectIHaveMessages(req.Peer, req.rpc.GetControl().GetIhave(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIHave, err) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIHave, err, isClusterPrefixed) return nil } } @@ -189,20 +189,21 @@ func (c *ControlMsgValidationInspector) processInspectRPCReq(req *InspectRPCRequ // Returns: // - DuplicateTopicErr: if there are any duplicate topics in the list of grafts // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. -func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) error { +// - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. +func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, bool) { tracker := make(duplicateStrTracker) for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft) + return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), false } tracker.set(topic.String()) - err := c.validateTopic(from, topic, activeClusterIDS) + err, isClusterPrefixed := c.validateTopic(from, topic, activeClusterIDS) if err != nil { - return err + return err, isClusterPrefixed } } - return nil + return nil, false } // inspectPruneMessages performs topic validation on all prunes in the control message using the provided validateTopic func while tracking duplicates. @@ -214,20 +215,21 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // - DuplicateTopicErr: if there are any duplicate topics found in the list of iHaves // or any duplicate message ids found inside a single iHave. // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. -func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) error { +// - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. +func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, bool) { tracker := make(duplicateStrTracker) for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune) + return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), false } tracker.set(topic.String()) - err := c.validateTopic(from, topic, activeClusterIDS) + err, isClusterPrefixed := c.validateTopic(from, topic, activeClusterIDS) if err != nil { - return err + return err, isClusterPrefixed } } - return nil + return nil, false } // inspectIHaveMessages performs topic validation on all ihaves in the control message using the provided validateTopic func while tracking duplicates. @@ -239,9 +241,10 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune // - DuplicateTopicErr: if there are any duplicate topics found in the list of iHaves // or any duplicate message ids found inside a single iHave. // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. -func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihaves []*pubsub_pb.ControlIHave, activeClusterIDS flow.ChainIDList) error { +// - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. +func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihaves []*pubsub_pb.ControlIHave, activeClusterIDS flow.ChainIDList) (error, bool) { if len(ihaves) == 0 { - return nil + return nil, false } lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). @@ -255,17 +258,17 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() if duplicateTopicTracker.isDuplicate(topic) { - return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave) + return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), false } duplicateTopicTracker.set(topic) - err := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) + err, isClusterPrefixed := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { - return err + return err, isClusterPrefixed } for _, messageID := range messageIds { if duplicateMessageIDTracker.isDuplicate(messageID) { - return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave) + return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave), false } duplicateMessageIDTracker.set(messageID) } @@ -273,7 +276,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave lg.Debug(). Int("total_message_ids", totalMessageIds). Msg("ihave control message validation complete") - return nil + return nil, false } // inspectIWantMessages inspects RPC iWant control messages. This func will sample the iWants and perform validation on each iWant in the sample. @@ -549,23 +552,23 @@ func (c *ControlMsgValidationInspector) performSample(ctrlMsg p2pmsg.ControlMess // // This func returns an exception in case of unexpected bug or state corruption if cluster prefixed topic validation // fails due to unexpected error returned when getting the active cluster IDS. -func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) error { +func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) (error, bool) { channel, ok := channels.ChannelFromTopic(topic) if !ok { - return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")) + return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")), false } // handle cluster prefixed topics if channels.IsClusterChannel(channel) { - return c.validateClusterPrefixedTopic(from, topic, activeClusterIds) + return c.validateClusterPrefixedTopic(from, topic, activeClusterIds), true } // non cluster prefixed topic validation err := channels.IsValidNonClusterFlowTopic(topic, c.sporkID) if err != nil { - return err + return err, false } - return nil + return nil, false } // validateClusterPrefixedTopic validates cluster prefixed topics. @@ -660,7 +663,7 @@ func (c *ControlMsgValidationInspector) checkClusterPrefixHardThreshold(nodeID f } // logAndDistributeErr logs the provided error and attempts to disseminate an invalid control message validation notification for the error. -func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *InspectRPCRequest, ctlMsgType p2pmsg.ControlMessageType, err error) { +func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *InspectRPCRequest, ctlMsgType p2pmsg.ControlMessageType, err error, isClusterPrefixed bool) { lg := c.logger.With(). Bool(logging.KeySuspicious, true). Bool(logging.KeyNetworkingSecurity, true). @@ -673,7 +676,7 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In case IsErrUnstakedPeer(err): lg.Warn().Err(err).Msg("control message received from unstaked peer") default: - err = c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err)) + err = c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, isClusterPrefixed)) if err != nil { lg.Error(). Err(err). From 7759c872880aeafd470d31b31e66870dfa0773ee Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 20 Oct 2023 12:20:27 -0400 Subject: [PATCH 02/70] apply more lenient penalty for invalid notification on cluster prefixed topics - update tests --- network/p2p/scoring/registry.go | 21 +++++++++++++++++---- network/p2p/scoring/registry_test.go | 27 ++++++++++++++++----------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 0bf85243e7a..194946b52d3 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -48,6 +48,9 @@ const ( iHaveMisbehaviourPenalty = -10 // iWantMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a iWant misbehaviour. iWantMisbehaviourPenalty = -10 + // clusterPrefixedPenaltyReductionFactor factor used to reduce the penalty for control message misbehaviours on cluster prefixed topics. This is allows a more lenient punishment for nodes + // that fall behind and may need to request old data. + clusterPrefixedPenaltyReductionFactor = .5 ) // GossipSubCtrlMsgPenaltyValue is the penalty value for each control message type. @@ -56,15 +59,19 @@ type GossipSubCtrlMsgPenaltyValue struct { Prune float64 // penalty value for an individual prune message misbehaviour. IHave float64 // penalty value for an individual iHave message misbehaviour. IWant float64 // penalty value for an individual iWant message misbehaviour. + // ClusterPrefixedPenaltyReductionFactor factor used to reduce the penalty for control message misbehaviours on cluster prefixed topics. This is allows a more lenient punishment for nodes + // that fall behind and may need to request old data. + ClusterPrefixedPenaltyReductionFactor float64 } // DefaultGossipSubCtrlMsgPenaltyValue returns the default penalty value for each control message type. func DefaultGossipSubCtrlMsgPenaltyValue() GossipSubCtrlMsgPenaltyValue { return GossipSubCtrlMsgPenaltyValue{ - Graft: graftMisbehaviourPenalty, - Prune: pruneMisbehaviourPenalty, - IHave: iHaveMisbehaviourPenalty, - IWant: iWantMisbehaviourPenalty, + Graft: graftMisbehaviourPenalty, + Prune: pruneMisbehaviourPenalty, + IHave: iHaveMisbehaviourPenalty, + IWant: iWantMisbehaviourPenalty, + ClusterPrefixedPenaltyReductionFactor: clusterPrefixedPenaltyReductionFactor, } } @@ -268,6 +275,12 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( // the error is considered fatal as it means that we have an unsupported misbehaviour type, we should crash the node to prevent routing attack vulnerability. lg.Fatal().Str("misbehavior_type", notification.MsgType.String()).Msg("unknown misbehaviour type") } + + // reduce penalty for cluster prefixed topics allowing nodes that are potentially behind to catch up + if notification.IsClusterPrefixed { + record.Penalty *= r.penalty.ClusterPrefixedPenaltyReductionFactor + } + return record }) if err != nil { diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 35842f29177..17ca9f1e3da 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -49,20 +49,23 @@ func TestNoPenaltyRecord(t *testing.T) { // penalty value as the app specific score. func TestPeerWithSpamRecord(t *testing.T) { t.Run("graft", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft, false) }) t.Run("prune", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune, false) }) t.Run("ihave", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave, false) }) t.Run("iwant", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant, false) + }) + t.Run("cluster prefixed", func(t *testing.T) { + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant*penaltyValueFixtures().ClusterPrefixedPenaltyReductionFactor, true) }) } -func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { +func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64, isClusterPrefixed bool) { peerID := peer.ID("peer-1") reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, @@ -79,8 +82,9 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ - PeerID: peerID, - MsgType: messageType, + PeerID: peerID, + MsgType: messageType, + IsClusterPrefixed: isClusterPrefixed, }) // the penalty should now be updated in the spamRecords @@ -468,9 +472,10 @@ func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.Go // that the tests are not passing because of the default values. func penaltyValueFixtures() scoring.GossipSubCtrlMsgPenaltyValue { return scoring.GossipSubCtrlMsgPenaltyValue{ - Graft: -100, - Prune: -50, - IHave: -20, - IWant: -10, + Graft: -100, + Prune: -50, + IHave: -20, + IWant: -10, + ClusterPrefixedPenaltyReductionFactor: .5, } } From 1de2f87e46dca1f42802fb1e2b4cae9146edd801 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 31 Oct 2023 13:00:04 +0200 Subject: [PATCH 03/70] Added throw to backend handlers when unable to get the latest finalized or sealed block --- engine/access/rpc/backend/backend.go | 13 ++++++- engine/access/rpc/backend/backend_accounts.go | 3 ++ .../rpc/backend/backend_accounts_test.go | 37 +++++++++++++++++++ .../rpc/backend/backend_block_details.go | 8 +++- .../rpc/backend/backend_block_headers.go | 8 +++- engine/access/rpc/backend/backend_events.go | 3 ++ engine/access/rpc/backend/backend_scripts.go | 3 ++ .../rpc/backend/backend_transactions.go | 7 ++++ engine/access/rpc/engine.go | 4 ++ 9 files changed, 81 insertions(+), 5 deletions(-) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 4bf1d77ad9a..f759e9f6da0 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" //nolint:gosec "fmt" + "github.com/onflow/flow-go/module/irrecoverable" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -75,7 +76,8 @@ type Backend struct { connFactory connection.ConnectionFactory // cache the response to GetNodeVersionInfo since it doesn't change - nodeInfo *access.NodeVersionInfo + nodeInfo *access.NodeVersionInfo + SignalCtx irrecoverable.SignalerContext } type Params struct { @@ -218,6 +220,15 @@ func New(params Params) (*Backend, error) { return b, nil } +func (b *Backend) HandleInconsistentProtocolState(ctx irrecoverable.SignalerContext) { + b.backendTransactions.SignalCtx = ctx + b.backendAccounts.SignalCtx = ctx + b.backendBlockDetails.SignalCtx = ctx + b.backendBlockHeaders.SignalCtx = ctx + b.backendEvents.SignalCtx = ctx + b.backendScripts.SignalCtx = ctx +} + // NewCache constructs cache for storing connections to other nodes. // No errors are expected during normal operations. func NewCache( diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index eaa0ded99bc..ab4355cfa42 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "github.com/onflow/flow-go/module/irrecoverable" "time" "github.com/rs/zerolog" @@ -31,6 +32,7 @@ type backendAccounts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode + SignalCtx irrecoverable.SignalerContext } // GetAccount returns the account details at the latest sealed block. @@ -43,6 +45,7 @@ func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) { sealed, err := b.state.Sealed().Head() if err != nil { + b.SignalCtx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index e876c2325e3..2a1c1fdc6be 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -3,6 +3,7 @@ package backend import ( "context" "fmt" + "github.com/onflow/flow-go/module/irrecoverable" "testing" execproto "github.com/onflow/flow/protobuf/go/flow/execution" @@ -44,6 +45,9 @@ type BackendAccountsSuite struct { block *flow.Block account *flow.Account failingAddress flow.Address + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc } func TestBackendAccountsSuite(t *testing.T) { @@ -72,9 +76,11 @@ func (s *BackendAccountsSuite) SetupTest() { s.Require().NoError(err) s.failingAddress = unittest.AddressFixture() + s.ctx, s.cancel = irrecoverable.NewMockSignalerContextWithCancel(s.T(), context.Background()) } func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { + return &backendAccounts{ log: s.log, state: s.state, @@ -82,6 +88,7 @@ func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { executionReceipts: s.receipts, connFactory: s.connectionFactory, nodeCommunicator: NewNodeCommunicator(false), + SignalCtx: s.ctx, } } @@ -316,6 +323,36 @@ func (s *BackendAccountsSuite) TestGetAccountFromFailover_ReturnsENErrors() { }) } +// TestGetAccountFromStorage_Fails tests that errors received from local storage are handled +// and converted to the appropriate status code +func (s *BackendAccountsSuite) TestGetAccountState_Fails() { + ctx := context.Background() + + scriptExecutor := execmock.NewScriptExecutor(s.T()) + + backend := s.defaultBackend() + backend.scriptExecMode = ScriptExecutionModeLocalOnly + backend.scriptExecutor = scriptExecutor + + s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", "head failed"), func() { //TODO: change + s.state.On("Sealed").Return(s.snapshot, nil).Once() + s.snapshot.On("Head").Return(nil, fmt.Errorf("head error")).Once() + + //defer func() { + // if r := recover(); r != nil { + // // Check if the panic was due to runtime.Goexit(). + // if r != nil { + // // Handle the case where runtime.Goexit was called. + // s.T().Logf("Caught runtime.Goexit()") + // } + // } + //}() + //backend.GetAccountAtLatestBlock(ctx, s.failingAddress) + + s.Assert().Panics(func() { backend.GetAccountAtLatestBlock(ctx, s.failingAddress) }, "head failed") + }) +} + func (s *BackendAccountsSuite) testGetAccount(ctx context.Context, backend *backendAccounts, statusCode codes.Code) { s.state.On("Sealed").Return(s.snapshot, nil).Once() s.snapshot.On("Head").Return(s.block.Header, nil).Once() diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index fc9eee618c4..bfc50c52bcd 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -2,6 +2,7 @@ package backend import ( "context" + "github.com/onflow/flow-go/module/irrecoverable" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,8 +14,9 @@ import ( ) type backendBlockDetails struct { - blocks storage.Blocks - state protocol.State + blocks storage.Blocks + state protocol.State + SignalCtx irrecoverable.SignalerContext } func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { @@ -40,6 +42,7 @@ func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) ( // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. + b.SignalCtx.Throw(err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block: %v", err) } @@ -93,6 +96,7 @@ func (b *backendBlockDetails) getBlockStatus(block *flow.Block) (flow.BlockStatu // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. + b.SignalCtx.Throw(err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index ac4116224d4..4873174c143 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -2,6 +2,7 @@ package backend import ( "context" + "github.com/onflow/flow-go/module/irrecoverable" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,8 +14,9 @@ import ( ) type backendBlockHeaders struct { - headers storage.Headers - state protocol.State + headers storage.Headers + state protocol.State + SignalCtx irrecoverable.SignalerContext } func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { @@ -39,6 +41,7 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed b // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. + b.SignalCtx.Throw(err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block header: %v", err) } @@ -86,6 +89,7 @@ func (b *backendBlockHeaders) getBlockStatus(header *flow.Header) (flow.BlockSta // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. + b.SignalCtx.Throw(err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 6db5efc25f7..89a4441638c 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/onflow/flow-go/module/irrecoverable" "time" "github.com/onflow/flow/protobuf/go/flow/entities" @@ -30,6 +31,7 @@ type backendEvents struct { log zerolog.Logger maxHeightRange uint nodeCommunicator Communicator + SignalCtx irrecoverable.SignalerContext } // GetEventsForHeightRange retrieves events for all sealed blocks between the start block height and @@ -54,6 +56,7 @@ func (b *backendEvents) GetEventsForHeightRange( head, err := b.state.Sealed().Head() if err != nil { // sealed block must be in the store, so return an Internal code even if we got NotFound + b.SignalCtx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get events: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index 7ebb04a0e73..d9c8f3e4aae 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -5,6 +5,7 @@ import ( "context" "crypto/md5" //nolint:gosec "errors" + "github.com/onflow/flow-go/module/irrecoverable" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -38,6 +39,7 @@ type backendScripts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode + SignalCtx irrecoverable.SignalerContext } // ExecuteScriptAtLatestBlock executes provided script at the latest sealed block. @@ -49,6 +51,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestHeader, err := b.state.Sealed().Head() if err != nil { // the latest sealed header MUST be available + b.SignalCtx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index b9c8f055708..fd7980af277 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -42,6 +43,8 @@ type backendTransactions struct { log zerolog.Logger nodeCommunicator Communicator txResultCache *lru.Cache[flow.Identifier, *access.TransactionResult] + + SignalCtx irrecoverable.SignalerContext } // SendTransaction forwards the transaction to the collection node @@ -576,12 +579,14 @@ func (b *backendTransactions) deriveTransactionStatus( // Not in a block, let's see if it's expired referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() if err != nil { + b.SignalCtx.Throw(err) return flow.TransactionStatusUnknown, err } refHeight := referenceBlock.Height // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { + b.SignalCtx.Throw(err) return flow.TransactionStatusUnknown, err } finalizedHeight := finalized.Height @@ -626,6 +631,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { + b.SignalCtx.Throw(err) return flow.TransactionStatusUnknown, err } @@ -741,6 +747,7 @@ func (b *backendTransactions) getHistoricalTransactionResult( func (b *backendTransactions) registerTransactionForRetry(tx *flow.TransactionBody) { referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() if err != nil { + b.SignalCtx.Throw(err) return } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 8bf8acd72c3..a7fe0080c54 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -129,6 +129,10 @@ func NewBuilder(log zerolog.Logger, AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). AddWorker(backendNotifierWorker). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + backend.HandleInconsistentProtocolState(ctx) + ready() + }). AddWorker(eng.shutdownWorker). Build() From 394fdb0ed67d174e9c907c16f094e78cfc39cae0 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 2 Nov 2023 18:10:33 +0200 Subject: [PATCH 04/70] Added unit tests. Linted --- engine/access/rpc/backend/backend.go | 14 +++--- engine/access/rpc/backend/backend_accounts.go | 6 +-- .../rpc/backend/backend_accounts_test.go | 44 ++++++++----------- .../rpc/backend/backend_block_details.go | 12 ++--- .../rpc/backend/backend_block_headers.go | 12 ++--- engine/access/rpc/backend/backend_events.go | 6 +-- engine/access/rpc/backend/backend_scripts.go | 6 +-- .../rpc/backend/backend_scripts_test.go | 27 ++++++++++++ .../rpc/backend/backend_transactions.go | 9 ++-- 9 files changed, 78 insertions(+), 58 deletions(-) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index f759e9f6da0..4752a3748c9 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -4,7 +4,6 @@ import ( "context" "crypto/md5" //nolint:gosec "fmt" - "github.com/onflow/flow-go/module/irrecoverable" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -20,6 +19,7 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/execution" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -221,12 +221,12 @@ func New(params Params) (*Backend, error) { } func (b *Backend) HandleInconsistentProtocolState(ctx irrecoverable.SignalerContext) { - b.backendTransactions.SignalCtx = ctx - b.backendAccounts.SignalCtx = ctx - b.backendBlockDetails.SignalCtx = ctx - b.backendBlockHeaders.SignalCtx = ctx - b.backendEvents.SignalCtx = ctx - b.backendScripts.SignalCtx = ctx + b.backendTransactions.ctx = ctx + b.backendAccounts.ctx = ctx + b.backendBlockDetails.ctx = ctx + b.backendBlockHeaders.ctx = ctx + b.backendEvents.ctx = ctx + b.backendScripts.ctx = ctx } // NewCache constructs cache for storing connections to other nodes. diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index ab4355cfa42..a384da297c8 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "github.com/onflow/flow-go/module/irrecoverable" "time" "github.com/rs/zerolog" @@ -19,6 +18,7 @@ import ( fvmerrors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/execution" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -32,7 +32,7 @@ type backendAccounts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode - SignalCtx irrecoverable.SignalerContext + ctx irrecoverable.SignalerContext } // GetAccount returns the account details at the latest sealed block. @@ -45,7 +45,7 @@ func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) { sealed, err := b.state.Sealed().Head() if err != nil { - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index 2a1c1fdc6be..b362eb90e44 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -3,10 +3,8 @@ package backend import ( "context" "fmt" - "github.com/onflow/flow-go/module/irrecoverable" "testing" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" @@ -19,10 +17,13 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/execution" execmock "github.com/onflow/flow-go/module/execution/mock" + "github.com/onflow/flow-go/module/irrecoverable" protocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/storage" storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" + + execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) type BackendAccountsSuite struct { @@ -46,8 +47,7 @@ type BackendAccountsSuite struct { account *flow.Account failingAddress flow.Address - ctx irrecoverable.SignalerContext - cancel context.CancelFunc + ctx irrecoverable.SignalerContext } func TestBackendAccountsSuite(t *testing.T) { @@ -76,7 +76,8 @@ func (s *BackendAccountsSuite) SetupTest() { s.Require().NoError(err) s.failingAddress = unittest.AddressFixture() - s.ctx, s.cancel = irrecoverable.NewMockSignalerContextWithCancel(s.T(), context.Background()) + + s.ctx = irrecoverable.NewMockSignalerContext(s.T(), context.Background()) } func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { @@ -88,7 +89,7 @@ func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { executionReceipts: s.receipts, connFactory: s.connectionFactory, nodeCommunicator: NewNodeCommunicator(false), - SignalCtx: s.ctx, + ctx: s.ctx, } } @@ -323,9 +324,9 @@ func (s *BackendAccountsSuite) TestGetAccountFromFailover_ReturnsENErrors() { }) } -// TestGetAccountFromStorage_Fails tests that errors received from local storage are handled -// and converted to the appropriate status code -func (s *BackendAccountsSuite) TestGetAccountState_Fails() { +// TestGetAccountAtLatestBlock_InconsistentState tests that signaler context received error when node state is +// inconsistent +func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_InconsistentState() { ctx := context.Background() scriptExecutor := execmock.NewScriptExecutor(s.T()) @@ -334,22 +335,15 @@ func (s *BackendAccountsSuite) TestGetAccountState_Fails() { backend.scriptExecMode = ScriptExecutionModeLocalOnly backend.scriptExecutor = scriptExecutor - s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", "head failed"), func() { //TODO: change - s.state.On("Sealed").Return(s.snapshot, nil).Once() - s.snapshot.On("Head").Return(nil, fmt.Errorf("head error")).Once() - - //defer func() { - // if r := recover(); r != nil { - // // Check if the panic was due to runtime.Goexit(). - // if r != nil { - // // Handle the case where runtime.Goexit was called. - // s.T().Logf("Caught runtime.Goexit()") - // } - // } - //}() - //backend.GetAccountAtLatestBlock(ctx, s.failingAddress) - - s.Assert().Panics(func() { backend.GetAccountAtLatestBlock(ctx, s.failingAddress) }, "head failed") + s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", "inconsistent node`s state"), func() { + s.state.On("Sealed").Return(s.snapshot, nil) + + err := fmt.Errorf("inconsistent node`s state") + s.snapshot.On("Head").Return(nil, err) + backend.ctx = irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + + _, err = backend.GetAccountAtLatestBlock(ctx, s.failingAddress) + s.Require().Error(err) }) } diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index bfc50c52bcd..b7f1ceefe38 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -2,21 +2,21 @@ package backend import ( "context" - "github.com/onflow/flow-go/module/irrecoverable" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) type backendBlockDetails struct { - blocks storage.Blocks - state protocol.State - SignalCtx irrecoverable.SignalerContext + blocks storage.Blocks + state protocol.State + ctx irrecoverable.SignalerContext } func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { @@ -42,7 +42,7 @@ func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) ( // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block: %v", err) } @@ -96,7 +96,7 @@ func (b *backendBlockDetails) getBlockStatus(block *flow.Block) (flow.BlockStatu // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index 4873174c143..d55ed15f210 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -2,21 +2,21 @@ package backend import ( "context" - "github.com/onflow/flow-go/module/irrecoverable" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) type backendBlockHeaders struct { - headers storage.Headers - state protocol.State - SignalCtx irrecoverable.SignalerContext + headers storage.Headers + state protocol.State + ctx irrecoverable.SignalerContext } func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { @@ -41,7 +41,7 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed b // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block header: %v", err) } @@ -89,7 +89,7 @@ func (b *backendBlockHeaders) getBlockStatus(header *flow.Header) (flow.BlockSta // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 89a4441638c..0e2c8c4b7ef 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/onflow/flow-go/module/irrecoverable" "time" "github.com/onflow/flow/protobuf/go/flow/entities" @@ -19,6 +18,7 @@ import ( "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -31,7 +31,7 @@ type backendEvents struct { log zerolog.Logger maxHeightRange uint nodeCommunicator Communicator - SignalCtx irrecoverable.SignalerContext + ctx irrecoverable.SignalerContext } // GetEventsForHeightRange retrieves events for all sealed blocks between the start block height and @@ -56,7 +56,7 @@ func (b *backendEvents) GetEventsForHeightRange( head, err := b.state.Sealed().Head() if err != nil { // sealed block must be in the store, so return an Internal code even if we got NotFound - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get events: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index d9c8f3e4aae..699d2021469 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -5,7 +5,6 @@ import ( "context" "crypto/md5" //nolint:gosec "errors" - "github.com/onflow/flow-go/module/irrecoverable" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -20,6 +19,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/execution" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" @@ -39,7 +39,7 @@ type backendScripts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode - SignalCtx irrecoverable.SignalerContext + ctx irrecoverable.SignalerContext } // ExecuteScriptAtLatestBlock executes provided script at the latest sealed block. @@ -51,7 +51,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestHeader, err := b.state.Sealed().Head() if err != nil { // the latest sealed header MUST be available - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 951adc9b50c..c65fbea99ce 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/execution" execmock "github.com/onflow/flow-go/module/execution/mock" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" protocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/storage" @@ -59,6 +60,7 @@ type BackendScriptsSuite struct { script []byte arguments [][]byte failingScript []byte + ctx irrecoverable.SignalerContext } func TestBackendScriptsSuite(t *testing.T) { @@ -85,6 +87,7 @@ func (s *BackendScriptsSuite) SetupTest() { s.script = []byte("pub fun main() { return 1 }") s.arguments = [][]byte{[]byte("arg1"), []byte("arg2")} s.failingScript = []byte("pub fun main() { panic(\"!!\") }") + s.ctx = irrecoverable.NewMockSignalerContext(s.T(), context.Background()) } func (s *BackendScriptsSuite) defaultBackend() *backendScripts { @@ -100,6 +103,7 @@ func (s *BackendScriptsSuite) defaultBackend() *backendScripts { loggedScripts: loggedScripts, connFactory: s.connectionFactory, nodeCommunicator: NewNodeCommunicator(false), + ctx: s.ctx, } } @@ -381,6 +385,29 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_ReturnsENErrors() { }) } +// TestExecuteScriptAtLatestBlockFromStorage_InconsistentState tests that signaler context received error when node state is +// inconsistent +func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_InconsistentState() { + ctx := context.Background() + + scriptExecutor := execmock.NewScriptExecutor(s.T()) + + backend := s.defaultBackend() + backend.scriptExecMode = ScriptExecutionModeLocalOnly + backend.scriptExecutor = scriptExecutor + + s.Run(fmt.Sprintf("ExecuteScriptAtLatestBlock - fails with %v", "inconsistent node`s state"), func() { + s.state.On("Sealed").Return(s.snapshot, nil) + + err := fmt.Errorf("inconsistent node`s state") + s.snapshot.On("Head").Return(nil, err) + backend.ctx = irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + + _, err = backend.ExecuteScriptAtLatestBlock(ctx, s.script, s.arguments) + s.Require().Error(err) + }) +} + func (s *BackendScriptsSuite) testExecuteScriptAtLatestBlock(ctx context.Context, backend *backendScripts, statusCode codes.Code) { s.state.On("Sealed").Return(s.snapshot, nil).Once() s.snapshot.On("Head").Return(s.block.Header, nil).Once() diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index fd7980af277..641a3899fb8 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -44,7 +44,7 @@ type backendTransactions struct { nodeCommunicator Communicator txResultCache *lru.Cache[flow.Identifier, *access.TransactionResult] - SignalCtx irrecoverable.SignalerContext + ctx irrecoverable.SignalerContext } // SendTransaction forwards the transaction to the collection node @@ -579,14 +579,13 @@ func (b *backendTransactions) deriveTransactionStatus( // Not in a block, let's see if it's expired referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() if err != nil { - b.SignalCtx.Throw(err) return flow.TransactionStatusUnknown, err } refHeight := referenceBlock.Height // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return flow.TransactionStatusUnknown, err } finalizedHeight := finalized.Height @@ -631,7 +630,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return flow.TransactionStatusUnknown, err } @@ -747,7 +746,7 @@ func (b *backendTransactions) getHistoricalTransactionResult( func (b *backendTransactions) registerTransactionForRetry(tx *flow.TransactionBody) { referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() if err != nil { - b.SignalCtx.Throw(err) + b.ctx.Throw(err) return } From 8756a4ad91b3c316480ad498a2e1ca7296bd9da7 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 3 Nov 2023 11:33:50 +0200 Subject: [PATCH 05/70] Added more tests --- engine/access/rpc/backend/backend.go | 3 +- engine/access/rpc/backend/backend_test.go | 188 +++++++++++++++------- 2 files changed, 132 insertions(+), 59 deletions(-) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 4752a3748c9..93fd2b3c135 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -76,8 +76,7 @@ type Backend struct { connFactory connection.ConnectionFactory // cache the response to GetNodeVersionInfo since it doesn't change - nodeInfo *access.NodeVersionInfo - SignalCtx irrecoverable.SignalerContext + nodeInfo *access.NodeVersionInfo } type Params struct { diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 20002bd0b3d..4143b43a345 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -25,6 +25,7 @@ import ( connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" bprotocol "github.com/onflow/flow-go/state/protocol/badger" protocol "github.com/onflow/flow-go/state/protocol/mock" @@ -396,29 +397,41 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_HistoryLimit() { func (suite *Suite) TestGetLatestSealedBlockHeader() { // setup the mocks suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - - block := unittest.BlockHeaderFixture() - suite.snapshot.On("Head").Return(block, nil).Once() - suite.state.On("Sealed").Return(suite.snapshot, nil) - suite.snapshot.On("Head").Return(block, nil).Once() params := suite.defaultBackendParams() backend, err := New(params) suite.Require().NoError(err) - // query the handler for the latest sealed block - header, stat, err := backend.GetLatestBlockHeader(context.Background(), true) - suite.checkResponse(header, err) + suite.Run("GetLatestSealedBlockHeader - happy path", func() { + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil).Once() + suite.snapshot.On("Head").Return(block, nil).Once() - // make sure we got the latest sealed block - suite.Require().Equal(block.ID(), header.ID()) - suite.Require().Equal(block.Height, header.Height) - suite.Require().Equal(block.ParentID, header.ParentID) - suite.Require().Equal(stat, flow.BlockStatusSealed) + // query the handler for the latest sealed block + header, stat, err := backend.GetLatestBlockHeader(context.Background(), true) + suite.checkResponse(header, err) - suite.assertAllExpectations() + // make sure we got the latest sealed block + suite.Require().Equal(block.ID(), header.ID()) + suite.Require().Equal(block.Height, header.Height) + suite.Require().Equal(block.ParentID, header.ParentID) + suite.Require().Equal(stat, flow.BlockStatusSealed) + + suite.assertAllExpectations() + }) + + suite.Run(fmt.Sprintf("GetLatestSealedBlockHeader - fails with %v", "inconsistent node`s state"), func() { + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + // mock signaler context expect an error + backend.backendBlockHeaders.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + + _, _, err = backend.GetLatestBlockHeader(context.Background(), true) + suite.Require().Error(err) + }) } func (suite *Suite) TestGetTransaction() { @@ -477,8 +490,6 @@ func (suite *Suite) TestGetTransactionResultByIndex() { blockId := block.ID() index := uint32(0) - suite.snapshot.On("Head").Return(block.Header, nil) - // block storage returns the corresponding block suite.blocks. On("ByID", blockId). @@ -486,11 +497,11 @@ func (suite *Suite) TestGetTransactionResultByIndex() { _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil).Maybe() // create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil).Maybe() exeEventReq := &execproto.GetTransactionByIndexRequest{ BlockId: blockId[:], @@ -511,20 +522,31 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.execClient. On("GetTransactionResultByIndex", ctx, exeEventReq). - Return(exeEventResp, nil). - Once() + Return(exeEventResp, nil) - result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) - suite.checkResponse(result, err) - suite.Assert().Equal(result.BlockHeight, block.Header.Height) + suite.Run("TestGetTransactionResultByIndex - happy path", func() { + suite.snapshot.On("Head").Return(block.Header, nil).Once() + result, err := backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + suite.checkResponse(result, err) + suite.Assert().Equal(result.BlockHeight, block.Header.Height) - suite.assertAllExpectations() + suite.assertAllExpectations() + }) + + suite.Run(fmt.Sprintf("TestGetTransactionResultByIndex - fails with %v", "inconsistent node`s state"), func() { + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + // mock signaler context expect an error + backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + + _, err = backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + suite.Require().Error(err) + }) } func (suite *Suite) TestGetTransactionResultsByBlockID() { - head := unittest.BlockHeaderFixture() suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Head").Return(head, nil).Maybe() ctx := context.Background() block := unittest.BlockFixture() @@ -561,13 +583,28 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.execClient. On("GetTransactionResultsByBlockID", ctx, exeEventReq). - Return(exeEventResp, nil). - Once() + Return(exeEventResp, nil) - result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) - suite.checkResponse(result, err) + suite.Run("GetTransactionResultsByBlockID - happy path", func() { + head := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(head, nil).Once() - suite.assertAllExpectations() + result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + suite.checkResponse(result, err) + + suite.assertAllExpectations() + }) + + suite.Run(fmt.Sprintf("GetTransactionResultsByBlockID - fails with %v", "inconsistent node`s state"), func() { + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + // mock signaler context expect an error + backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + + _, err = backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + suite.Require().Error(err) + }) } // TestTransactionStatusTransition tests that the status of transaction changes from Finalized to Sealed @@ -944,40 +981,53 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - // setup the mocks - expected := unittest.BlockFixture() - header := expected.Header + params := suite.defaultBackendParams() - suite.snapshot. - On("Head"). - Return(header, nil).Once() + backend, err := New(params) + suite.Require().NoError(err) - headerClone := *header - headerClone.Height = 0 + suite.Run("GetLatestFinalizedBlock - happy path", func() { + // setup the mocks + expected := unittest.BlockFixture() + header := expected.Header - suite.snapshot. - On("Head"). - Return(&headerClone, nil). - Once() + suite.snapshot. + On("Head"). + Return(header, nil).Once() - suite.blocks. - On("ByHeight", header.Height). - Return(&expected, nil) + headerClone := *header + headerClone.Height = 0 - params := suite.defaultBackendParams() + suite.snapshot. + On("Head"). + Return(&headerClone, nil). + Once() - backend, err := New(params) - suite.Require().NoError(err) + suite.blocks. + On("ByHeight", header.Height). + Return(&expected, nil) - // query the handler for the latest finalized header - actual, stat, err := backend.GetLatestBlock(context.Background(), false) - suite.checkResponse(actual, err) + // query the handler for the latest finalized header + actual, stat, err := backend.GetLatestBlock(context.Background(), false) + suite.checkResponse(actual, err) - // make sure we got the latest header - suite.Require().Equal(expected, *actual) - suite.Assert().Equal(stat, flow.BlockStatusFinalized) + // make sure we got the latest header + suite.Require().Equal(expected, *actual) + suite.Assert().Equal(stat, flow.BlockStatusFinalized) - suite.assertAllExpectations() + suite.assertAllExpectations() + }) + + suite.Run(fmt.Sprintf("GetLatestFinalizedBlock - fails with %v", "inconsistent node`s state"), func() { + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + // mock signaler context expect an error + backend.backendBlockDetails.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + + _, _, err = backend.GetLatestBlock(context.Background(), false) + suite.Require().Error(err) + }) } type mockCloser struct{} @@ -1260,7 +1310,6 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { } func (suite *Suite) TestGetEventsForHeightRange() { - ctx := context.Background() const minHeight uint64 = 5 const maxHeight uint64 = 10 @@ -1376,6 +1425,31 @@ func (suite *Suite) TestGetEventsForHeightRange() { return results } + suite.Run("inconsistent node`s state", func() { + headHeight = maxHeight - 1 + setupHeadHeight(headHeight) + + // setup mocks + stateParams.On("SporkID").Return(unittest.IdentifierFixture(), nil) + stateParams.On("ProtocolVersion").Return(uint(unittest.Uint64InRange(10, 30)), nil) + stateParams.On("SporkRootBlockHeight").Return(headHeight, nil) + stateParams.On("SealedRoot").Return(head, nil) + + params := suite.defaultBackendParams() + params.State = state + + backend, err := New(params) + suite.Require().NoError(err) + + backend.backendEvents.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + err = fmt.Errorf("inconsistent node`s state") + snapshot.On("Head").Return(nil, err) + + _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, + entitiesproto.EventEncodingVersion_JSON_CDC_V0) + suite.Require().Error(err) + }) + connFactory := suite.setupConnectionFactory() //suite.state = state From f4fe82043884f11dce0eaf7ec6462f13b1ff6c15 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 3 Nov 2023 16:06:17 +0200 Subject: [PATCH 06/70] Added more comments --- engine/access/rpc/backend/backend.go | 2 ++ engine/access/rpc/backend/backend_test.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 93fd2b3c135..922573dda04 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -219,6 +219,8 @@ func New(params Params) (*Backend, error) { return b, nil } +// HandleInconsistentProtocolState is helper function that initializes an irrecoverable SignalerContext for backends +// and makes it available to endpoints handling inconsistent or corrupted node's state. func (b *Backend) HandleInconsistentProtocolState(ctx irrecoverable.SignalerContext) { b.backendTransactions.ctx = ctx b.backendAccounts.ctx = ctx diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 4143b43a345..106b7fd6703 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -422,6 +422,7 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { suite.assertAllExpectations() }) + // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("GetLatestSealedBlockHeader - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -533,6 +534,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.assertAllExpectations() }) + // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("TestGetTransactionResultByIndex - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -595,6 +597,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.assertAllExpectations() }) + // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("GetTransactionResultsByBlockID - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -1018,6 +1021,7 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { suite.assertAllExpectations() }) + // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("GetLatestFinalizedBlock - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -1425,6 +1429,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { return results } + // tests that signaler context received error when node state is inconsistent suite.Run("inconsistent node`s state", func() { headHeight = maxHeight - 1 setupHeadHeight(headHeight) From e5f9d4ea793237e47e9eeec1088c286e7db68a89 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 6 Nov 2023 13:57:39 +0200 Subject: [PATCH 07/70] Fixed GetTransactionResultsByBlockID unit test --- engine/access/rpc/backend/backend_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 106b7fd6703..e199a2ca2ca 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -548,7 +548,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { } func (suite *Suite) TestGetTransactionResultsByBlockID() { - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + suite.state.On("Sealed").Return(suite.snapshot, nil) ctx := context.Background() block := unittest.BlockFixture() @@ -560,8 +560,8 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(&block, nil) _, fixedENIDs := suite.setupReceipts(&block) - suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) + suite.state.On("Final").Return(suite.snapshot, nil) + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil).Maybe() // create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) @@ -600,7 +600,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("GetTransactionResultsByBlockID - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") - suite.snapshot.On("Head").Return(nil, err) + suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) From 8b9e414180d8f15a63af4fb97ba094d52fdebdd1 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 6 Nov 2023 15:09:16 +0200 Subject: [PATCH 08/70] Removed unnecessary TODO comments --- engine/access/rpc/backend/backend_block_details.go | 2 -- engine/access/rpc/backend/backend_block_headers.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index b7f1ceefe38..8b57072a76c 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -37,7 +37,6 @@ func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) ( // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, // we should halt processing requests, but do throw an exception which might cause a crash: // - It is unsafe to process requests if we have an internally bad state. - // TODO: https://github.com/onflow/flow-go/issues/4028 // - We would like to avoid throwing an exception as a result of an Access API request by policy // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will @@ -91,7 +90,6 @@ func (b *backendBlockDetails) getBlockStatus(block *flow.Block) (flow.BlockStatu // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, // we should halt processing requests, but do throw an exception which might cause a crash: // - It is unsafe to process requests if we have an internally bad state. - // TODO: https://github.com/onflow/flow-go/issues/4028 // - We would like to avoid throwing an exception as a result of an Access API request by policy // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index d55ed15f210..1520aa770db 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -36,7 +36,6 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed b // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, // we should halt processing requests, but do throw an exception which might cause a crash: // - It is unsafe to process requests if we have an internally bad state. - // TODO: https://github.com/onflow/flow-go/issues/4028 // - We would like to avoid throwing an exception as a result of an Access API request by policy // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will @@ -84,7 +83,6 @@ func (b *backendBlockHeaders) getBlockStatus(header *flow.Header) (flow.BlockSta // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, // we should halt processing requests, but do throw an exception which might cause a crash: // - It is unsafe to process requests if we have an internally bad State. - // TODO: https://github.com/onflow/flow-go/issues/4028 // - We would like to avoid throwing an exception as a result of an Access API request by policy // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will From 1b8119e8ba00418c39b2ab9a8df5f0a70e97d6cb Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 6 Nov 2023 16:34:05 +0200 Subject: [PATCH 09/70] Updated rpc backend tests --- engine/access/rpc/backend/backend_test.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index e199a2ca2ca..479094faf59 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -498,11 +498,11 @@ func (suite *Suite) TestGetTransactionResultByIndex() { _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil).Maybe() + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) // create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil).Maybe() + connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) exeEventReq := &execproto.GetTransactionByIndexRequest{ BlockId: blockId[:], @@ -537,7 +537,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { // tests that signaler context received error when node state is inconsistent suite.Run(fmt.Sprintf("TestGetTransactionResultByIndex - fails with %v", "inconsistent node`s state"), func() { err := fmt.Errorf("inconsistent node`s state") - suite.snapshot.On("Head").Return(nil, err) + suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) @@ -548,7 +548,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { } func (suite *Suite) TestGetTransactionResultsByBlockID() { - suite.state.On("Sealed").Return(suite.snapshot, nil) + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() ctx := context.Background() block := unittest.BlockFixture() @@ -560,8 +560,8 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(&block, nil) _, fixedENIDs := suite.setupReceipts(&block) - suite.state.On("Final").Return(suite.snapshot, nil) - suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil).Maybe() + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) // create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) @@ -588,8 +588,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { Return(exeEventResp, nil) suite.Run("GetTransactionResultsByBlockID - happy path", func() { - head := unittest.BlockHeaderFixture() - suite.snapshot.On("Head").Return(head, nil).Once() + suite.snapshot.On("Head").Return(block.Header, nil).Once() result, err := backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.checkResponse(result, err) From 128156c9b1182af7fa19e5557109fa37fbe96d45 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 7 Nov 2023 13:02:20 +0200 Subject: [PATCH 10/70] Fixed TestGetTransactionResultsByBlockID unit test --- engine/access/rpc/backend/backend_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 479094faf59..06cffe4c154 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -423,7 +423,7 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { }) // tests that signaler context received error when node state is inconsistent - suite.Run(fmt.Sprintf("GetLatestSealedBlockHeader - fails with %v", "inconsistent node`s state"), func() { + suite.Run("GetLatestSealedBlockHeader - fails with inconsistent node`s state", func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -535,7 +535,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { }) // tests that signaler context received error when node state is inconsistent - suite.Run(fmt.Sprintf("TestGetTransactionResultByIndex - fails with %v", "inconsistent node`s state"), func() { + suite.Run("TestGetTransactionResultByIndex - fails with inconsistent node`s state", func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err).Once() @@ -551,7 +551,12 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() ctx := context.Background() + params := suite.defaultBackendParams() + block := unittest.BlockFixture() + sporkRootBlockHeight, err := suite.state.Params().SporkRootBlockHeight() + suite.Require().NoError(err) + block.Header.Height = sporkRootBlockHeight + 1 blockId := block.ID() // block storage returns the corresponding block @@ -575,7 +580,6 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { TransactionResults: []*execproto.GetTransactionResultResponse{{}}, } - params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client params.ConnFactory = connFactory params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() @@ -597,7 +601,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { }) // tests that signaler context received error when node state is inconsistent - suite.Run(fmt.Sprintf("GetTransactionResultsByBlockID - fails with %v", "inconsistent node`s state"), func() { + suite.Run("GetTransactionResultsByBlockID - fails with inconsistent node`s state", func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err).Once() @@ -1021,7 +1025,7 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { }) // tests that signaler context received error when node state is inconsistent - suite.Run(fmt.Sprintf("GetLatestFinalizedBlock - fails with %v", "inconsistent node`s state"), func() { + suite.Run("GetLatestFinalizedBlock - fails with inconsistent node`s state", func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) From f512095ff6cb88cabb5718666715dabe6d504b8e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Sun, 12 Nov 2023 23:35:47 -0500 Subject: [PATCH 11/70] add comment to IsClusterPrefixed field --- network/p2p/consumers.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index 2e65e38c6b1..8ecff25e04d 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -33,7 +33,11 @@ type InvCtrlMsgNotif struct { Error error // MsgType the control message type. MsgType p2pmsg.ControlMessageType - // IsClusterPrefixed indicates if the error occurred on a cluster prefixed topic of the control message. + // IsClusterPrefixed reports whether the error occurred on a cluster-prefixed topic within the control message. + // Notifications must be explicitly marked as cluster-prefixed or not because the penalty applied to the GossipSub score + // for an error on a cluster-prefixed topic is more lenient than the penalty applied to a non-cluster-prefixed topic. + // This distinction ensures that nodes engaged in cluster-prefixed topic communication are not penalized too harshly, + // as such communication is vital to the progress of the chain. IsClusterPrefixed bool } From ae59eb23581f560eb06c6ed85ec20d38f208a6e3 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 13 Nov 2023 01:09:36 -0500 Subject: [PATCH 12/70] correct penalty application in score registry - apply penalty reduction to notification penalty, not the entire record record.Penalty - add tests that ensures concurrent notifications one with cluster prefix one without computes the expected penalty and app score --- network/p2p/scoring/registry.go | 15 +++-- network/p2p/scoring/registry_test.go | 92 ++++++++++++++++++++++------ 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 1bdec348776..a7df3815cca 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -266,17 +266,18 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( } record, err := r.spamScoreCache.Update(notification.PeerID, func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + penalty := 0.0 switch notification.MsgType { case p2pmsg.CtrlMsgGraft: - record.Penalty += r.penalty.Graft + penalty += r.penalty.Graft case p2pmsg.CtrlMsgPrune: - record.Penalty += r.penalty.Prune + penalty += r.penalty.Prune case p2pmsg.CtrlMsgIHave: - record.Penalty += r.penalty.IHave + penalty += r.penalty.IHave case p2pmsg.CtrlMsgIWant: - record.Penalty += r.penalty.IWant + penalty += r.penalty.IWant case p2pmsg.RpcPublishMessage: - record.Penalty += r.penalty.RpcPublishMessage + penalty += r.penalty.RpcPublishMessage default: // the error is considered fatal as it means that we have an unsupported misbehaviour type, we should crash the node to prevent routing attack vulnerability. lg.Fatal().Str("misbehavior_type", notification.MsgType.String()).Msg("unknown misbehaviour type") @@ -284,9 +285,11 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( // reduce penalty for cluster prefixed topics allowing nodes that are potentially behind to catch up if notification.IsClusterPrefixed { - record.Penalty *= r.penalty.ClusterPrefixedPenaltyReductionFactor + penalty *= r.penalty.ClusterPrefixedPenaltyReductionFactor } + record.Penalty += penalty + return record }) if err != nil { diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index ab078af0e9a..771d2d4fefb 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -3,6 +3,7 @@ package scoring_test import ( "fmt" "math" + "sync" "testing" "time" @@ -25,7 +26,7 @@ import ( // app specific reward. This is the default reward for a staked peer that has valid subscriptions and has not been // penalized. func TestNoPenaltyRecord(t *testing.T) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), @@ -49,27 +50,24 @@ func TestNoPenaltyRecord(t *testing.T) { // penalty value as the app specific score. func TestPeerWithSpamRecord(t *testing.T) { t.Run("graft", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft, false) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune, false) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave, false) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant, false) - }) - t.Run("cluster prefixed", func(t *testing.T) { - testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant*penaltyValueFixtures().ClusterPrefixedPenaltyReductionFactor, true) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) t.Run("RpcPublishMessage", func(t *testing.T) { testPeerWithSpamRecord(t, p2pmsg.RpcPublishMessage, penaltyValueFixtures().RpcPublishMessage) }) } -func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64, isClusterPrefixed bool) { - peerID := peer.ID("peer-1") +func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), @@ -85,9 +83,8 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ - PeerID: peerID, - MsgType: messageType, - IsClusterPrefixed: isClusterPrefixed, + PeerID: peerID, + MsgType: messageType, }) // the penalty should now be updated in the spamRecords @@ -124,7 +121,7 @@ func TestSpamRecord_With_UnknownIdentity(t *testing.T) { // testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and // the peer id has an unknown identity. func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withUnknownIdentity(peerID), @@ -177,7 +174,7 @@ func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { // testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and // the peer id has an invalid subscription as well. func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), @@ -211,7 +208,7 @@ func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.Cont // TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. func TestSpamPenaltyDecaysInCache(t *testing.T) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, _ := newGossipSubAppSpecificScoreRegistry(t, withStakedIdentity(peerID), withValidSubscriptions(peerID)) @@ -274,7 +271,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { // TestSpamPenaltyDecayToZero tests that the spam penalty decays to zero over time, and when the spam penalty of // a peer is set back to zero, its app specific penalty is also reset to the initial state. func TestSpamPenaltyDecayToZero(t *testing.T) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), @@ -320,7 +317,7 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { // TestPersistingUnknownIdentityPenalty tests that even though the spam penalty is decayed to zero, the unknown identity penalty // is persisted. This is because the unknown identity penalty is not decayed. func TestPersistingUnknownIdentityPenalty(t *testing.T) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withUnknownIdentity(peerID), // the peer id has an unknown identity. @@ -377,7 +374,7 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { // TestPersistingInvalidSubscriptionPenalty tests that even though the spam penalty is decayed to zero, the invalid subscription penalty // is persisted. This is because the invalid subscription penalty is not decayed. func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { - peerID := peer.ID("peer-1") + peerID := unittest.PeerIdFixture(t) reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, withStakedIdentity(peerID), @@ -426,6 +423,63 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. } +// TestPeerSpamPenaltyClusterPrefixed evaluates the application-specific penalty calculation for a node when a spam record is present +// for cluster-prefixed topics. In the case of an invalid control message notification marked as cluster-prefixed, +// the application-specific penalty should be reduced by the default reduction factor. This test verifies the accurate computation +// of the application-specific score under these conditions. +func TestPeerSpamPenaltyClusterPrefixed(t *testing.T) { + peerID := unittest.PeerIdFixture(t) + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withValidSubscriptions(peerID)) + + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + score := reg.AppSpecificScoreFunc()(peerID) + assert.Equal(t, scoring.MaxAppSpecificReward, score) + + // Report consecutive misbehavior's for the specified peer ID. Two misbehavior's are reported concurrently: + // 1. With IsClusterPrefixed set to false, ensuring the penalty applied to the application-specific score is not reduced. + // 2. With IsClusterPrefixed set to true, reducing the penalty added to the overall app-specific score by the default reduction factor. + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + IsClusterPrefixed: false, + }) + }() + go func() { + defer wg.Done() + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + IsClusterPrefixed: true, + }) + }() + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // expected penalty should be penaltyValueFixtures().Graft * (1 + clusterReductionFactor) + expectedPenalty := penaltyValueFixtures().Graft * (1 + penaltyValueFixtures().ClusterPrefixedPenaltyReductionFactor) + + // the penalty should now be updated in the spamRecords + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) + // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, + // and the peer should be deprived of the default reward for its valid staked role. + score = reg.AppSpecificScoreFunc()(peerID) + assert.Less(t, math.Abs(expectedPenalty-score), 10e-3) +} + // withStakedIdentity returns a function that sets the identity provider to return an staked identity for the given peer id. // It is used for testing purposes, and causes the given peer id to benefit from the staked identity reward in GossipSub. func withStakedIdentity(peerId peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { From 1acc984c87e444a5e2cdaf4d957ff12e74f1de71 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 13 Nov 2023 01:21:51 -0500 Subject: [PATCH 13/70] update control validation inspector tests ensure disseminated notifications have correct IsClusterPrefixed value set --- .../validation_inspector_test.go | 6 ++++ ...ntrol_message_validation_inspector_test.go | 28 ++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 8cc13683864..aa03de6df96 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -62,6 +62,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, channels.IsInvalidTopicErr(notification.Error)) switch notification.MsgType { @@ -197,6 +198,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.True(t, validation.IsDuplicateTopicErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) switch notification.MsgType { @@ -302,6 +304,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.True(t, validation.IsDuplicateTopicErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.CtrlMsgIHave, fmt.Sprintf("unexpected control message type %s error: %s", notification.MsgType, notification.Error)) @@ -410,6 +413,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.True(t, notification.IsClusterPrefixed) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, channels.IsUnknownClusterIDErr(notification.Error)) switch notification.MsgType { @@ -794,6 +798,7 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { return func(args mockery.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.CtrlMsgIWant, fmt.Sprintf("unexpected control message type %s error: %s", notification.MsgType, notification.Error)) require.True(t, validation.IsIWantCacheMissThresholdErr(notification.Error)) @@ -927,6 +932,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { return func(args mockery.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.RpcPublishMessage, fmt.Sprintf("unexpected control message type %s error: %s", notification.MsgType, notification.Error)) require.True(t, validation.IsInvalidRpcPublishMessagesErr(notification.Error)) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index e04f404c710..09dd59276b1 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -389,6 +389,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(func(args mock.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, from, notification.PeerID) require.Contains(t, []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave}, notification.MsgType) require.True(t, validation.IsDuplicateTopicErr(notification.Error)) @@ -421,7 +422,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) suite.inspector.Start(suite.signalerCtx) @@ -450,7 +451,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) suite.inspector.Start(suite.signalerCtx) @@ -480,7 +481,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) suite.inspector.Start(suite.signalerCtx) @@ -506,7 +507,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) @@ -528,7 +529,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { @@ -557,7 +558,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold @@ -634,7 +635,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns return topics })) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) @@ -654,7 +655,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) // set topic oracle to return list of topics excluding first topic sent require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) require.NoError(t, suite.inspector.Inspect(from, rpc)) @@ -677,7 +678,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns // set topic oracle to return list of topics excluding first topic sent require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) require.NoError(t, suite.inspector.Inspect(from, rpc)) @@ -719,7 +720,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) suite.idProvider.On("ByPeerID", from).Return(nil, false).Times(501) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) require.NoError(t, suite.inspector.Inspect(from, rpc)) @@ -747,7 +748,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns suite.idProvider.On("ByPeerID", from).Return(id, true).Times(501) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) require.NoError(t, err, "failed to get inspect message request") - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) suite.inspector.Start(suite.signalerCtx) require.NoError(t, suite.inspector.Inspect(from, rpc)) @@ -829,7 +830,7 @@ func (suite *ControlMsgValidationInspectorSuite) TestNewControlMsgValidationInsp from := unittest.PeerIdFixture(t) identity := unittest.IdentityFixture() suite.idProvider.On("ByPeerID", from).Return(identity, true).Times(11) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsUnknownClusterIDErr) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsUnknownClusterIDErr, true) inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) suite.inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) @@ -909,10 +910,11 @@ func invalidTopics(t *testing.T, sporkID flow.Identifier) (string, string, strin } // checkNotificationFunc returns util func used to ensure invalid control message notification disseminated contains expected information. -func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType p2pmsg.ControlMessageType, isExpectedErr func(err error) bool) func(args mock.Arguments) { +func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType p2pmsg.ControlMessageType, isExpectedErr func(err error) bool, isClusterPrefixed bool) func(args mock.Arguments) { return func(args mock.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) + require.Equal(t, isClusterPrefixed, notification.IsClusterPrefixed) require.Equal(t, expectedPeerID, notification.PeerID) require.Equal(t, expectedMsgType, notification.MsgType) require.True(t, isExpectedErr(notification.Error)) From 78381fbca6bd9111ed284a205b0583b9c351247c Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 14 Nov 2023 00:42:55 -0500 Subject: [PATCH 14/70] add IsClusterPrefixed value to log message --- .../inspector/validation/control_message_validation_inspector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 0b5c118b6a6..a632f5aacca 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -817,6 +817,7 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In Str("control_message_type", ctlMsgType.String()). Bool(logging.KeySuspicious, true). Bool(logging.KeyNetworkingSecurity, true). + Bool("is_cluster_prefixed", isClusterPrefixed). Uint64("error_count", count). Str("peer_id", p2plogging.PeerId(req.Peer)). Logger() From 1e123ee14e77e1685a1854a8b2df1347caf51240 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 16 Nov 2023 16:18:03 +0200 Subject: [PATCH 15/70] Implemented second approach for issue, added unit test --- .../access/handle_irrecoverable_state_test.go | 226 ++++++++++++++++++ engine/access/rpc/backend/backend.go | 12 - engine/access/rpc/backend/backend_accounts.go | 3 +- .../rpc/backend/backend_accounts_test.go | 6 +- .../rpc/backend/backend_block_details.go | 19 +- .../rpc/backend/backend_block_headers.go | 19 +- engine/access/rpc/backend/backend_events.go | 3 +- engine/access/rpc/backend/backend_scripts.go | 3 +- .../rpc/backend/backend_scripts_test.go | 61 ++--- engine/access/rpc/backend/backend_test.go | 17 +- .../rpc/backend/backend_transactions.go | 8 +- engine/access/rpc/engine.go | 7 +- module/grpcserver/server.go | 17 +- module/grpcserver/server_builder.go | 23 +- module/irrecoverable/irrecoverable.go | 5 +- 15 files changed, 320 insertions(+), 109 deletions(-) create mode 100644 engine/access/handle_irrecoverable_state_test.go diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go new file mode 100644 index 00000000000..5ffc8c70bd9 --- /dev/null +++ b/engine/access/handle_irrecoverable_state_test.go @@ -0,0 +1,226 @@ +package access + +import ( + "context" + "fmt" + "io" + "os" + "testing" + "time" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/onflow/flow-go/crypto" + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" + statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow-go/utils/unittest" +) + +// IrrecoverableStateTestSuite tests that Access node indicate an inconsistent or corrupted node state +type IrrecoverableStateTestSuite struct { + suite.Suite + state *protocol.State + snapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + log zerolog.Logger + net *network.EngineRegistry + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + chainID flow.ChainID + metrics *metrics.NoopCollector + rpcEng *rpc.Engine + publicKey crypto.PublicKey + + // storage + blocks *storagemock.Blocks + headers *storagemock.Headers + collections *storagemock.Collections + transactions *storagemock.Transactions + receipts *storagemock.ExecutionReceipts + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer +} + +func (suite *IrrecoverableStateTestSuite) SetupTest() { + suite.log = zerolog.New(os.Stdout) + suite.net = new(network.EngineRegistry) + suite.state = new(protocol.State) + suite.snapshot = new(protocol.Snapshot) + + rootHeader := unittest.BlockHeaderFixture() + params := new(protocol.Params) + params.On("SporkID").Return(unittest.IdentifierFixture(), nil) + params.On("ProtocolVersion").Return(uint(unittest.Uint64InRange(10, 30)), nil) + params.On("SporkRootBlockHeight").Return(rootHeader.Height, nil) + params.On("SealedRoot").Return(rootHeader, nil) + + suite.epochQuery = new(protocol.EpochQuery) + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.state.On("Params").Return(params, nil).Maybe() + suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.blocks = new(storagemock.Blocks) + suite.headers = new(storagemock.Headers) + suite.transactions = new(storagemock.Transactions) + suite.collections = new(storagemock.Collections) + suite.receipts = new(storagemock.ExecutionReceipts) + + suite.collClient = new(accessmock.AccessAPIClient) + suite.execClient = new(accessmock.ExecutionAPIClient) + + suite.request = new(module.Requester) + suite.request.On("EntityByID", mock.Anything, mock.Anything) + + suite.me = new(module.Local) + + accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) + suite.me. + On("NodeID"). + Return(accessIdentity.NodeID) + + suite.chainID = flow.Testnet + suite.metrics = metrics.NewNoopCollector() + + config := rpc.Config{ + UnsecureGRPCListenAddr: unittest.DefaultAddress, + SecureGRPCListenAddr: unittest.DefaultAddress, + HTTPListenAddr: unittest.DefaultAddress, + } + + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + // save the public key to use later in tests later + suite.publicKey = networkingKey.PublicKey() + + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil).Build() + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil).Once() + + bnd, err := backend.New(backend.Params{ + State: suite.state, + CollectionRPC: suite.collClient, + Blocks: suite.blocks, + Headers: suite.headers, + Collections: suite.collections, + Transactions: suite.transactions, + ChainID: suite.chainID, + AccessMetrics: suite.metrics, + MaxHeightRange: 0, + Log: suite.log, + SnapshotHistoryLimit: 0, + Communicator: backend.NewNodeCommunicator(false), + }) + suite.Require().NoError(err) + + stateStreamConfig := statestreambackend.Config{} + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + suite.me, + bnd, + bnd, + suite.secureGrpcServer, + suite.unsecureGrpcServer, + nil, + stateStreamConfig, + ) + assert.NoError(suite.T(), err) + suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() + assert.NoError(suite.T(), err) + + err = fmt.Errorf("inconsistent node`s state") + ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.rpcEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(ctx) + suite.unsecureGrpcServer.Start(ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + + // wait for the engine to startup + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) +} + +func TestIrrecoverableState(t *testing.T) { + suite.Run(t, new(IrrecoverableStateTestSuite)) +} + +func (suite *IrrecoverableStateTestSuite) TearDownTest() { + if suite.cancel != nil { + suite.cancel() + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) + } +} + +func (suite *IrrecoverableStateTestSuite) TestInconsistentNodeState() { + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + conn, err := grpc.Dial( + suite.unsecureGrpcServer.GRPCAddress().String(), + grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(suite.T(), err) + + defer io.Closer(conn).Close() + client := accessproto.NewAccessAPIClient(conn) + + req := &accessproto.GetAccountAtLatestBlockRequest{ + Address: unittest.AddressFixture().Bytes(), + } + _, err = client.GetAccountAtLatestBlock(context.Background(), req) + suite.Require().Error(err) +} diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index eaad6a6a2c9..25961ee51ab 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -19,7 +19,6 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/execution" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -219,17 +218,6 @@ func New(params Params) (*Backend, error) { return b, nil } -// HandleInconsistentProtocolState is helper function that initializes an irrecoverable SignalerContext for backends -// and makes it available to endpoints handling inconsistent or corrupted node's state. -func (b *Backend) HandleInconsistentProtocolState(ctx irrecoverable.SignalerContext) { - b.backendTransactions.ctx = ctx - b.backendAccounts.ctx = ctx - b.backendBlockDetails.ctx = ctx - b.backendBlockHeaders.ctx = ctx - b.backendEvents.ctx = ctx - b.backendScripts.ctx = ctx -} - // NewCache constructs cache for storing connections to other nodes. // No errors are expected during normal operations. func NewCache( diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index a384da297c8..86dfb78766c 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -32,7 +32,6 @@ type backendAccounts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode - ctx irrecoverable.SignalerContext } // GetAccount returns the account details at the latest sealed block. @@ -45,7 +44,7 @@ func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) { sealed, err := b.state.Sealed().Head() if err != nil { - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index b362eb90e44..b251720cb43 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -89,7 +89,6 @@ func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { executionReceipts: s.receipts, connFactory: s.connectionFactory, nodeCommunicator: NewNodeCommunicator(false), - ctx: s.ctx, } } @@ -327,8 +326,6 @@ func (s *BackendAccountsSuite) TestGetAccountFromFailover_ReturnsENErrors() { // TestGetAccountAtLatestBlock_InconsistentState tests that signaler context received error when node state is // inconsistent func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_InconsistentState() { - ctx := context.Background() - scriptExecutor := execmock.NewScriptExecutor(s.T()) backend := s.defaultBackend() @@ -340,7 +337,8 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - backend.ctx = irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + + ctx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) //Uliana: всі ctx _, err = backend.GetAccountAtLatestBlock(ctx, s.failingAddress) s.Require().Error(err) diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index 8b57072a76c..364d43a740d 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -16,10 +16,9 @@ import ( type backendBlockDetails struct { blocks storage.Blocks state protocol.State - ctx irrecoverable.SignalerContext } -func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { +func (b *backendBlockDetails) GetLatestBlock(ctx context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { var header *flow.Header var err error @@ -41,7 +40,7 @@ func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) ( // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block: %v", err) } @@ -51,40 +50,40 @@ func (b *backendBlockDetails) GetLatestBlock(_ context.Context, isSealed bool) ( return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block: %v", err) } - stat, err := b.getBlockStatus(block) + stat, err := b.getBlockStatus(ctx, block) if err != nil { return nil, stat, err } return block, stat, nil } -func (b *backendBlockDetails) GetBlockByID(_ context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) { +func (b *backendBlockDetails) GetBlockByID(ctx context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) { block, err := b.blocks.ByID(id) if err != nil { return nil, flow.BlockStatusUnknown, rpc.ConvertStorageError(err) } - stat, err := b.getBlockStatus(block) + stat, err := b.getBlockStatus(ctx, block) if err != nil { return nil, stat, err } return block, stat, nil } -func (b *backendBlockDetails) GetBlockByHeight(_ context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) { +func (b *backendBlockDetails) GetBlockByHeight(ctx context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) { block, err := b.blocks.ByHeight(height) if err != nil { return nil, flow.BlockStatusUnknown, rpc.ConvertStorageError(err) } - stat, err := b.getBlockStatus(block) + stat, err := b.getBlockStatus(ctx, block) if err != nil { return nil, stat, err } return block, stat, nil } -func (b *backendBlockDetails) getBlockStatus(block *flow.Block) (flow.BlockStatus, error) { +func (b *backendBlockDetails) getBlockStatus(ctx context.Context, block *flow.Block) (flow.BlockStatus, error) { sealed, err := b.state.Sealed().Head() if err != nil { // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, @@ -94,7 +93,7 @@ func (b *backendBlockDetails) getBlockStatus(block *flow.Block) (flow.BlockStatu // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index 1520aa770db..cf8a04c676d 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -16,10 +16,9 @@ import ( type backendBlockHeaders struct { headers storage.Headers state protocol.State - ctx irrecoverable.SignalerContext } -func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { +func (b *backendBlockHeaders) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { var header *flow.Header var err error @@ -40,44 +39,44 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(_ context.Context, isSealed b // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block header: %v", err) } - stat, err := b.getBlockStatus(header) + stat, err := b.getBlockStatus(ctx, header) if err != nil { return nil, stat, err } return header, stat, nil } -func (b *backendBlockHeaders) GetBlockHeaderByID(_ context.Context, id flow.Identifier) (*flow.Header, flow.BlockStatus, error) { +func (b *backendBlockHeaders) GetBlockHeaderByID(ctx context.Context, id flow.Identifier) (*flow.Header, flow.BlockStatus, error) { header, err := b.headers.ByBlockID(id) if err != nil { return nil, flow.BlockStatusUnknown, rpc.ConvertStorageError(err) } - stat, err := b.getBlockStatus(header) + stat, err := b.getBlockStatus(ctx, header) if err != nil { return nil, stat, err } return header, stat, nil } -func (b *backendBlockHeaders) GetBlockHeaderByHeight(_ context.Context, height uint64) (*flow.Header, flow.BlockStatus, error) { +func (b *backendBlockHeaders) GetBlockHeaderByHeight(ctx context.Context, height uint64) (*flow.Header, flow.BlockStatus, error) { header, err := b.headers.ByHeight(height) if err != nil { return nil, flow.BlockStatusUnknown, rpc.ConvertStorageError(err) } - stat, err := b.getBlockStatus(header) + stat, err := b.getBlockStatus(ctx, header) if err != nil { return nil, stat, err } return header, stat, nil } -func (b *backendBlockHeaders) getBlockStatus(header *flow.Header) (flow.BlockStatus, error) { +func (b *backendBlockHeaders) getBlockStatus(ctx context.Context, header *flow.Header) (flow.BlockStatus, error) { sealed, err := b.state.Sealed().Head() if err != nil { // In the RPC engine, if we encounter an error from the protocol state indicating state corruption, @@ -87,7 +86,7 @@ func (b *backendBlockHeaders) getBlockStatus(header *flow.Header) (flow.BlockSta // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 0e2c8c4b7ef..8b4b9dda5bb 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -31,7 +31,6 @@ type backendEvents struct { log zerolog.Logger maxHeightRange uint nodeCommunicator Communicator - ctx irrecoverable.SignalerContext } // GetEventsForHeightRange retrieves events for all sealed blocks between the start block height and @@ -56,7 +55,7 @@ func (b *backendEvents) GetEventsForHeightRange( head, err := b.state.Sealed().Head() if err != nil { // sealed block must be in the store, so return an Internal code even if we got NotFound - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return nil, status.Errorf(codes.Internal, "failed to get events: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index 69549b729e1..63270fc7e1c 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -38,7 +38,6 @@ type backendScripts struct { nodeCommunicator Communicator scriptExecutor execution.ScriptExecutor scriptExecMode ScriptExecutionMode - ctx irrecoverable.SignalerContext } // scriptExecutionRequest encapsulates the data needed to execute a script to make it easier @@ -74,7 +73,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestHeader, err := b.state.Sealed().Head() if err != nil { // the latest sealed header MUST be available - b.ctx.Throw(err) + irrecoverable.Throw(ctx, err) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index c65fbea99ce..16199fa9d7e 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -103,7 +103,6 @@ func (s *BackendScriptsSuite) defaultBackend() *backendScripts { loggedScripts: loggedScripts, connFactory: s.connectionFactory, nodeCommunicator: NewNodeCommunicator(false), - ctx: s.ctx, } } @@ -153,8 +152,6 @@ func (s *BackendScriptsSuite) setupENFailingResponse(blockID flow.Identifier, er // TestExecuteScriptOnExecutionNode_HappyPath tests that the backend successfully executes scripts // on execution nodes func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_HappyPath() { - ctx := context.Background() - s.setupExecutionNodes(s.block) s.setupENSuccessResponse(s.block.ID()) @@ -162,23 +159,21 @@ func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_HappyPath() { backend.scriptExecMode = ScriptExecutionModeExecutionNodesOnly s.Run("GetAccount", func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) }) } // TestExecuteScriptOnExecutionNode_Fails tests that the backend returns an error when the execution // node returns an error func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_Fails() { - ctx := context.Background() - // use a status code that's not used in the API to make sure it's passed through statusCode := codes.FailedPrecondition errToReturn := status.Error(statusCode, "random error") @@ -190,23 +185,21 @@ func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_Fails() { backend.scriptExecMode = ScriptExecutionModeExecutionNodesOnly s.Run("GetAccount", func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, statusCode) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(ctx, backend, statusCode) + s.testExecuteScriptAtBlockID(s.ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, statusCode) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, statusCode) }) } // TestExecuteScriptFromStorage_HappyPath tests that the backend successfully executes scripts using // the local storage func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_HappyPath() { - ctx := context.Background() - scriptExecutor := execmock.NewScriptExecutor(s.T()) scriptExecutor.On("ExecuteAtBlockHeight", mock.Anything, s.script, s.arguments, s.block.Header.Height). Return(expectedResponse, nil) @@ -216,23 +209,21 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_HappyPath() { backend.scriptExecutor = scriptExecutor s.Run("GetAccount - happy path", func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) }) s.Run("GetAccountAtLatestBlock - happy path", func() { - s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) }) s.Run("GetAccountAtBlockHeight - happy path", func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) }) } // TestExecuteScriptFromStorage_Fails tests that errors received from local storage are handled // and converted to the appropriate status code func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { - ctx := context.Background() - scriptExecutor := execmock.NewScriptExecutor(s.T()) backend := s.defaultBackend() @@ -270,15 +261,15 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { Return(nil, tt.err).Times(3) s.Run(fmt.Sprintf("GetAccount - fails with %v", tt.err), func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, tt.statusCode) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, tt.statusCode) }) s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", tt.err), func() { - s.testExecuteScriptAtBlockID(ctx, backend, tt.statusCode) + s.testExecuteScriptAtBlockID(s.ctx, backend, tt.statusCode) }) s.Run(fmt.Sprintf("GetAccountAtBlockHeight - fails with %v", tt.err), func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, tt.statusCode) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, tt.statusCode) }) } } @@ -286,8 +277,6 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { // TestExecuteScriptWithFailover_HappyPath tests that when an error is returned executing a script // from local storage, the backend will attempt to run it on an execution node func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { - ctx := context.Background() - errors := []error{ execution.ErrDataNotAvailable, storage.ErrNotFound, @@ -310,15 +299,15 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { Return(nil, errToReturn).Times(3) s.Run(fmt.Sprintf("ExecuteScriptAtLatestBlock - recovers %v", errToReturn), func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) }) s.Run(fmt.Sprintf("ExecuteScriptAtBlockID - recovers %v", errToReturn), func() { - s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) }) s.Run(fmt.Sprintf("ExecuteScriptAtBlockHeight - recovers %v", errToReturn), func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) }) } } @@ -326,8 +315,6 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { // TestExecuteScriptWithFailover_SkippedForInvalidArgument tests that failover is skipped for // FVM errors that result in InvalidArgument errors func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_SkippedForInvalidArgument() { - ctx := context.Background() - // configure local script executor to fail scriptExecutor := execmock.NewScriptExecutor(s.T()) scriptExecutor.On("ExecuteAtBlockHeight", mock.Anything, s.failingScript, s.arguments, s.block.Header.Height). @@ -338,23 +325,21 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_SkippedForInvalidArg backend.scriptExecutor = scriptExecutor s.Run("ExecuteScriptAtLatestBlock", func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.InvalidArgument) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtBlockID(s.ctx, backend, codes.InvalidArgument) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.InvalidArgument) }) } // TestExecuteScriptWithFailover_ReturnsENErrors tests that when an error is returned from the execution // node during a failover, it is returned to the caller. func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_ReturnsENErrors() { - ctx := context.Background() - // use a status code that's not used in the API to make sure it's passed through statusCode := codes.FailedPrecondition errToReturn := status.Error(statusCode, "random error") @@ -373,23 +358,21 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_ReturnsENErrors() { backend.scriptExecutor = scriptExecutor s.Run("ExecuteScriptAtLatestBlock", func() { - s.testExecuteScriptAtLatestBlock(ctx, backend, statusCode) + s.testExecuteScriptAtLatestBlock(s.ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(ctx, backend, statusCode) + s.testExecuteScriptAtBlockID(s.ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(ctx, backend, statusCode) + s.testExecuteScriptAtBlockHeight(s.ctx, backend, statusCode) }) } // TestExecuteScriptAtLatestBlockFromStorage_InconsistentState tests that signaler context received error when node state is // inconsistent func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_InconsistentState() { - ctx := context.Background() - scriptExecutor := execmock.NewScriptExecutor(s.T()) backend := s.defaultBackend() @@ -401,7 +384,7 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - backend.ctx = irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + ctx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) _, err = backend.ExecuteScriptAtLatestBlock(ctx, s.script, s.arguments) s.Require().Error(err) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 06cffe4c154..f9cc0fa0f7f 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -428,9 +428,9 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - backend.backendBlockHeaders.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) - _, _, err = backend.GetLatestBlockHeader(context.Background(), true) + _, _, err = backend.GetLatestBlockHeader(ctx, true) suite.Require().Error(err) }) } @@ -540,7 +540,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) _, err = backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) @@ -606,7 +606,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - backend.backendTransactions.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) _, err = backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) @@ -961,7 +961,6 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // TestTransactionResultUnknown tests that the status of transaction is reported as unknown when it is not found in the // local storage func (suite *Suite) TestTransactionResultUnknown() { - ctx := context.Background() txID := unittest.IdentifierFixture() @@ -984,6 +983,8 @@ func (suite *Suite) TestTransactionResultUnknown() { } func (suite *Suite) TestGetLatestFinalizedBlock() { + ctx := context.Background() + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() @@ -1030,9 +1031,9 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - backend.backendBlockDetails.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) - _, _, err = backend.GetLatestBlock(context.Background(), false) + _, _, err = backend.GetLatestBlock(ctx, false) suite.Require().Error(err) }) } @@ -1449,7 +1450,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - backend.backendEvents.ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) err = fmt.Errorf("inconsistent node`s state") snapshot.On("Head").Return(nil, err) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 641a3899fb8..011de54551f 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -21,7 +21,6 @@ import ( "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -43,8 +42,6 @@ type backendTransactions struct { log zerolog.Logger nodeCommunicator Communicator txResultCache *lru.Cache[flow.Identifier, *access.TransactionResult] - - ctx irrecoverable.SignalerContext } // SendTransaction forwards the transaction to the collection node @@ -585,7 +582,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - b.ctx.Throw(err) + //irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } finalizedHeight := finalized.Height @@ -630,7 +627,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - b.ctx.Throw(err) + //irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } @@ -746,7 +743,6 @@ func (b *backendTransactions) getHistoricalTransactionResult( func (b *backendTransactions) registerTransactionForRetry(tx *flow.TransactionBody) { referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() if err != nil { - b.ctx.Throw(err) return } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index a7fe0080c54..8efe778a549 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -129,10 +129,6 @@ func NewBuilder(log zerolog.Logger, AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). AddWorker(backendNotifierWorker). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - backend.HandleInconsistentProtocolState(ctx) - ready() - }). AddWorker(eng.shutdownWorker). Build() @@ -233,6 +229,9 @@ func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.Re return } e.restServer = r + e.restServer.BaseContext = func(_ net.Listener) context.Context { + return ctx + } l, err := net.Listen("tcp", e.config.RestConfig.ListenAddress) if err != nil { diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index 7ca1c478688..231fa1dd6dc 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -4,6 +4,8 @@ import ( "net" "sync" + "go.uber.org/atomic" + "github.com/rs/zerolog" "google.golang.org/grpc" @@ -20,8 +22,9 @@ import ( // into different engines making it possible to use single grpc server for multiple services which live in different modules. type GrpcServer struct { component.Component - log zerolog.Logger - Server *grpc.Server + log zerolog.Logger + Server *grpc.Server + grpcSignalerCtx *atomic.Pointer[irrecoverable.SignalerContext] grpcListenAddr string // the GRPC server address as ip:port @@ -35,11 +38,13 @@ var _ component.Component = (*GrpcServer)(nil) func NewGrpcServer(log zerolog.Logger, grpcListenAddr string, grpcServer *grpc.Server, + grpcSignalerCtx *atomic.Pointer[irrecoverable.SignalerContext], ) *GrpcServer { server := &GrpcServer{ - log: log, - Server: grpcServer, - grpcListenAddr: grpcListenAddr, + log: log, + Server: grpcServer, + grpcListenAddr: grpcListenAddr, + grpcSignalerCtx: grpcSignalerCtx, } server.Component = component.NewComponentManagerBuilder(). AddWorker(server.serveGRPCWorker). @@ -54,6 +59,8 @@ func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready co g.log = g.log.With().Str("grpc_address", g.grpcListenAddr).Logger() g.log.Info().Msg("starting grpc server on address") + g.grpcSignalerCtx.Store(&ctx) + l, err := net.Listen("tcp", g.grpcListenAddr) if err != nil { g.log.Err(err).Msg("failed to start the grpc server") diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index d42196cdf12..c4ee5c9af5f 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -1,14 +1,18 @@ package grpcserver import ( + "context" + + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/rs/zerolog" + "go.uber.org/atomic" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "github.com/onflow/flow-go/engine/common/rpc" + "github.com/onflow/flow-go/module/irrecoverable" ) type Option func(*GrpcServerBuilder) @@ -33,6 +37,7 @@ type GrpcServerBuilder struct { log zerolog.Logger gRPCListenAddr string server *grpc.Server + signalerCtx *atomic.Pointer[irrecoverable.SignalerContext] transportCredentials credentials.TransportCredentials // the GRPC credentials stateStreamInterceptorEnable bool @@ -57,12 +62,23 @@ func NewGrpcServerBuilder(log zerolog.Logger, applyOption(grpcServerBuilder) } + signalerCtx := atomic.NewPointer[irrecoverable.SignalerContext](nil) + // create a GRPC server to serve GRPC clients grpcOpts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(int(maxMsgSize)), grpc.MaxSendMsgSize(int(maxMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors + interceptors = append(interceptors, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + if signalerCtx := signalerCtx.Load(); ctx != nil { + resp, err = handler(*signalerCtx, req) + } else { + resp, err = handler(ctx, req) + } + return + }) + // if rpc metrics is enabled, first create the grpc metrics interceptor if rpcMetricsEnabled { interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) @@ -98,10 +114,11 @@ func NewGrpcServerBuilder(log zerolog.Logger, } grpcServerBuilder.log = log grpcServerBuilder.server = grpc.NewServer(grpcOpts...) + grpcServerBuilder.signalerCtx = signalerCtx return grpcServerBuilder } func (b *GrpcServerBuilder) Build() *GrpcServer { - return NewGrpcServer(b.log, b.gRPCListenAddr, b.server) + return NewGrpcServer(b.log, b.gRPCListenAddr, b.server, b.signalerCtx) } diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index 1ef79f5f4ab..15dae6a072e 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -75,9 +75,10 @@ func Throw(ctx context.Context, err error) { signalerAbleContext, ok := ctx.(SignalerContext) if ok { signalerAbleContext.Throw(err) + } else { + // Be spectacular on how this does not -but should- handle irrecoverables: + log.Fatalf("irrecoverable error signaler not found for context, please implement! Unhandled irrecoverable error %v", err) } - // Be spectacular on how this does not -but should- handle irrecoverables: - log.Fatalf("irrecoverable error signaler not found for context, please implement! Unhandled irrecoverable error %v", err) } // WithSignallerAndCancel returns an irrecoverable context, the cancel From 8df6e31e645535d40bff5ccb6e1db71dfbb5424b Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 20 Nov 2023 18:32:10 +0200 Subject: [PATCH 16/70] Fixed casting SignalerContext, added rest unit test for checking handle irrecoverable state --- .../access/handle_irrecoverable_state_test.go | 65 ++++++++++++++----- .../rpc/backend/backend_accounts_test.go | 5 +- .../rpc/backend/backend_scripts_test.go | 6 +- engine/access/rpc/backend/backend_test.go | 28 ++++---- .../rpc/backend/backend_transactions.go | 14 ++-- engine/access/rpc/backend/retry.go | 3 +- engine/access/rpc/engine.go | 2 +- module/grpcserver/server_builder.go | 5 +- module/irrecoverable/irrecoverable.go | 4 +- 9 files changed, 86 insertions(+), 46 deletions(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 5ffc8c70bd9..e08bde33e71 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/antihax/optional" accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -17,8 +18,12 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + restclient "github.com/onflow/flow/openapi/go-client-generated" + "github.com/onflow/flow-go/crypto" accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rest" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" @@ -58,8 +63,7 @@ type IrrecoverableStateTestSuite struct { transactions *storagemock.Transactions receipts *storagemock.ExecutionReceipts - ctx irrecoverable.SignalerContext - cancel context.CancelFunc + ctx irrecoverable.SignalerContext // grpc servers secureGrpcServer *grpcserver.GrpcServer @@ -110,6 +114,9 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { UnsecureGRPCListenAddr: unittest.DefaultAddress, SecureGRPCListenAddr: unittest.DefaultAddress, HTTPListenAddr: unittest.DefaultAddress, + RestConfig: rest.Config{ + ListenAddress: unittest.DefaultAddress, + }, } // generate a server certificate that will be served by the GRPC server @@ -137,8 +144,8 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { nil, nil).Build() - block := unittest.BlockHeaderFixture() - suite.snapshot.On("Head").Return(block, nil).Once() + blockHeader := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(blockHeader, nil).Once() bnd, err := backend.New(backend.Params{ State: suite.state, @@ -179,8 +186,7 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { err = fmt.Errorf("inconsistent node`s state") ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) - suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) - suite.rpcEng.Start(suite.ctx) + suite.rpcEng.Start(ctx) suite.secureGrpcServer.Start(ctx) suite.unsecureGrpcServer.Start(ctx) @@ -197,16 +203,7 @@ func TestIrrecoverableState(t *testing.T) { suite.Run(t, new(IrrecoverableStateTestSuite)) } -func (suite *IrrecoverableStateTestSuite) TearDownTest() { - if suite.cancel != nil { - suite.cancel() - unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) - unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) - } -} - -func (suite *IrrecoverableStateTestSuite) TestInconsistentNodeState() { +func (suite *IrrecoverableStateTestSuite) TestGRPCInconsistentNodeState() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -214,13 +211,45 @@ func (suite *IrrecoverableStateTestSuite) TestInconsistentNodeState() { suite.unsecureGrpcServer.GRPCAddress().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) assert.NoError(suite.T(), err) - defer io.Closer(conn).Close() + client := accessproto.NewAccessAPIClient(conn) req := &accessproto.GetAccountAtLatestBlockRequest{ Address: unittest.AddressFixture().Bytes(), } - _, err = client.GetAccountAtLatestBlock(context.Background(), req) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + _, err = client.GetAccountAtLatestBlock(ctx, req) suite.Require().Error(err) } + +func (suite *IrrecoverableStateTestSuite) TestRestInconsistentNodeState() { + collections := unittest.CollectionListFixture(1) + blockHeader := unittest.BlockWithGuaranteesFixture( + unittest.CollectionGuaranteesWithCollectionIDFixture(collections), + ) + suite.blocks.On("ByID", blockHeader.ID()).Return(blockHeader, nil) + + err := fmt.Errorf("inconsistent node`s state") + suite.snapshot.On("Head").Return(nil, err) + + config := restclient.NewConfiguration() + config.BasePath = fmt.Sprintf("http://%s/v1", suite.rpcEng.RestApiAddress().String()) + client := restclient.NewAPIClient(config) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + _, _, err = client.BlocksApi.BlocksIdGet(ctx, []string{blockHeader.ID().String()}, optionsForBlocksIdGetOpts()) + suite.Require().Error(err) +} + +func optionsForBlocksIdGetOpts() *restclient.BlocksApiBlocksIdGetOpts { + return &restclient.BlocksApiBlocksIdGetOpts{ + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), + Select_: optional.NewInterface([]string{"header.id"}), + } +} diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index b251720cb43..e0fc2157807 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -338,9 +338,10 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - ctx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) //Uliana: всі ctx + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetAccountAtLatestBlock(ctx, s.failingAddress) + _, err = backend.GetAccountAtLatestBlock(valueCtx, s.failingAddress) s.Require().Error(err) }) } diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 16199fa9d7e..3004a977ca2 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -384,9 +384,11 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - ctx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) - _, err = backend.ExecuteScriptAtLatestBlock(ctx, s.script, s.arguments) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + + _, err = backend.ExecuteScriptAtLatestBlock(valueCtx, s.script, s.arguments) s.Require().Error(err) }) } diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index f9cc0fa0f7f..90301e6e8f1 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -428,9 +428,10 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, _, err = backend.GetLatestBlockHeader(ctx, true) + _, _, err = backend.GetLatestBlockHeader(valueCtx, true) suite.Require().Error(err) }) } @@ -522,7 +523,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.Require().NoError(err) suite.execClient. - On("GetTransactionResultByIndex", ctx, exeEventReq). + On("GetTransactionResultByIndex", mock.Anything, exeEventReq). Return(exeEventResp, nil) suite.Run("TestGetTransactionResultByIndex - happy path", func() { @@ -540,9 +541,10 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetTransactionResultByIndex(ctx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetTransactionResultByIndex(valueCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) }) } @@ -588,7 +590,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.Require().NoError(err) suite.execClient. - On("GetTransactionResultsByBlockID", ctx, exeEventReq). + On("GetTransactionResultsByBlockID", mock.Anything, exeEventReq). Return(exeEventResp, nil) suite.Run("GetTransactionResultsByBlockID - happy path", func() { @@ -600,15 +602,16 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.assertAllExpectations() }) - // tests that signaler context received error when node state is inconsistent + //tests that signaler context received error when node state is inconsistent suite.Run("GetTransactionResultsByBlockID - fails with inconsistent node`s state", func() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetTransactionResultsByBlockID(ctx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + _, err = backend.GetTransactionResultsByBlockID(valueCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) }) } @@ -983,8 +986,6 @@ func (suite *Suite) TestTransactionResultUnknown() { } func (suite *Suite) TestGetLatestFinalizedBlock() { - ctx := context.Background() - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() @@ -1031,9 +1032,10 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, _, err = backend.GetLatestBlock(ctx, false) + _, _, err = backend.GetLatestBlock(valueCtx, false) suite.Require().Error(err) }) } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 011de54551f..469e027f811 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -315,7 +316,7 @@ func (b *backendTransactions) GetTransactionResult( } // derive status of the transaction - txStatus, err := b.deriveTransactionStatus(tx, transactionWasExecuted, block) + txStatus, err := b.deriveTransactionStatus(ctx, tx, transactionWasExecuted, block) if err != nil { return nil, rpc.ConvertStorageError(err) } @@ -432,7 +433,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( txResult := resp.TransactionResults[i] // tx body is irrelevant to status if it's in an executed block - txStatus, err := b.deriveTransactionStatus(nil, true, block) + txStatus, err := b.deriveTransactionStatus(ctx, nil, true, block) if err != nil { return nil, rpc.ConvertStorageError(err) } @@ -486,7 +487,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, status.Errorf(codes.Internal, "could not get system chunk transaction: %v", err) } systemTxResult := resp.TransactionResults[len(resp.TransactionResults)-1] - systemTxStatus, err := b.deriveTransactionStatus(systemTx, true, block) + systemTxStatus, err := b.deriveTransactionStatus(ctx, systemTx, true, block) if err != nil { return nil, rpc.ConvertStorageError(err) } @@ -544,7 +545,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( } // tx body is irrelevant to status if it's in an executed block - txStatus, err := b.deriveTransactionStatus(nil, true, block) + txStatus, err := b.deriveTransactionStatus(ctx, nil, true, block) if err != nil { return nil, rpc.ConvertStorageError(err) } @@ -567,6 +568,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( // deriveTransactionStatus derives the transaction status based on current protocol state func (b *backendTransactions) deriveTransactionStatus( + ctx context.Context, tx *flow.TransactionBody, executed bool, block *flow.Block, @@ -582,7 +584,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - //irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } finalizedHeight := finalized.Height @@ -627,7 +629,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - //irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } diff --git a/engine/access/rpc/backend/retry.go b/engine/access/rpc/backend/retry.go index 5d967e657bb..287e34c475e 100644 --- a/engine/access/rpc/backend/retry.go +++ b/engine/access/rpc/backend/retry.go @@ -109,8 +109,9 @@ func (r *Retry) retryTxsAtHeight(heightToRetry uint64) { block = nil } + // TODO: change context.Background() for SignalerContext // find the transaction status - status, err := r.backend.deriveTransactionStatus(tx, false, block) + status, err := r.backend.deriveTransactionStatus(context.Background(), tx, false, block) if err != nil { continue } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 8efe778a549..e96f7f2acfa 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -230,7 +230,7 @@ func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.Re } e.restServer = r e.restServer.BaseContext = func(_ net.Listener) context.Context { - return ctx + return context.WithValue(ctx, irrecoverable.SignalerContextKey{}, ctx) } l, err := net.Listen("tcp", e.config.RestConfig.ListenAddress) diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index c4ee5c9af5f..e8bfab070a0 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -71,8 +71,9 @@ func NewGrpcServerBuilder(log zerolog.Logger, } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors interceptors = append(interceptors, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { - if signalerCtx := signalerCtx.Load(); ctx != nil { - resp, err = handler(*signalerCtx, req) + if signalerCtx := signalerCtx.Load(); signalerCtx != nil { + valueCtx := context.WithValue(ctx, irrecoverable.SignalerContextKey{}, *signalerCtx) + resp, err = handler(valueCtx, req) } else { resp, err = handler(ctx, req) } diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index 15dae6a072e..baa3fd708d2 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -48,6 +48,8 @@ type SignalerContext interface { sealed() // private, to constrain builder to using WithSignaler } +type SignalerContextKey struct{} + // private, to force context derivation / WithSignaler type signalerCtx struct { context.Context @@ -72,7 +74,7 @@ func WithSignaler(parent context.Context) (SignalerContext, <-chan error) { // Throw can be a drop-in replacement anywhere we have a context.Context likely // to support Irrecoverables. Note: this is not a method func Throw(ctx context.Context, err error) { - signalerAbleContext, ok := ctx.(SignalerContext) + signalerAbleContext, ok := ctx.Value(SignalerContextKey{}).(SignalerContext) if ok { signalerAbleContext.Throw(err) } else { From a5d90db3dc4a928db0f4b984bb1cb63e97a9386f Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 20 Nov 2023 18:44:30 +0200 Subject: [PATCH 17/70] Updated unit tests --- .../rpc/backend/backend_scripts_test.go | 58 +++++++++++-------- engine/access/rpc/backend/retry.go | 2 +- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 3004a977ca2..158579165e9 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -60,7 +60,6 @@ type BackendScriptsSuite struct { script []byte arguments [][]byte failingScript []byte - ctx irrecoverable.SignalerContext } func TestBackendScriptsSuite(t *testing.T) { @@ -87,7 +86,6 @@ func (s *BackendScriptsSuite) SetupTest() { s.script = []byte("pub fun main() { return 1 }") s.arguments = [][]byte{[]byte("arg1"), []byte("arg2")} s.failingScript = []byte("pub fun main() { panic(\"!!\") }") - s.ctx = irrecoverable.NewMockSignalerContext(s.T(), context.Background()) } func (s *BackendScriptsSuite) defaultBackend() *backendScripts { @@ -152,6 +150,8 @@ func (s *BackendScriptsSuite) setupENFailingResponse(blockID flow.Identifier, er // TestExecuteScriptOnExecutionNode_HappyPath tests that the backend successfully executes scripts // on execution nodes func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_HappyPath() { + ctx := context.Background() + s.setupExecutionNodes(s.block) s.setupENSuccessResponse(s.block.ID()) @@ -159,21 +159,23 @@ func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_HappyPath() { backend.scriptExecMode = ScriptExecutionModeExecutionNodesOnly s.Run("GetAccount", func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) }) } // TestExecuteScriptOnExecutionNode_Fails tests that the backend returns an error when the execution // node returns an error func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_Fails() { + ctx := context.Background() + // use a status code that's not used in the API to make sure it's passed through statusCode := codes.FailedPrecondition errToReturn := status.Error(statusCode, "random error") @@ -185,21 +187,23 @@ func (s *BackendScriptsSuite) TestExecuteScriptOnExecutionNode_Fails() { backend.scriptExecMode = ScriptExecutionModeExecutionNodesOnly s.Run("GetAccount", func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, statusCode) + s.testExecuteScriptAtLatestBlock(ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, statusCode) + s.testExecuteScriptAtBlockID(ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, statusCode) + s.testExecuteScriptAtBlockHeight(ctx, backend, statusCode) }) } // TestExecuteScriptFromStorage_HappyPath tests that the backend successfully executes scripts using // the local storage func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_HappyPath() { + ctx := context.Background() + scriptExecutor := execmock.NewScriptExecutor(s.T()) scriptExecutor.On("ExecuteAtBlockHeight", mock.Anything, s.script, s.arguments, s.block.Header.Height). Return(expectedResponse, nil) @@ -209,21 +213,23 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_HappyPath() { backend.scriptExecutor = scriptExecutor s.Run("GetAccount - happy path", func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) }) s.Run("GetAccountAtLatestBlock - happy path", func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) }) s.Run("GetAccountAtBlockHeight - happy path", func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) }) } // TestExecuteScriptFromStorage_Fails tests that errors received from local storage are handled // and converted to the appropriate status code func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { + ctx := context.Background() + scriptExecutor := execmock.NewScriptExecutor(s.T()) backend := s.defaultBackend() @@ -261,15 +267,15 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { Return(nil, tt.err).Times(3) s.Run(fmt.Sprintf("GetAccount - fails with %v", tt.err), func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, tt.statusCode) + s.testExecuteScriptAtLatestBlock(ctx, backend, tt.statusCode) }) s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", tt.err), func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, tt.statusCode) + s.testExecuteScriptAtBlockID(ctx, backend, tt.statusCode) }) s.Run(fmt.Sprintf("GetAccountAtBlockHeight - fails with %v", tt.err), func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, tt.statusCode) + s.testExecuteScriptAtBlockHeight(ctx, backend, tt.statusCode) }) } } @@ -277,6 +283,8 @@ func (s *BackendScriptsSuite) TestExecuteScriptFromStorage_Fails() { // TestExecuteScriptWithFailover_HappyPath tests that when an error is returned executing a script // from local storage, the backend will attempt to run it on an execution node func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { + ctx := context.Background() + errors := []error{ execution.ErrDataNotAvailable, storage.ErrNotFound, @@ -299,15 +307,15 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { Return(nil, errToReturn).Times(3) s.Run(fmt.Sprintf("ExecuteScriptAtLatestBlock - recovers %v", errToReturn), func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.OK) + s.testExecuteScriptAtLatestBlock(ctx, backend, codes.OK) }) s.Run(fmt.Sprintf("ExecuteScriptAtBlockID - recovers %v", errToReturn), func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockID(ctx, backend, codes.OK) }) s.Run(fmt.Sprintf("ExecuteScriptAtBlockHeight - recovers %v", errToReturn), func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.OK) + s.testExecuteScriptAtBlockHeight(ctx, backend, codes.OK) }) } } @@ -315,6 +323,8 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_HappyPath() { // TestExecuteScriptWithFailover_SkippedForInvalidArgument tests that failover is skipped for // FVM errors that result in InvalidArgument errors func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_SkippedForInvalidArgument() { + ctx := context.Background() + // configure local script executor to fail scriptExecutor := execmock.NewScriptExecutor(s.T()) scriptExecutor.On("ExecuteAtBlockHeight", mock.Anything, s.failingScript, s.arguments, s.block.Header.Height). @@ -325,21 +335,23 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_SkippedForInvalidArg backend.scriptExecutor = scriptExecutor s.Run("ExecuteScriptAtLatestBlock", func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtLatestBlock(ctx, backend, codes.InvalidArgument) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtBlockID(ctx, backend, codes.InvalidArgument) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, codes.InvalidArgument) + s.testExecuteScriptAtBlockHeight(ctx, backend, codes.InvalidArgument) }) } // TestExecuteScriptWithFailover_ReturnsENErrors tests that when an error is returned from the execution // node during a failover, it is returned to the caller. func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_ReturnsENErrors() { + ctx := context.Background() + // use a status code that's not used in the API to make sure it's passed through statusCode := codes.FailedPrecondition errToReturn := status.Error(statusCode, "random error") @@ -358,15 +370,15 @@ func (s *BackendScriptsSuite) TestExecuteScriptWithFailover_ReturnsENErrors() { backend.scriptExecutor = scriptExecutor s.Run("ExecuteScriptAtLatestBlock", func() { - s.testExecuteScriptAtLatestBlock(s.ctx, backend, statusCode) + s.testExecuteScriptAtLatestBlock(ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockID", func() { - s.testExecuteScriptAtBlockID(s.ctx, backend, statusCode) + s.testExecuteScriptAtBlockID(ctx, backend, statusCode) }) s.Run("ExecuteScriptAtBlockHeight", func() { - s.testExecuteScriptAtBlockHeight(s.ctx, backend, statusCode) + s.testExecuteScriptAtBlockHeight(ctx, backend, statusCode) }) } diff --git a/engine/access/rpc/backend/retry.go b/engine/access/rpc/backend/retry.go index 287e34c475e..b5390a2495c 100644 --- a/engine/access/rpc/backend/retry.go +++ b/engine/access/rpc/backend/retry.go @@ -109,7 +109,7 @@ func (r *Retry) retryTxsAtHeight(heightToRetry uint64) { block = nil } - // TODO: change context.Background() for SignalerContext + // TODO: change context.Background() to SignalerContext // find the transaction status status, err := r.backend.deriveTransactionStatus(context.Background(), tx, false, block) if err != nil { From a22739f1ddae0b6e669324d4cd1a4cc3dcf2f7de Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 20 Nov 2023 18:47:58 +0200 Subject: [PATCH 18/70] Removed unnecessary ctx value in unit tests --- engine/access/rpc/backend/backend_accounts_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index e0fc2157807..cf3e90e3965 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -46,8 +46,6 @@ type BackendAccountsSuite struct { block *flow.Block account *flow.Account failingAddress flow.Address - - ctx irrecoverable.SignalerContext } func TestBackendAccountsSuite(t *testing.T) { @@ -76,12 +74,9 @@ func (s *BackendAccountsSuite) SetupTest() { s.Require().NoError(err) s.failingAddress = unittest.AddressFixture() - - s.ctx = irrecoverable.NewMockSignalerContext(s.T(), context.Background()) } func (s *BackendAccountsSuite) defaultBackend() *backendAccounts { - return &backendAccounts{ log: s.log, state: s.state, From 66023eb4392d6f66429b9abb6071172dbbb980b0 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 23 Nov 2023 16:21:09 +0200 Subject: [PATCH 19/70] Handled errors in Retry for rpc backend, updated handling errors in transactions backend, updated unit tests --- engine/access/rpc/backend/backend.go | 2 +- .../rpc/backend/backend_transactions.go | 38 +++++++++++----- engine/access/rpc/backend/retry.go | 45 ++++++++++--------- engine/access/rpc/backend/retry_test.go | 25 ++++++----- engine/access/rpc/engine.go | 11 +++-- 5 files changed, 72 insertions(+), 49 deletions(-) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 25961ee51ab..3667be7a49b 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -105,7 +105,7 @@ type Params struct { // New creates backend instance func New(params Params) (*Backend, error) { - retry := newRetry() + retry := newRetry(params.Log) if params.RetryEnabled { retry.Activate() } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 469e027f811..70f2db46cf5 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -194,7 +195,7 @@ func (b *backendTransactions) GetTransaction(ctx context.Context, txID flow.Iden } func (b *backendTransactions) GetTransactionsByBlockID( - ctx context.Context, + _ context.Context, blockID flow.Identifier, ) ([]*flow.TransactionBody, error) { var transactions []*flow.TransactionBody @@ -316,8 +317,11 @@ func (b *backendTransactions) GetTransactionResult( } // derive status of the transaction - txStatus, err := b.deriveTransactionStatus(ctx, tx, transactionWasExecuted, block) + txStatus, err := b.deriveTransactionStatus(tx, transactionWasExecuted, block) if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + irrecoverable.Throw(ctx, err) + } return nil, rpc.ConvertStorageError(err) } @@ -433,8 +437,11 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( txResult := resp.TransactionResults[i] // tx body is irrelevant to status if it's in an executed block - txStatus, err := b.deriveTransactionStatus(ctx, nil, true, block) + txStatus, err := b.deriveTransactionStatus(nil, true, block) if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + irrecoverable.Throw(ctx, err) + } return nil, rpc.ConvertStorageError(err) } events, err := convert.MessagesToEventsWithEncodingConversion(txResult.GetEvents(), resp.GetEventEncodingVersion(), requiredEventEncodingVersion) @@ -487,8 +494,11 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, status.Errorf(codes.Internal, "could not get system chunk transaction: %v", err) } systemTxResult := resp.TransactionResults[len(resp.TransactionResults)-1] - systemTxStatus, err := b.deriveTransactionStatus(ctx, systemTx, true, block) + systemTxStatus, err := b.deriveTransactionStatus(systemTx, true, block) if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + irrecoverable.Throw(ctx, err) + } return nil, rpc.ConvertStorageError(err) } @@ -545,8 +555,11 @@ func (b *backendTransactions) GetTransactionResultByIndex( } // tx body is irrelevant to status if it's in an executed block - txStatus, err := b.deriveTransactionStatus(ctx, nil, true, block) + txStatus, err := b.deriveTransactionStatus(nil, true, block) if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + irrecoverable.Throw(ctx, err) + } return nil, rpc.ConvertStorageError(err) } @@ -567,13 +580,15 @@ func (b *backendTransactions) GetTransactionResultByIndex( } // deriveTransactionStatus derives the transaction status based on current protocol state + +// Error returns: +// - ErrUnknownSnapshotReference - block referenced by transaction has not been found. +// - all other errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). func (b *backendTransactions) deriveTransactionStatus( - ctx context.Context, tx *flow.TransactionBody, executed bool, block *flow.Block, ) (flow.TransactionStatus, error) { - if block == nil { // Not in a block, let's see if it's expired referenceBlock, err := b.state.AtBlockID(tx.ReferenceBlockID).Head() @@ -584,7 +599,6 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } finalizedHeight := finalized.Height @@ -629,7 +643,6 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - irrecoverable.Throw(ctx, err) return flow.TransactionStatusUnknown, err } @@ -651,6 +664,9 @@ func (b *backendTransactions) isExpired(refHeight, compareToHeight uint64) bool return compareToHeight-refHeight > flow.DefaultTransactionExpiry } +// Error returns: +// - ErrNotFound - collection referenced by transaction or block by a collection has not been found. +// - all other errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). func (b *backendTransactions) lookupBlock(txID flow.Identifier) (*flow.Block, error) { collection, err := b.collections.LightByTransactionID(txID) @@ -786,8 +802,8 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return events, resp.GetStatusCode(), resp.GetErrorMessage(), nil } -func (b *backendTransactions) NotifyFinalizedBlockHeight(height uint64) { - b.retry.Retry(height) +func (b *backendTransactions) ProcessFinalizedBlockHeight(height uint64) error { + return b.retry.Retry(height) } func (b *backendTransactions) getTransactionResultFromAnyExeNode( diff --git a/engine/access/rpc/backend/retry.go b/engine/access/rpc/backend/retry.go index b5390a2495c..07f1c3d58c8 100644 --- a/engine/access/rpc/backend/retry.go +++ b/engine/access/rpc/backend/retry.go @@ -3,9 +3,13 @@ package backend import ( "context" "errors" + "fmt" "sync" + "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/storage" ) @@ -19,10 +23,12 @@ type Retry struct { transactionByReferencBlockHeight map[uint64]map[flow.Identifier]*flow.TransactionBody backend *Backend active bool + log zerolog.Logger } -func newRetry() *Retry { +func newRetry(log zerolog.Logger) *Retry { return &Retry{ + log: log, transactionByReferencBlockHeight: map[uint64]map[flow.Identifier]*flow.TransactionBody{}, } } @@ -41,10 +47,10 @@ func (r *Retry) SetBackend(b *Backend) *Retry { return r } -func (r *Retry) Retry(height uint64) { +func (r *Retry) Retry(height uint64) error { // No need to retry if height is lower than DefaultTransactionExpiry if height < flow.DefaultTransactionExpiry { - return + return nil } // naive cleanup for now, prune every 120 Blocks @@ -55,20 +61,13 @@ func (r *Retry) Retry(height uint64) { heightToRetry := height - flow.DefaultTransactionExpiry + retryFrequency for heightToRetry < height { - r.retryTxsAtHeight(heightToRetry) - + err := r.retryTxsAtHeight(heightToRetry) + if err != nil { + return err + } heightToRetry = heightToRetry + retryFrequency } - -} - -func (b *Retry) Notify(signal interface{}) bool { - height, ok := signal.(uint64) - if !ok { - return false - } - b.Retry(height) - return true + return nil } // RegisterTransaction adds a transaction that could possibly be retried @@ -95,7 +94,7 @@ func (r *Retry) prune(height uint64) { } } -func (r *Retry) retryTxsAtHeight(heightToRetry uint64) { +func (r *Retry) retryTxsAtHeight(heightToRetry uint64) error { r.mu.Lock() defer r.mu.Unlock() txsAtHeight := r.transactionByReferencBlockHeight[heightToRetry] @@ -104,22 +103,28 @@ func (r *Retry) retryTxsAtHeight(heightToRetry uint64) { block, err := r.backend.lookupBlock(txID) if err != nil { if !errors.Is(err, storage.ErrNotFound) { - continue + return err } block = nil } - // TODO: change context.Background() to SignalerContext // find the transaction status - status, err := r.backend.deriveTransactionStatus(context.Background(), tx, false, block) + status, err := r.backend.deriveTransactionStatus(tx, false, block) if err != nil { + if !errors.Is(err, state.ErrUnknownSnapshotReference) { + return err + } continue } if status == flow.TransactionStatusPending { - _ = r.backend.SendRawTransaction(context.Background(), tx) + err = r.backend.SendRawTransaction(context.Background(), tx) + if err != nil { + r.log.Info().Str("retry", fmt.Sprintf("retryTxsAtHeight: %v", heightToRetry)).Err(err).Msg("failed to send raw transactions") + } } else if status != flow.TransactionStatusUnknown { // not pending or unknown, don't need to retry anymore delete(txsAtHeight, txID) } } + return nil } diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index 3b705049ff5..544c9a9669b 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -18,8 +18,6 @@ import ( // TestTransactionRetry tests that the retry mechanism will send retries at specific times func (suite *Suite) TestTransactionRetry() { - - // ctx := context.Background() collection := unittest.CollectionFixture(1) transactionBody := collection.Transactions[0] block := unittest.BlockFixture() @@ -44,7 +42,7 @@ func (suite *Suite) TestTransactionRetry() { backend, err := New(params) suite.Require().NoError(err) - retry := newRetry().SetBackend(backend).Activate() + retry := newRetry(suite.log).SetBackend(backend).Activate() backend.retry = retry retry.RegisterTransaction(block.Header.Height, transactionBody) @@ -52,17 +50,20 @@ func (suite *Suite) TestTransactionRetry() { suite.colClient.On("SendTransaction", mock.Anything, mock.Anything).Return(&access.SendTransactionResponse{}, nil) // Don't retry on every height - retry.Retry(block.Header.Height + 1) + err = retry.Retry(block.Header.Height + 1) + suite.Require().NoError(err) suite.colClient.AssertNotCalled(suite.T(), "SendTransaction", mock.Anything, mock.Anything) // Retry every `retryFrequency` - retry.Retry(block.Header.Height + retryFrequency) + err = retry.Retry(block.Header.Height + retryFrequency) + suite.Require().NoError(err) suite.colClient.AssertNumberOfCalls(suite.T(), "SendTransaction", 1) // do not retry if expired - retry.Retry(block.Header.Height + retryFrequency + flow.DefaultTransactionExpiry) + err = retry.Retry(block.Header.Height + retryFrequency + flow.DefaultTransactionExpiry) + suite.Require().NoError(err) // Should've still only been called once suite.colClient.AssertNumberOfCalls(suite.T(), "SendTransaction", 1) @@ -72,7 +73,6 @@ func (suite *Suite) TestTransactionRetry() { // TestSuccessfulTransactionsDontRetry tests that the retry mechanism will send retries at specific times func (suite *Suite) TestSuccessfulTransactionsDontRetry() { - ctx := context.Background() collection := unittest.CollectionFixture(1) transactionBody := collection.Transactions[0] @@ -118,7 +118,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { backend, err := New(params) suite.Require().NoError(err) - retry := newRetry().SetBackend(backend).Activate() + retry := newRetry(suite.log).SetBackend(backend).Activate() backend.retry = retry retry.RegisterTransaction(block.Header.Height, transactionBody) @@ -144,17 +144,20 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { suite.Assert().Equal(flow.TransactionStatusFinalized, result.Status) // Don't retry now now that block is finalized - retry.Retry(block.Header.Height + 1) + err = retry.Retry(block.Header.Height + 1) + suite.Require().NoError(err) suite.colClient.AssertNotCalled(suite.T(), "SendTransaction", mock.Anything, mock.Anything) // Don't retry now now that block is finalized - retry.Retry(block.Header.Height + retryFrequency) + err = retry.Retry(block.Header.Height + retryFrequency) + suite.Require().NoError(err) suite.colClient.AssertNotCalled(suite.T(), "SendTransaction", mock.Anything, mock.Anything) // Don't retry now now that block is finalized - retry.Retry(block.Header.Height + retryFrequency + flow.DefaultTransactionExpiry) + err = retry.Retry(block.Header.Height + retryFrequency + flow.DefaultTransactionExpiry) + suite.Require().NoError(err) // Should've still should not be called suite.colClient.AssertNotCalled(suite.T(), "SendTransaction", mock.Anything, mock.Anything) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index e96f7f2acfa..a686db1c96e 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -113,7 +113,7 @@ func NewBuilder(log zerolog.Logger, stateStreamBackend: stateStreamBackend, stateStreamConfig: stateStreamConfig, } - backendNotifierActor, backendNotifierWorker := events.NewFinalizationActor(eng.notifyBackendOnBlockFinalized) + backendNotifierActor, backendNotifierWorker := events.NewFinalizationActor(eng.processOnFinalizedBlock) eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). @@ -171,12 +171,11 @@ func (e *Engine) OnFinalizedBlock(block *model.Block) { e.backendNotifierActor.OnFinalizedBlock(block) } -// notifyBackendOnBlockFinalized is invoked by the FinalizationActor when a new block is finalized. -// It notifies the backend of the newly finalized block. -func (e *Engine) notifyBackendOnBlockFinalized(_ *model.Block) error { +// processOnFinalizedBlock is invoked by the FinalizationActor when a new block is finalized. +// It informs the backend of the newly finalized block. +func (e *Engine) processOnFinalizedBlock(_ *model.Block) error { finalizedHeader := e.finalizedHeaderCache.Get() - e.backend.NotifyFinalizedBlockHeight(finalizedHeader.Height) - return nil + return e.backend.ProcessFinalizedBlockHeight(finalizedHeader.Height) } // RestApiAddress returns the listen address of the REST API server. From f00081c0f7ebff6088ca6febe57b229e0cac210b Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 27 Nov 2023 14:27:40 +0200 Subject: [PATCH 20/70] Extended code documentation and added more comments --- engine/access/rpc/backend/retry.go | 2 +- engine/access/rpc/engine.go | 3 +++ module/grpcserver/server_builder.go | 24 +++++++++++++++++++++++- module/irrecoverable/irrecoverable.go | 1 + 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/engine/access/rpc/backend/retry.go b/engine/access/rpc/backend/retry.go index 07f1c3d58c8..4b9bb58387f 100644 --- a/engine/access/rpc/backend/retry.go +++ b/engine/access/rpc/backend/retry.go @@ -23,7 +23,7 @@ type Retry struct { transactionByReferencBlockHeight map[uint64]map[flow.Identifier]*flow.TransactionBody backend *Backend active bool - log zerolog.Logger + log zerolog.Logger // default logger } func newRetry(log zerolog.Logger) *Retry { diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index a686db1c96e..be27440adea 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -211,6 +211,7 @@ func (e *Engine) serveGRPCWebProxyWorker(ctx irrecoverable.SignalerContext, read // serveREST is a worker routine which starts the HTTP REST server. // The ready callback is called after the server address is bound and set. +// Note: The original REST BaseContext is discarded, and the irrecoverable.SignalerContext is used for error handling. func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { if e.config.RestConfig.ListenAddress == "" { e.log.Debug().Msg("no REST API address specified - not starting the server") @@ -228,6 +229,8 @@ func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.Re return } e.restServer = r + + // Set up the irrecoverable.SignalerContext for error handling in the REST server. e.restServer.BaseContext = func(_ net.Listener) context.Context { return context.WithValue(ctx, irrecoverable.SignalerContextKey{}, ctx) } diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index e8bfab070a0..ab03584da92 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -43,7 +43,20 @@ type GrpcServerBuilder struct { stateStreamInterceptorEnable bool } -// NewGrpcServerBuilder helps to build a new grpc server. +// NewGrpcServerBuilder creates a new builder for configuring and initializing a gRPC server. +// +// The builder is configured with the provided parameters such as logger, gRPC server address, maximum message size, +// API rate limits, and additional options. The builder also sets up the necessary interceptors, including handling +// irrecoverable errors using the irrecoverable.SignalerContext. The gRPC server can be configured with options such +// as maximum message sizes and interceptors for handling RPC calls. +// +// If RPC metrics are enabled, the builder adds the gRPC Prometheus interceptor for collecting metrics. Additionally, +// it can enable a state stream interceptor based on the configuration. Rate limiting interceptors can be added based +// on specified API rate limits. Logging and custom interceptors are applied, and the final gRPC server is returned. +// +// If transport credentials are provided, a secure gRPC server is created; otherwise, an unsecured server is initialized. +// +// Note: The gRPC server is created with the specified options and is ready for further configuration or starting. func NewGrpcServerBuilder(log zerolog.Logger, gRPCListenAddr string, maxMsgSize uint, @@ -70,6 +83,15 @@ func NewGrpcServerBuilder(log zerolog.Logger, grpc.MaxSendMsgSize(int(maxMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors + // This interceptor is responsible for ensuring that irrecoverable errors are properly propagated using + // the irrecoverable.SignalerContext. It replaces the original gRPC context with a new one that includes + // the irrecoverable.SignalerContextKey if available, allowing the server to handle error conditions indicating + // an inconsistent or corrupted node state. If no irrecoverable.SignalerContext is present, the original context + // is used to process the gRPC request. + // + // The interceptor follows the grpc.UnaryServerInterceptor signature, where it takes the incoming gRPC context, + // request, unary server information, and handler function. It returns the response and error after handling + // the request. This mechanism ensures consistent error handling for gRPC requests across the server. interceptors = append(interceptors, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { if signalerCtx := signalerCtx.Load(); signalerCtx != nil { valueCtx := context.WithValue(ctx, irrecoverable.SignalerContextKey{}, *signalerCtx) diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index baa3fd708d2..8f8a08c37f4 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -48,6 +48,7 @@ type SignalerContext interface { sealed() // private, to constrain builder to using WithSignaler } +// SignalerContextKey represents the key type for retrieving a SignalerContext from a context.Context. type SignalerContextKey struct{} // private, to force context derivation / WithSignaler From 37db6174be00ea866670de4f0309fb5c66dbd394 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 27 Nov 2023 16:43:03 +0200 Subject: [PATCH 21/70] Added more comment for test --- engine/access/handle_irrecoverable_state_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index e08bde33e71..3ad96257342 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -203,6 +203,7 @@ func TestIrrecoverableState(t *testing.T) { suite.Run(t, new(IrrecoverableStateTestSuite)) } +// TestGRPCInconsistentNodeState tests the behavior when gRPC encounters an inconsistent node state. func (suite *IrrecoverableStateTestSuite) TestGRPCInconsistentNodeState() { err := fmt.Errorf("inconsistent node`s state") suite.snapshot.On("Head").Return(nil, err) @@ -226,6 +227,7 @@ func (suite *IrrecoverableStateTestSuite) TestGRPCInconsistentNodeState() { suite.Require().Error(err) } +// TestRestInconsistentNodeState tests the behavior when the REST API encounters an inconsistent node state. func (suite *IrrecoverableStateTestSuite) TestRestInconsistentNodeState() { collections := unittest.CollectionListFixture(1) blockHeader := unittest.BlockWithGuaranteesFixture( @@ -247,6 +249,7 @@ func (suite *IrrecoverableStateTestSuite) TestRestInconsistentNodeState() { suite.Require().Error(err) } +// optionsForBlocksIdGetOpts returns options for the BlocksApi.BlocksIdGet function. func optionsForBlocksIdGetOpts() *restclient.BlocksApiBlocksIdGetOpts { return &restclient.BlocksApiBlocksIdGetOpts{ Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), From d0b4dfcfd861246c5b0200d345bf4365f12771b1 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 13:07:50 +0200 Subject: [PATCH 22/70] Updated error msg, added additianal checks for tests --- engine/access/rpc/backend/backend_accounts.go | 3 +- .../rpc/backend/backend_accounts_test.go | 7 ++- .../rpc/backend/backend_block_details.go | 10 +++- .../rpc/backend/backend_block_headers.go | 9 +++- engine/access/rpc/backend/backend_events.go | 2 +- engine/access/rpc/backend/backend_scripts.go | 3 +- .../rpc/backend/backend_scripts_test.go | 7 ++- engine/access/rpc/backend/backend_test.go | 51 +++++++++++++------ .../rpc/backend/backend_transactions.go | 4 +- module/grpcserver/server_builder.go | 4 +- module/irrecoverable/irrecoverable.go | 2 +- 11 files changed, 72 insertions(+), 30 deletions(-) diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index 86dfb78766c..3ced5a10dca 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "time" "github.com/rs/zerolog" @@ -44,7 +45,7 @@ func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) { sealed, err := b.state.Sealed().Head() if err != nil { - irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index cf3e90e3965..aaa52a26505 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -333,11 +333,14 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetAccountAtLatestBlock(valueCtx, s.failingAddress) + actual, err := backend.GetAccountAtLatestBlock(valueCtx, s.failingAddress) s.Require().Error(err) + s.Require().Equal(codes.Internal, status.Code(err)) + s.Require().Nil(actual) }) } diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index 364d43a740d..6b70cc1a250 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -2,6 +2,7 @@ package backend import ( "context" + "fmt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -25,9 +26,16 @@ func (b *backendBlockDetails) GetLatestBlock(ctx context.Context, isSealed bool) if isSealed { // get the latest seal header from storage header, err = b.state.Sealed().Head() + if err != nil { + err = fmt.Errorf("failed to lookup sealed header: %w", err) + + } } else { // get the finalized header from state header, err = b.state.Final().Head() + if err != nil { + err = fmt.Errorf("failed to lookup final header: %w", err) + } } if err != nil { @@ -93,7 +101,7 @@ func (b *backendBlockDetails) getBlockStatus(ctx context.Context, block *flow.Bl // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index cf8a04c676d..8c1e5a0d50d 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -2,6 +2,7 @@ package backend import ( "context" + "fmt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -25,9 +26,15 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(ctx context.Context, isSealed if isSealed { // get the latest seal header from storage header, err = b.state.Sealed().Head() + if err != nil { + err = fmt.Errorf("failed to lookup sealed header: %w", err) + } } else { // get the finalized header from state header, err = b.state.Final().Head() + if err != nil { + err = fmt.Errorf("failed to lookup final header: %w", err) + } } if err != nil { @@ -86,7 +93,7 @@ func (b *backendBlockHeaders) getBlockStatus(ctx context.Context, header *flow.H // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 8b4b9dda5bb..829fc1a2ed3 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -55,7 +55,7 @@ func (b *backendEvents) GetEventsForHeightRange( head, err := b.state.Sealed().Head() if err != nil { // sealed block must be in the store, so return an Internal code even if we got NotFound - irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) return nil, status.Errorf(codes.Internal, "failed to get events: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index 78568ebcaac..c49b6403b1f 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" //nolint:gosec "errors" + "fmt" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -73,7 +74,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestHeader, err := b.state.Sealed().Head() if err != nil { // the latest sealed header MUST be available - irrecoverable.Throw(ctx, err) + irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) } diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 5e0617b43fa..2a11805e5e1 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -417,11 +417,14 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis err := fmt.Errorf("inconsistent node`s state") s.snapshot.On("Head").Return(nil, err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.ExecuteScriptAtLatestBlock(valueCtx, s.script, s.arguments) + actual, err := backend.ExecuteScriptAtLatestBlock(valueCtx, s.script, s.arguments) s.Require().Error(err) + s.Require().Equal(codes.Internal, status.Code(err)) + s.Require().Nil(actual) }) } diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 5844584c05e..a5084382c49 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -429,11 +429,15 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, _, err = backend.GetLatestBlockHeader(valueCtx, true) + actualHeader, actualStatus, err := backend.GetLatestBlockHeader(valueCtx, true) suite.Require().Error(err) + suite.Require().Equal(codes.Internal, status.Code(err)) + suite.Require().Nil(actualHeader) + suite.Require().Equal(flow.BlockStatusUnknown, actualStatus) }) } @@ -542,11 +546,14 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetTransactionResultByIndex(valueCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + actual, err := backend.GetTransactionResultByIndex(valueCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) + suite.Require().Equal(codes.Internal, status.Code(err)) + suite.Require().Nil(actual) }) } @@ -609,11 +616,14 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetTransactionResultsByBlockID(valueCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + actual, err := backend.GetTransactionResultsByBlockID(valueCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) + suite.Require().Equal(codes.Internal, status.Code(err)) + suite.Require().Nil(actual) }) } @@ -1033,11 +1043,15 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup final header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, _, err = backend.GetLatestBlock(valueCtx, false) + actualBlock, actualStatus, err := backend.GetLatestBlock(valueCtx, false) suite.Require().Error(err) + suite.Require().Equal(codes.Internal, status.Code(err)) + suite.Require().Nil(actualBlock) + suite.Require().Equal(flow.BlockStatusUnknown, actualStatus) }) } @@ -1341,11 +1355,6 @@ func (suite *Suite) TestGetEventsForHeightRange() { stateParams.On("FinalizedRoot").Return(rootHeader, nil) state.On("Params").Return(stateParams).Maybe() - // mock snapshot to return head backend - snapshot.On("Head").Return( - func() *flow.Header { return head }, - func() error { return nil }, - ) snapshot.On("Identities", mock.Anything).Return( func(_ flow.IdentityFilter) flow.IdentityList { return nodeIdentities @@ -1453,16 +1462,26 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - ctx = irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) err = fmt.Errorf("inconsistent node`s state") - snapshot.On("Head").Return(nil, err) + snapshot.On("Head").Return(nil, err).Once() + + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) + valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) - _, err = backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight, + actual, err := backend.GetEventsForHeightRange(valueCtx, string(flow.EventAccountCreated), minHeight, maxHeight, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) + suite.Require().Equal(codes.Internal, status.Code(err)) + suite.Require().Nil(actual) }) connFactory := suite.setupConnectionFactory() + // mock snapshot to return head backend + snapshot.On("Head").Return( + func() *flow.Header { return head }, + func() error { return nil }, + ) //suite.state = state suite.Run("invalid request max height < min height", func() { diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 69293c2755f..e82f7881b5c 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -599,7 +599,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - return flow.TransactionStatusUnknown, err + return flow.TransactionStatusUnknown, fmt.Errorf("failed to lookup final header: %w", err) } finalizedHeight := finalized.Height @@ -643,7 +643,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - return flow.TransactionStatusUnknown, err + return flow.TransactionStatusUnknown, fmt.Errorf("failed to lookup sealed header: %w", err) } if block.Header.Height > sealed.Height { diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index ab03584da92..76928a8a99e 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -93,8 +93,8 @@ func NewGrpcServerBuilder(log zerolog.Logger, // request, unary server information, and handler function. It returns the response and error after handling // the request. This mechanism ensures consistent error handling for gRPC requests across the server. interceptors = append(interceptors, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { - if signalerCtx := signalerCtx.Load(); signalerCtx != nil { - valueCtx := context.WithValue(ctx, irrecoverable.SignalerContextKey{}, *signalerCtx) + if sigCtx := signalerCtx.Load(); sigCtx != nil { + valueCtx := context.WithValue(ctx, irrecoverable.SignalerContextKey{}, *sigCtx) resp, err = handler(valueCtx, req) } else { resp, err = handler(ctx, req) diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index 8f8a08c37f4..395f8ac896a 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -80,7 +80,7 @@ func Throw(ctx context.Context, err error) { signalerAbleContext.Throw(err) } else { // Be spectacular on how this does not -but should- handle irrecoverables: - log.Fatalf("irrecoverable error signaler not found for context, please implement! Unhandled irrecoverable error %v", err) + log.Fatalf("irrecoverable error signaler not found for context, please implement! Unhandled irrecoverable error: %v", err) } } From c6e460dbef0fcb6a25c2ef88a325dbc93caaf763 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 13:46:43 +0200 Subject: [PATCH 23/70] Updated MockSignalerContext --- module/irrecoverable/unittest.go | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/module/irrecoverable/unittest.go b/module/irrecoverable/unittest.go index 814eaba53a4..8f59cc27960 100644 --- a/module/irrecoverable/unittest.go +++ b/module/irrecoverable/unittest.go @@ -4,14 +4,16 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) -// MockSignalerContext is a SignalerContext which will immediately fail a test if an error is thrown. +// MockSignalerContext is a SignalerContext that can be used in tests to assert that an error is thrown. +// It embeds a mock.Mock, so it can be used it to assert that Throw is called with a specific error. +// Use NewMockSignalerContextExpectError to create a new MockSignalerContext that expects a specific error, otherwise NewMockSignalerContext. type MockSignalerContext struct { context.Context - t *testing.T - expectError error + *mock.Mock } var _ SignalerContext = &MockSignalerContext{} @@ -19,29 +21,32 @@ var _ SignalerContext = &MockSignalerContext{} func (m MockSignalerContext) sealed() {} func (m MockSignalerContext) Throw(err error) { - if m.expectError != nil { - assert.EqualError(m.t, err, m.expectError.Error()) - return - } - m.t.Fatalf("mock signaler context received error: %v", err) + m.Called(err) } func NewMockSignalerContext(t *testing.T, ctx context.Context) *MockSignalerContext { - return &MockSignalerContext{ + m := &MockSignalerContext{ Context: ctx, - t: t, + Mock: &mock.Mock{}, } + m.Mock.Test(t) + t.Cleanup(func() { m.AssertExpectations(t) }) + return m } +// NewMockSignalerContextWithCancel creates a new MockSignalerContext with a cancel function. func NewMockSignalerContextWithCancel(t *testing.T, parent context.Context) (*MockSignalerContext, context.CancelFunc) { ctx, cancel := context.WithCancel(parent) return NewMockSignalerContext(t, ctx), cancel } +// NewMockSignalerContextExpectError creates a new MockSignalerContext which expects a specific error to be thrown. func NewMockSignalerContextExpectError(t *testing.T, ctx context.Context, err error) *MockSignalerContext { - return &MockSignalerContext{ - Context: ctx, - t: t, - expectError: err, - } + require.NotNil(t, err) + m := NewMockSignalerContext(t, ctx) + + // since we expect an error, we should expect a call to Throw + m.On("Throw", err).Once().Return() + + return m } From c5c20ae2c90c415f1065f260ca8999dc6b6b863c Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 13:50:43 +0200 Subject: [PATCH 24/70] Updated test --- engine/access/handle_irrecoverable_state_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 3ad96257342..395795ae814 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -184,7 +184,8 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { assert.NoError(suite.T(), err) err = fmt.Errorf("inconsistent node`s state") - ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), err) + signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) suite.rpcEng.Start(ctx) From 84798e222406e62b5e95321510ded9c2183d4f32 Mon Sep 17 00:00:00 2001 From: Gregor Gololicic Date: Tue, 28 Nov 2023 14:57:16 +0100 Subject: [PATCH 25/70] move setup to preprocess --- fvm/transactionInvoker.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index 03ba76878e5..17041855b72 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -4,6 +4,8 @@ import ( "fmt" "strconv" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/rs/zerolog" @@ -12,7 +14,6 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/evm" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" @@ -181,6 +182,21 @@ func (executor *transactionExecutor) preprocess() error { // infrequently modified and are expensive to compute. For now this includes // reading meter parameter overrides and parsing programs. func (executor *transactionExecutor) preprocessTransactionBody() error { + // setup evm + if executor.ctx.EVMEnabled { + chain := executor.ctx.Chain + err := evm.SetupEnvironment( + chain.ChainID(), + executor.env, + executor.cadenceRuntime.TxRuntimeEnv, + chain.ServiceAddress(), + FlowTokenAddress(chain), + ) + if err != nil { + return err + } + } + meterParams, err := getBodyMeterParameters( executor.ctx, executor.proc, @@ -225,21 +241,6 @@ func (executor *transactionExecutor) execute() error { } func (executor *transactionExecutor) ExecuteTransactionBody() error { - // setup evm - if executor.ctx.EVMEnabled { - chain := executor.ctx.Chain - err := evm.SetupEnvironment( - chain.ChainID(), - executor.env, - executor.cadenceRuntime.TxRuntimeEnv, - chain.ServiceAddress(), - FlowTokenAddress(chain), - ) - if err != nil { - return err - } - } - var invalidator derived.TransactionInvalidator if !executor.errs.CollectedError() { From 4627a169e901cd651ec0768b105d82e106f6b958 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 15:59:10 +0200 Subject: [PATCH 26/70] Fixed typos, updated returning errors, updated tests --- .../access/handle_irrecoverable_state_test.go | 8 ++--- engine/access/rpc/backend/backend_accounts.go | 6 ++-- .../rpc/backend/backend_accounts_test.go | 6 ++-- .../rpc/backend/backend_block_details.go | 13 ++++---- .../rpc/backend/backend_block_headers.go | 13 ++++---- engine/access/rpc/backend/backend_events.go | 5 ++-- engine/access/rpc/backend/backend_scripts.go | 6 ++-- .../rpc/backend/backend_scripts_test.go | 6 ++-- engine/access/rpc/backend/backend_test.go | 30 +++++++++---------- .../rpc/backend/backend_transactions.go | 5 ++-- engine/access/rpc/engine.go | 4 +-- module/grpcserver/server_builder.go | 6 ++-- 12 files changed, 56 insertions(+), 52 deletions(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 395795ae814..be9a876ad5b 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -183,8 +183,8 @@ func (suite *IrrecoverableStateTestSuite) SetupTest() { suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) - err = fmt.Errorf("inconsistent node`s state") - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + err = fmt.Errorf("inconsistent node's state") + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) ctx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) suite.rpcEng.Start(ctx) @@ -206,7 +206,7 @@ func TestIrrecoverableState(t *testing.T) { // TestGRPCInconsistentNodeState tests the behavior when gRPC encounters an inconsistent node state. func (suite *IrrecoverableStateTestSuite) TestGRPCInconsistentNodeState() { - err := fmt.Errorf("inconsistent node`s state") + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err) conn, err := grpc.Dial( @@ -236,7 +236,7 @@ func (suite *IrrecoverableStateTestSuite) TestRestInconsistentNodeState() { ) suite.blocks.On("ByID", blockHeader.ID()).Return(blockHeader, nil) - err := fmt.Errorf("inconsistent node`s state") + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err) config := restclient.NewConfiguration() diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index 3ced5a10dca..f53804dca15 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "fmt" "time" "github.com/rs/zerolog" @@ -45,8 +44,9 @@ func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error) { sealed, err := b.state.Sealed().Head() if err != nil { - irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) - return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return nil, status.Errorf(codes.Internal, err.Error()) } sealedBlockID := sealed.ID() diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index aaa52a26505..398633bbdb6 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -327,13 +327,13 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste backend.scriptExecMode = ScriptExecutionModeLocalOnly backend.scriptExecutor = scriptExecutor - s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", "inconsistent node`s state"), func() { + s.Run(fmt.Sprintf("GetAccountAtLatestBlock - fails with %v", "inconsistent node's state"), func() { s.state.On("Sealed").Return(s.snapshot, nil) - err := fmt.Errorf("inconsistent node`s state") + err := fmt.Errorf("inconsistent node's state") s.snapshot.On("Head").Return(nil, err) - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index 6b70cc1a250..110e695eb7a 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -2,7 +2,6 @@ package backend import ( "context" - "fmt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,14 +26,14 @@ func (b *backendBlockDetails) GetLatestBlock(ctx context.Context, isSealed bool) // get the latest seal header from storage header, err = b.state.Sealed().Head() if err != nil { - err = fmt.Errorf("failed to lookup sealed header: %w", err) + err = irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) } } else { // get the finalized header from state header, err = b.state.Final().Head() if err != nil { - err = fmt.Errorf("failed to lookup final header: %w", err) + err = irrecoverable.NewExceptionf("failed to lookup final header: %w", err) } } @@ -49,7 +48,7 @@ func (b *backendBlockDetails) GetLatestBlock(ctx context.Context, isSealed bool) // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. irrecoverable.Throw(ctx, err) - return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block: %v", err) + return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) } // since we are querying a finalized or sealed block, we can use the height index and save an ID computation @@ -91,6 +90,7 @@ func (b *backendBlockDetails) GetBlockByHeight(ctx context.Context, height uint6 return block, stat, nil } +// No errors are expected during normal operations. func (b *backendBlockDetails) getBlockStatus(ctx context.Context, block *flow.Block) (flow.BlockStatus, error) { sealed, err := b.state.Sealed().Head() if err != nil { @@ -101,8 +101,9 @@ func (b *backendBlockDetails) getBlockStatus(ctx context.Context, block *flow.Bl // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) - return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) } if block.Header.Height > sealed.Height { diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index 8c1e5a0d50d..f66b3916180 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -2,7 +2,6 @@ package backend import ( "context" - "fmt" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,13 +26,13 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(ctx context.Context, isSealed // get the latest seal header from storage header, err = b.state.Sealed().Head() if err != nil { - err = fmt.Errorf("failed to lookup sealed header: %w", err) + err = irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) } } else { // get the finalized header from state header, err = b.state.Final().Head() if err != nil { - err = fmt.Errorf("failed to lookup final header: %w", err) + err = irrecoverable.NewExceptionf("failed to lookup final header: %w", err) } } @@ -47,7 +46,7 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(ctx context.Context, isSealed // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. irrecoverable.Throw(ctx, err) - return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, "could not get latest block header: %v", err) + return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) } stat, err := b.getBlockStatus(ctx, header) @@ -83,6 +82,7 @@ func (b *backendBlockHeaders) GetBlockHeaderByHeight(ctx context.Context, height return header, stat, nil } +// No errors are expected during normal operations. func (b *backendBlockHeaders) getBlockStatus(ctx context.Context, header *flow.Header) (flow.BlockStatus, error) { sealed, err := b.state.Sealed().Head() if err != nil { @@ -93,8 +93,9 @@ func (b *backendBlockHeaders) getBlockStatus(ctx context.Context, header *flow.H // because this can cause DOS potential // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. - irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) - return flow.BlockStatusUnknown, status.Errorf(codes.Internal, "failed to find latest sealed header: %v", err) + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) } if header.Height > sealed.Height { diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 829fc1a2ed3..5700113ed0b 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -55,8 +55,9 @@ func (b *backendEvents) GetEventsForHeightRange( head, err := b.state.Sealed().Head() if err != nil { // sealed block must be in the store, so return an Internal code even if we got NotFound - irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) - return nil, status.Errorf(codes.Internal, "failed to get events: %v", err) + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return nil, status.Errorf(codes.Internal, err.Error()) } // start height should not be beyond the last sealed height diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index c49b6403b1f..d0768e994d1 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -4,7 +4,6 @@ import ( "context" "crypto/md5" //nolint:gosec "errors" - "fmt" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -74,8 +73,9 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestHeader, err := b.state.Sealed().Head() if err != nil { // the latest sealed header MUST be available - irrecoverable.Throw(ctx, fmt.Errorf("failed to lookup sealed header: %w", err)) - return nil, status.Errorf(codes.Internal, "failed to get latest sealed header: %v", err) + err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) + irrecoverable.Throw(ctx, err) + return nil, status.Errorf(codes.Internal, err.Error()) } return b.executeScript(ctx, newScriptExecutionRequest(latestHeader.ID(), latestHeader.Height, script, arguments)) diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 2a11805e5e1..8754310d60e 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -411,13 +411,13 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis backend.scriptExecMode = ScriptExecutionModeLocalOnly backend.scriptExecutor = scriptExecutor - s.Run(fmt.Sprintf("ExecuteScriptAtLatestBlock - fails with %v", "inconsistent node`s state"), func() { + s.Run(fmt.Sprintf("ExecuteScriptAtLatestBlock - fails with %v", "inconsistent node's state"), func() { s.state.On("Sealed").Return(s.snapshot, nil) - err := fmt.Errorf("inconsistent node`s state") + err := fmt.Errorf("inconsistent node's state") s.snapshot.On("Head").Return(nil, err) - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index a5084382c49..3691b05065e 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -424,12 +424,12 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { }) // tests that signaler context received error when node state is inconsistent - suite.Run("GetLatestSealedBlockHeader - fails with inconsistent node`s state", func() { - err := fmt.Errorf("inconsistent node`s state") + suite.Run("GetLatestSealedBlockHeader - fails with inconsistent node's state", func() { + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) @@ -541,12 +541,12 @@ func (suite *Suite) TestGetTransactionResultByIndex() { }) // tests that signaler context received error when node state is inconsistent - suite.Run("TestGetTransactionResultByIndex - fails with inconsistent node`s state", func() { - err := fmt.Errorf("inconsistent node`s state") + suite.Run("TestGetTransactionResultByIndex - fails with inconsistent node's state", func() { + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) @@ -611,12 +611,12 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { }) //tests that signaler context received error when node state is inconsistent - suite.Run("GetTransactionResultsByBlockID - fails with inconsistent node`s state", func() { - err := fmt.Errorf("inconsistent node`s state") + suite.Run("GetTransactionResultsByBlockID - fails with inconsistent node's state", func() { + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err).Once() // mock signaler context expect an error - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) @@ -1038,12 +1038,12 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { }) // tests that signaler context received error when node state is inconsistent - suite.Run("GetLatestFinalizedBlock - fails with inconsistent node`s state", func() { - err := fmt.Errorf("inconsistent node`s state") + suite.Run("GetLatestFinalizedBlock - fails with inconsistent node's state", func() { + err := fmt.Errorf("inconsistent node's state") suite.snapshot.On("Head").Return(nil, err) // mock signaler context expect an error - signCtxErr := fmt.Errorf("failed to lookup final header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup final header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) @@ -1446,7 +1446,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { } // tests that signaler context received error when node state is inconsistent - suite.Run("inconsistent node`s state", func() { + suite.Run("inconsistent node's state", func() { headHeight = maxHeight - 1 setupHeadHeight(headHeight) @@ -1462,10 +1462,10 @@ func (suite *Suite) TestGetEventsForHeightRange() { backend, err := New(params) suite.Require().NoError(err) - err = fmt.Errorf("inconsistent node`s state") + err = fmt.Errorf("inconsistent node's state") snapshot.On("Head").Return(nil, err).Once() - signCtxErr := fmt.Errorf("failed to lookup sealed header: %w", err) + signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index e82f7881b5c..40cc4f04723 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -580,7 +580,6 @@ func (b *backendTransactions) GetTransactionResultByIndex( } // deriveTransactionStatus derives the transaction status based on current protocol state - // Error returns: // - ErrUnknownSnapshotReference - block referenced by transaction has not been found. // - all other errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). @@ -599,7 +598,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest finalized block from the state finalized, err := b.state.Final().Head() if err != nil { - return flow.TransactionStatusUnknown, fmt.Errorf("failed to lookup final header: %w", err) + return flow.TransactionStatusUnknown, irrecoverable.NewExceptionf("failed to lookup final header: %w", err) } finalizedHeight := finalized.Height @@ -643,7 +642,7 @@ func (b *backendTransactions) deriveTransactionStatus( // get the latest sealed block from the State sealed, err := b.state.Sealed().Head() if err != nil { - return flow.TransactionStatusUnknown, fmt.Errorf("failed to lookup sealed header: %w", err) + return flow.TransactionStatusUnknown, irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) } if block.Header.Height > sealed.Height { diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index be27440adea..91ea6276514 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -171,8 +171,8 @@ func (e *Engine) OnFinalizedBlock(block *model.Block) { e.backendNotifierActor.OnFinalizedBlock(block) } -// processOnFinalizedBlock is invoked by the FinalizationActor when a new block is finalized. -// It informs the backend of the newly finalized block. +// The input to this callback is treated as trusted. +// No errors expected during normal operations. func (e *Engine) processOnFinalizedBlock(_ *model.Block) error { finalizedHeader := e.finalizedHeaderCache.Get() return e.backend.ProcessFinalizedBlockHeight(finalizedHeader.Height) diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index 76928a8a99e..95ce67985ae 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -5,9 +5,7 @@ import ( grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/rs/zerolog" - "go.uber.org/atomic" - "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -75,6 +73,10 @@ func NewGrpcServerBuilder(log zerolog.Logger, applyOption(grpcServerBuilder) } + // we use an atomic pointer to setup an interceptor for handling irrecoverable errors, the necessity of this approach + // is dictated by complex startup order of grpc server and other services. At the point where we need to register + // an interceptor we don't have an `irrecoverable.SignalerContext`, it becomes available only when we start + // the server but at that point we can't register interceptors anymore, so we inject it using this approach. signalerCtx := atomic.NewPointer[irrecoverable.SignalerContext](nil) // create a GRPC server to serve GRPC clients From 881523f4d317406658eb1c395f77efc828ac4b23 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 16:58:10 +0200 Subject: [PATCH 27/70] Added utility function according to comment --- .../rpc/backend/backend_accounts_test.go | 5 ++-- .../rpc/backend/backend_scripts_test.go | 6 ++-- engine/access/rpc/backend/backend_test.go | 30 +++++++++---------- .../rpc/backend/backend_transactions.go | 1 + engine/access/rpc/engine.go | 4 ++- module/grpcserver/server_builder.go | 3 +- module/irrecoverable/irrecoverable.go | 5 ++++ 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index 398633bbdb6..177632fb3b7 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -334,10 +334,9 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste s.snapshot.On("Head").Return(nil, err) signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr)) - actual, err := backend.GetAccountAtLatestBlock(valueCtx, s.failingAddress) + actual, err := backend.GetAccountAtLatestBlock(signalerCtx, s.failingAddress) s.Require().Error(err) s.Require().Equal(codes.Internal, status.Code(err)) s.Require().Nil(actual) diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 8754310d60e..9dc34575bed 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -418,10 +418,10 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis s.snapshot.On("Head").Return(nil, err) signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(s.T(), context.Background(), signCtxErr)) - actual, err := backend.ExecuteScriptAtLatestBlock(valueCtx, s.script, s.arguments) + actual, err := backend.ExecuteScriptAtLatestBlock(signalerCtx, s.script, s.arguments) s.Require().Error(err) s.Require().Equal(codes.Internal, status.Code(err)) s.Require().Nil(actual) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 3691b05065e..7eece8bdbb7 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -430,10 +430,10 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { // mock signaler context expect an error signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr)) - actualHeader, actualStatus, err := backend.GetLatestBlockHeader(valueCtx, true) + actualHeader, actualStatus, err := backend.GetLatestBlockHeader(signalerCtx, true) suite.Require().Error(err) suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actualHeader) @@ -547,10 +547,10 @@ func (suite *Suite) TestGetTransactionResultByIndex() { // mock signaler context expect an error signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr)) - actual, err := backend.GetTransactionResultByIndex(valueCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + actual, err := backend.GetTransactionResultByIndex(signalerCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actual) @@ -617,10 +617,10 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { // mock signaler context expect an error signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr)) - actual, err := backend.GetTransactionResultsByBlockID(valueCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) + actual, err := backend.GetTransactionResultsByBlockID(signalerCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actual) @@ -1044,10 +1044,10 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { // mock signaler context expect an error signCtxErr := irrecoverable.NewExceptionf("failed to lookup final header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr)) - actualBlock, actualStatus, err := backend.GetLatestBlock(valueCtx, false) + actualBlock, actualStatus, err := backend.GetLatestBlock(signalerCtx, false) suite.Require().Error(err) suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actualBlock) @@ -1466,10 +1466,10 @@ func (suite *Suite) TestGetEventsForHeightRange() { snapshot.On("Head").Return(nil, err).Once() signCtxErr := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) - signalerCtx := irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr) - valueCtx := context.WithValue(context.Background(), irrecoverable.SignalerContextKey{}, *signalerCtx) + signalerCtx := irrecoverable.WithSignalerContext(context.Background(), + irrecoverable.NewMockSignalerContextExpectError(suite.T(), context.Background(), signCtxErr)) - actual, err := backend.GetEventsForHeightRange(valueCtx, string(flow.EventAccountCreated), minHeight, maxHeight, + actual, err := backend.GetEventsForHeightRange(signalerCtx, string(flow.EventAccountCreated), minHeight, maxHeight, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) suite.Require().Equal(codes.Internal, status.Code(err)) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 40cc4f04723..7f3771bc266 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -801,6 +801,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return events, resp.GetStatusCode(), resp.GetErrorMessage(), nil } +// No errors expected during normal operations. func (b *backendTransactions) ProcessFinalizedBlockHeight(height uint64) error { return b.retry.Retry(height) } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 91ea6276514..4137a1ad976 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -171,6 +171,8 @@ func (e *Engine) OnFinalizedBlock(block *model.Block) { e.backendNotifierActor.OnFinalizedBlock(block) } +// processOnFinalizedBlock is invoked by the FinalizationActor when a new block is finalized. +// It informs the backend of the newly finalized block. // The input to this callback is treated as trusted. // No errors expected during normal operations. func (e *Engine) processOnFinalizedBlock(_ *model.Block) error { @@ -232,7 +234,7 @@ func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.Re // Set up the irrecoverable.SignalerContext for error handling in the REST server. e.restServer.BaseContext = func(_ net.Listener) context.Context { - return context.WithValue(ctx, irrecoverable.SignalerContextKey{}, ctx) + return irrecoverable.WithSignalerContext(ctx, ctx) } l, err := net.Listen("tcp", e.config.RestConfig.ListenAddress) diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index 95ce67985ae..a42cdc0e269 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -96,8 +96,7 @@ func NewGrpcServerBuilder(log zerolog.Logger, // the request. This mechanism ensures consistent error handling for gRPC requests across the server. interceptors = append(interceptors, func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { if sigCtx := signalerCtx.Load(); sigCtx != nil { - valueCtx := context.WithValue(ctx, irrecoverable.SignalerContextKey{}, *sigCtx) - resp, err = handler(valueCtx, req) + resp, err = handler(irrecoverable.WithSignalerContext(ctx, *sigCtx), req) } else { resp, err = handler(ctx, req) } diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index 395f8ac896a..c1dd6030594 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -65,6 +65,11 @@ func WithSignaler(parent context.Context) (SignalerContext, <-chan error) { return &signalerCtx{parent, sig}, errChan } +// WithSignalerContext converts a SignalerContext to a context.Context with the same underlying context +func WithSignalerContext(parent context.Context, ctx SignalerContext) context.Context { + return context.WithValue(parent, SignalerContextKey{}, ctx) +} + // Throw enables throwing an irrecoverable error using any context.Context. // // If we have an SignalerContext, we can directly ctx.Throw. From c7ffbbb2da3450367dc58057bdaf149a01f89446 Mon Sep 17 00:00:00 2001 From: Gregor Gololicic Date: Tue, 28 Nov 2023 16:36:14 +0100 Subject: [PATCH 28/70] add test for evm transaction --- fvm/fvm_test.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index fcf44bcff2b..a91621a77a9 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" + "github.com/onflow/flow-go/fvm/evm/stdlib" + "github.com/onflow/cadence" "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" @@ -2910,3 +2912,64 @@ func TestEntropyCallExpectsNoParameters(t *testing.T) { }, )(t) } + +func TestEVM(t *testing.T) { + + t.Run("successful transaction", newVMTest(). + withBootstrapProcedureOptions(fvm.WithSetupEVMEnabled(true)). + // we keep this dissabled during bootstrap and later overwrite in the test for test transaction + withContextOptions( + fvm.WithEVMEnabled(false), + fvm.WithCadenceLogging(true), + ). + run(func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + // generate test address + genArr := make([]cadence.Value, 20) + for i := range genArr { + genArr[i] = cadence.UInt8(i) + } + addrBytes := cadence.NewArray(genArr).WithType(stdlib.EVMAddressBytesCadenceType) + encodedArg, err := jsoncdc.Encode(addrBytes) + require.NoError(t, err) + + txBody := flow.NewTransactionBody(). + SetScript([]byte(fmt.Sprintf(` + import EVM from %s + + transaction(bytes: [UInt8; 20]) { + execute { + let addr = EVM.EVMAddress(bytes: bytes) + log(addr) + } + } + `, chain.ServiceAddress().HexWithPrefix()))). + SetProposalKey(chain.ServiceAddress(), 0, 0). + SetPayer(chain.ServiceAddress()). + AddArgument(encodedArg) + + err = testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + + ctx = fvm.NewContextFromParent(ctx, fvm.WithEVMEnabled(true)) + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + + require.NoError(t, err) + require.NoError(t, output.Err) + require.Len(t, output.Logs, 1) + require.Equal(t, output.Logs[0], fmt.Sprintf( + "A.%s.EVM.EVMAddress(bytes: %s)", + chain.ServiceAddress(), + addrBytes.String(), + )) + }), + ) +} From cd8bf2e5d665ed5d6a56f4274267a10383286205 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 17:40:28 +0200 Subject: [PATCH 29/70] Added godoc --- engine/access/rpc/backend/backend_transactions.go | 4 ++++ engine/access/rpc/backend/retry.go | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 7f3771bc266..0423eea3515 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -801,6 +801,10 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return events, resp.GetStatusCode(), resp.GetErrorMessage(), nil } +// ATTENTION: might be a source of problems in future. We run this code on finalization gorotuine, +// potentially lagging finalization events if operations take long time. +// We might need to move this logic on dedicated goroutine and provide a way to skip finalization events if they are delivered +// too often for this engine. An example of similar approach - https://github.com/onflow/flow-go/blob/master/engine/common/follower/compliance_engine.go#L201. // No errors expected during normal operations. func (b *backendTransactions) ProcessFinalizedBlockHeight(height uint64) error { return b.retry.Retry(height) diff --git a/engine/access/rpc/backend/retry.go b/engine/access/rpc/backend/retry.go index 4b9bb58387f..bd6e6744ae9 100644 --- a/engine/access/rpc/backend/retry.go +++ b/engine/access/rpc/backend/retry.go @@ -47,6 +47,12 @@ func (r *Retry) SetBackend(b *Backend) *Retry { return r } +// Retry attempts to resend transactions for a specified block height. +// It performs cleanup operations, including pruning old transactions, and retries sending +// transactions that are still pending. +// The method takes a block height as input. If the provided height is lower than +// flow.DefaultTransactionExpiry, no retries are performed, and the method returns nil. +// No errors expected during normal operations. func (r *Retry) Retry(height uint64) error { // No need to retry if height is lower than DefaultTransactionExpiry if height < flow.DefaultTransactionExpiry { @@ -94,6 +100,12 @@ func (r *Retry) prune(height uint64) { } } +// retryTxsAtHeight retries transactions at a specific block height. +// It looks up transactions at the specified height and retries sending +// raw transactions for those that are still pending. It also cleans up +// transactions that are no longer pending or have an unknown status. +// Error returns: +// - errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). func (r *Retry) retryTxsAtHeight(heightToRetry uint64) error { r.mu.Lock() defer r.mu.Unlock() From 5c206e45fc09f5e3dfbcef553441537b1db0ef0a Mon Sep 17 00:00:00 2001 From: Gregor Gololicic Date: Tue, 28 Nov 2023 16:46:15 +0100 Subject: [PATCH 30/70] fix import change --- fvm/transactionInvoker.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index 17041855b72..e381f7f464a 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -4,8 +4,6 @@ import ( "fmt" "strconv" - "github.com/onflow/flow-go/fvm/evm" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/rs/zerolog" @@ -14,6 +12,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/fvm/evm" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" From 44484c8f63373c39a1cbe8f60e60de806fb24cc0 Mon Sep 17 00:00:00 2001 From: Uliana Andrukhiv Date: Tue, 28 Nov 2023 18:36:09 +0200 Subject: [PATCH 31/70] Update module/irrecoverable/irrecoverable.go Co-authored-by: Yurii Oleksyshyn --- module/irrecoverable/irrecoverable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index c1dd6030594..9aabff5f1cf 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -65,7 +65,7 @@ func WithSignaler(parent context.Context) (SignalerContext, <-chan error) { return &signalerCtx{parent, sig}, errChan } -// WithSignalerContext converts a SignalerContext to a context.Context with the same underlying context +// WithSignalerContext wraps `SignalerContext` using `context.WithValue` so it can later be used with `Throw`. func WithSignalerContext(parent context.Context, ctx SignalerContext) context.Context { return context.WithValue(parent, SignalerContextKey{}, ctx) } From e78bf18a4f1d3f64488dcc3e1f6566dc9b98fbdd Mon Sep 17 00:00:00 2001 From: Uliana Andrukhiv Date: Tue, 28 Nov 2023 18:41:21 +0200 Subject: [PATCH 32/70] Updated godoc for SignalerContextKey Co-authored-by: Yurii Oleksyshyn --- module/irrecoverable/irrecoverable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index 9aabff5f1cf..0877732bcd8 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -48,7 +48,7 @@ type SignalerContext interface { sealed() // private, to constrain builder to using WithSignaler } -// SignalerContextKey represents the key type for retrieving a SignalerContext from a context.Context. +// SignalerContextKey represents the key type for retrieving a SignalerContext from a value `context.Context`. type SignalerContextKey struct{} // private, to force context derivation / WithSignaler From bedc452266a0644a7ba3bcf2a6601a38b344ba53 Mon Sep 17 00:00:00 2001 From: Uliana Andrukhiv Date: Tue, 28 Nov 2023 18:51:54 +0200 Subject: [PATCH 33/70] Updated godoc for lookupBlock Co-authored-by: Yurii Oleksyshyn --- engine/access/rpc/backend/backend_transactions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 0423eea3515..06b1975559a 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -664,7 +664,7 @@ func (b *backendTransactions) isExpired(refHeight, compareToHeight uint64) bool } // Error returns: -// - ErrNotFound - collection referenced by transaction or block by a collection has not been found. +// - `storage.ErrNotFound` - collection referenced by transaction or block by a collection has not been found. // - all other errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). func (b *backendTransactions) lookupBlock(txID flow.Identifier) (*flow.Block, error) { From 7f3c440c1307a85e1ff1b31843d6d69b90c58163 Mon Sep 17 00:00:00 2001 From: Uliana Andrukhiv Date: Tue, 28 Nov 2023 18:53:37 +0200 Subject: [PATCH 34/70] Updated godoc for ProcessFinalizedBlockHeight Co-authored-by: Yurii Oleksyshyn --- engine/access/rpc/backend/backend_transactions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 06b1975559a..3a4d0913368 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -804,7 +804,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( // ATTENTION: might be a source of problems in future. We run this code on finalization gorotuine, // potentially lagging finalization events if operations take long time. // We might need to move this logic on dedicated goroutine and provide a way to skip finalization events if they are delivered -// too often for this engine. An example of similar approach - https://github.com/onflow/flow-go/blob/master/engine/common/follower/compliance_engine.go#L201. +// too often for this engine. An example of similar approach - https://github.com/onflow/flow-go/blob/10b0fcbf7e2031674c00f3cdd280f27bd1b16c47/engine/common/follower/compliance_engine.go#L201.. // No errors expected during normal operations. func (b *backendTransactions) ProcessFinalizedBlockHeight(height uint64) error { return b.retry.Retry(height) From 28fd7ccaa5a5207f272d69a790bb6e3a79af7219 Mon Sep 17 00:00:00 2001 From: Uliana Andrukhiv Date: Tue, 28 Nov 2023 18:54:27 +0200 Subject: [PATCH 35/70] Updated godoc for deriveTransactionStatus Co-authored-by: Yurii Oleksyshyn --- engine/access/rpc/backend/backend_transactions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 3a4d0913368..c928e167e88 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -581,7 +581,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( // deriveTransactionStatus derives the transaction status based on current protocol state // Error returns: -// - ErrUnknownSnapshotReference - block referenced by transaction has not been found. +// - state.ErrUnknownSnapshotReference - block referenced by transaction has not been found. // - all other errors are unexpected and potentially symptoms of internal implementation bugs or state corruption (fatal). func (b *backendTransactions) deriveTransactionStatus( tx *flow.TransactionBody, From 6e3d15d244ffbb9012d7bdb61d3e6c52cf12ca6c Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 19:09:12 +0200 Subject: [PATCH 36/70] Updated returning errors, updated tests --- engine/access/handle_irrecoverable_state_test.go | 6 ++++-- engine/access/rpc/backend/backend_accounts.go | 2 +- engine/access/rpc/backend/backend_accounts_test.go | 1 - engine/access/rpc/backend/backend_block_details.go | 4 ++-- engine/access/rpc/backend/backend_block_headers.go | 7 ++----- engine/access/rpc/backend/backend_events.go | 2 +- engine/access/rpc/backend/backend_scripts.go | 2 +- engine/access/rpc/backend/backend_scripts_test.go | 1 - engine/access/rpc/backend/backend_test.go | 5 ----- 9 files changed, 11 insertions(+), 19 deletions(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index be9a876ad5b..8445dd669c1 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -224,8 +224,9 @@ func (suite *IrrecoverableStateTestSuite) TestGRPCInconsistentNodeState() { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - _, err = client.GetAccountAtLatestBlock(ctx, req) + actual, err := client.GetAccountAtLatestBlock(ctx, req) suite.Require().Error(err) + suite.Require().Nil(actual) } // TestRestInconsistentNodeState tests the behavior when the REST API encounters an inconsistent node state. @@ -246,8 +247,9 @@ func (suite *IrrecoverableStateTestSuite) TestRestInconsistentNodeState() { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - _, _, err = client.BlocksApi.BlocksIdGet(ctx, []string{blockHeader.ID().String()}, optionsForBlocksIdGetOpts()) + actual, _, err := client.BlocksApi.BlocksIdGet(ctx, []string{blockHeader.ID().String()}, optionsForBlocksIdGetOpts()) suite.Require().Error(err) + suite.Require().Nil(actual) } // optionsForBlocksIdGetOpts returns options for the BlocksApi.BlocksIdGet function. diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index f53804dca15..dbb2c03ab64 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -46,7 +46,7 @@ func (b *backendAccounts) GetAccountAtLatestBlock(ctx context.Context, address f if err != nil { err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) irrecoverable.Throw(ctx, err) - return nil, status.Errorf(codes.Internal, err.Error()) + return nil, err } sealedBlockID := sealed.ID() diff --git a/engine/access/rpc/backend/backend_accounts_test.go b/engine/access/rpc/backend/backend_accounts_test.go index 177632fb3b7..614f91bfb49 100644 --- a/engine/access/rpc/backend/backend_accounts_test.go +++ b/engine/access/rpc/backend/backend_accounts_test.go @@ -338,7 +338,6 @@ func (s *BackendAccountsSuite) TestGetAccountAtLatestBlockFromStorage_Inconsiste actual, err := backend.GetAccountAtLatestBlock(signalerCtx, s.failingAddress) s.Require().Error(err) - s.Require().Equal(codes.Internal, status.Code(err)) s.Require().Nil(actual) }) } diff --git a/engine/access/rpc/backend/backend_block_details.go b/engine/access/rpc/backend/backend_block_details.go index 110e695eb7a..698e66b0727 100644 --- a/engine/access/rpc/backend/backend_block_details.go +++ b/engine/access/rpc/backend/backend_block_details.go @@ -48,7 +48,7 @@ func (b *backendBlockDetails) GetLatestBlock(ctx context.Context, isSealed bool) // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. irrecoverable.Throw(ctx, err) - return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) + return nil, flow.BlockStatusUnknown, err } // since we are querying a finalized or sealed block, we can use the height index and save an ID computation @@ -103,7 +103,7 @@ func (b *backendBlockDetails) getBlockStatus(ctx context.Context, block *flow.Bl // observe the protocol state error and throw an exception. err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) irrecoverable.Throw(ctx, err) - return flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) + return flow.BlockStatusUnknown, err } if block.Header.Height > sealed.Height { diff --git a/engine/access/rpc/backend/backend_block_headers.go b/engine/access/rpc/backend/backend_block_headers.go index f66b3916180..a61fcab711a 100644 --- a/engine/access/rpc/backend/backend_block_headers.go +++ b/engine/access/rpc/backend/backend_block_headers.go @@ -3,9 +3,6 @@ package backend import ( "context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" @@ -46,7 +43,7 @@ func (b *backendBlockHeaders) GetLatestBlockHeader(ctx context.Context, isSealed // - Since the protocol state is widely shared, we assume that in practice another component will // observe the protocol state error and throw an exception. irrecoverable.Throw(ctx, err) - return nil, flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) + return nil, flow.BlockStatusUnknown, err } stat, err := b.getBlockStatus(ctx, header) @@ -95,7 +92,7 @@ func (b *backendBlockHeaders) getBlockStatus(ctx context.Context, header *flow.H // observe the protocol state error and throw an exception. err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) irrecoverable.Throw(ctx, err) - return flow.BlockStatusUnknown, status.Errorf(codes.Internal, err.Error()) + return flow.BlockStatusUnknown, err } if header.Height > sealed.Height { diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index 5700113ed0b..a5bebada2e7 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -57,7 +57,7 @@ func (b *backendEvents) GetEventsForHeightRange( // sealed block must be in the store, so return an Internal code even if we got NotFound err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) irrecoverable.Throw(ctx, err) - return nil, status.Errorf(codes.Internal, err.Error()) + return nil, err } // start height should not be beyond the last sealed height diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index d0768e994d1..54e97817d89 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -75,7 +75,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( // the latest sealed header MUST be available err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err) irrecoverable.Throw(ctx, err) - return nil, status.Errorf(codes.Internal, err.Error()) + return nil, err } return b.executeScript(ctx, newScriptExecutionRequest(latestHeader.ID(), latestHeader.Height, script, arguments)) diff --git a/engine/access/rpc/backend/backend_scripts_test.go b/engine/access/rpc/backend/backend_scripts_test.go index 9dc34575bed..fbcc4105f9c 100644 --- a/engine/access/rpc/backend/backend_scripts_test.go +++ b/engine/access/rpc/backend/backend_scripts_test.go @@ -423,7 +423,6 @@ func (s *BackendScriptsSuite) TestExecuteScriptAtLatestBlockFromStorage_Inconsis actual, err := backend.ExecuteScriptAtLatestBlock(signalerCtx, s.script, s.arguments) s.Require().Error(err) - s.Require().Equal(codes.Internal, status.Code(err)) s.Require().Nil(actual) }) } diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 7eece8bdbb7..81a9c5fc593 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -435,7 +435,6 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { actualHeader, actualStatus, err := backend.GetLatestBlockHeader(signalerCtx, true) suite.Require().Error(err) - suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actualHeader) suite.Require().Equal(flow.BlockStatusUnknown, actualStatus) }) @@ -552,7 +551,6 @@ func (suite *Suite) TestGetTransactionResultByIndex() { actual, err := backend.GetTransactionResultByIndex(signalerCtx, blockId, index, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) - suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actual) }) } @@ -622,7 +620,6 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { actual, err := backend.GetTransactionResultsByBlockID(signalerCtx, blockId, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) - suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actual) }) } @@ -1049,7 +1046,6 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { actualBlock, actualStatus, err := backend.GetLatestBlock(signalerCtx, false) suite.Require().Error(err) - suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actualBlock) suite.Require().Equal(flow.BlockStatusUnknown, actualStatus) }) @@ -1472,7 +1468,6 @@ func (suite *Suite) TestGetEventsForHeightRange() { actual, err := backend.GetEventsForHeightRange(signalerCtx, string(flow.EventAccountCreated), minHeight, maxHeight, entitiesproto.EventEncodingVersion_JSON_CDC_V0) suite.Require().Error(err) - suite.Require().Equal(codes.Internal, status.Code(err)) suite.Require().Nil(actual) }) From 84bcf4ed455662fc8074f87e426593a0588185a9 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 19:21:55 +0200 Subject: [PATCH 37/70] Updated creating nocked interfaces --- .../access/handle_irrecoverable_state_test.go | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 8445dd669c1..29f10cb34b1 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -3,6 +3,7 @@ package access import ( "context" "fmt" + "github.com/onflow/flow-go/network/mocknetwork" "io" "os" "testing" @@ -12,7 +13,6 @@ import ( accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -32,7 +32,6 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/network" protocol "github.com/onflow/flow-go/state/protocol/mock" storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/grpcutils" @@ -46,7 +45,7 @@ type IrrecoverableStateTestSuite struct { snapshot *protocol.Snapshot epochQuery *protocol.EpochQuery log zerolog.Logger - net *network.EngineRegistry + net *mocknetwork.EngineRegistry request *module.Requester collClient *accessmock.AccessAPIClient execClient *accessmock.ExecutionAPIClient @@ -72,40 +71,38 @@ type IrrecoverableStateTestSuite struct { func (suite *IrrecoverableStateTestSuite) SetupTest() { suite.log = zerolog.New(os.Stdout) - suite.net = new(network.EngineRegistry) - suite.state = new(protocol.State) - suite.snapshot = new(protocol.Snapshot) + suite.net = mocknetwork.NewEngineRegistry(suite.T()) + suite.state = protocol.NewState(suite.T()) + suite.snapshot = protocol.NewSnapshot(suite.T()) rootHeader := unittest.BlockHeaderFixture() - params := new(protocol.Params) + params := protocol.NewParams(suite.T()) params.On("SporkID").Return(unittest.IdentifierFixture(), nil) params.On("ProtocolVersion").Return(uint(unittest.Uint64InRange(10, 30)), nil) params.On("SporkRootBlockHeight").Return(rootHeader.Height, nil) params.On("SealedRoot").Return(rootHeader, nil) - suite.epochQuery = new(protocol.EpochQuery) + suite.epochQuery = protocol.NewEpochQuery(suite.T()) suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.state.On("Params").Return(params, nil).Maybe() suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() - suite.blocks = new(storagemock.Blocks) - suite.headers = new(storagemock.Headers) - suite.transactions = new(storagemock.Transactions) - suite.collections = new(storagemock.Collections) - suite.receipts = new(storagemock.ExecutionReceipts) + suite.blocks = storagemock.NewBlocks(suite.T()) + suite.headers = storagemock.NewHeaders(suite.T()) + suite.transactions = storagemock.NewTransactions(suite.T()) + suite.collections = storagemock.NewCollections(suite.T()) + suite.receipts = storagemock.NewExecutionReceipts(suite.T()) - suite.collClient = new(accessmock.AccessAPIClient) - suite.execClient = new(accessmock.ExecutionAPIClient) + suite.collClient = accessmock.NewAccessAPIClient(suite.T()) + suite.execClient = accessmock.NewExecutionAPIClient(suite.T()) - suite.request = new(module.Requester) - suite.request.On("EntityByID", mock.Anything, mock.Anything) - - suite.me = new(module.Local) + suite.request = module.NewRequester(suite.T()) + suite.me = module.NewLocal(suite.T()) accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) suite.me. On("NodeID"). - Return(accessIdentity.NodeID) + Return(accessIdentity.NodeID).Maybe() suite.chainID = flow.Testnet suite.metrics = metrics.NewNoopCollector() From eda6f7712f57d0cd98ebda73b7bd8576f870db0c Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 20:24:32 +0200 Subject: [PATCH 38/70] Linted --- engine/access/handle_irrecoverable_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/access/handle_irrecoverable_state_test.go b/engine/access/handle_irrecoverable_state_test.go index 29f10cb34b1..d57750dc86a 100644 --- a/engine/access/handle_irrecoverable_state_test.go +++ b/engine/access/handle_irrecoverable_state_test.go @@ -3,7 +3,6 @@ package access import ( "context" "fmt" - "github.com/onflow/flow-go/network/mocknetwork" "io" "os" "testing" @@ -32,6 +31,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/mocknetwork" protocol "github.com/onflow/flow-go/state/protocol/mock" storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/grpcutils" From 04571313b506f46961fc46a578d88f8350bf06e1 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 13:47:28 -0500 Subject: [PATCH 39/70] Update network/p2p/scoring/registry.go Co-authored-by: Misha <15269764+gomisha@users.noreply.github.com> --- network/p2p/scoring/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index a7df3815cca..4aafcfb335b 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -48,7 +48,7 @@ const ( iHaveMisbehaviourPenalty = -10 // iWantMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a iWant misbehaviour. iWantMisbehaviourPenalty = -10 - // clusterPrefixedPenaltyReductionFactor factor used to reduce the penalty for control message misbehaviours on cluster prefixed topics. This is allows a more lenient punishment for nodes + // clusterPrefixedPenaltyReductionFactor factor used to reduce the penalty for control message misbehaviours on cluster prefixed topics. This allows a more lenient punishment for nodes // that fall behind and may need to request old data. clusterPrefixedPenaltyReductionFactor = .5 // rpcPublishMessageMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a RpcPublishMessageMisbehaviourPenalty misbehaviour. From da60233de45bb9cfbbc76d5f39babefe7df0f8ff Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 13:50:39 -0500 Subject: [PATCH 40/70] Update network/p2p/scoring/registry_test.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/scoring/registry_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 771d2d4fefb..51ba13057f3 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -477,7 +477,12 @@ func TestPeerSpamPenaltyClusterPrefixed(t *testing.T) { // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, // and the peer should be deprived of the default reward for its valid staked role. score = reg.AppSpecificScoreFunc()(peerID) - assert.Less(t, math.Abs(expectedPenalty-score), 10e-3) + tolerance := 10e-3 // 0.1% + if expectedPenalty == 0 { + assert.Less(t, math.Abs(expectedPenalty), tolerance) + } else { + assert.Less(t, math.Abs(expectedPenalty-score)/expectedPenalty, tolerance) + } } // withStakedIdentity returns a function that sets the identity provider to return an staked identity for the given peer id. From 6f1b1f4e073f41804fe2c399867baa7d0ab43d18 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:22:45 -0500 Subject: [PATCH 41/70] add CtrlMsgTopicType type --- network/p2p/consumers.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index 6554c0b388e..d3961df220f 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -24,6 +24,27 @@ type GossipSubInspectorNotifDistributor interface { AddConsumer(GossipSubInvCtrlMsgNotifConsumer) } +// CtrlMsgTopicType represents the type of the topic within a control message. +type CtrlMsgTopicType int + +const ( + // CtrlMsgNonClusterTopicType represents a non-cluster-prefixed topic. + CtrlMsgNonClusterTopicType CtrlMsgTopicType = iota + // CtrlMsgTopicTypeClusterPrefixed represents a cluster-prefixed topic. + CtrlMsgTopicTypeClusterPrefixed +) + +func (t CtrlMsgTopicType) String() string { + switch t { + case CtrlMsgNonClusterTopicType: + return "non-cluster-prefixed" + case CtrlMsgTopicTypeClusterPrefixed: + return "cluster-prefixed" + default: + return "unknown" + } +} + // InvCtrlMsgNotif is the notification sent to the consumer when an invalid control message is received. // It models the information that is available to the consumer about a misbehaving peer. type InvCtrlMsgNotif struct { @@ -35,12 +56,12 @@ type InvCtrlMsgNotif struct { MsgType p2pmsg.ControlMessageType // Count the number of errors. Count uint64 - // IsClusterPrefixed reports whether the error occurred on a cluster-prefixed topic within the control message. + // TopicType reports whether the error occurred on a cluster-prefixed topic within the control message. // Notifications must be explicitly marked as cluster-prefixed or not because the penalty applied to the GossipSub score // for an error on a cluster-prefixed topic is more lenient than the penalty applied to a non-cluster-prefixed topic. // This distinction ensures that nodes engaged in cluster-prefixed topic communication are not penalized too harshly, // as such communication is vital to the progress of the chain. - IsClusterPrefixed bool + TopicType CtrlMsgTopicType } // NewInvalidControlMessageNotification returns a new *InvCtrlMsgNotif From 8598bda8e9a0292513bbd5dd802540f702f33a96 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:22:57 -0500 Subject: [PATCH 42/70] Update network/p2p/consumers.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/consumers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index 6554c0b388e..a030ef39f37 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -52,7 +52,7 @@ type InvCtrlMsgNotif struct { // // Returns: // - *InvCtlMsgNotif: invalid control message notification. -func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.ControlMessageType, err error, count uint64, isClusterPrefixed bool) *InvCtrlMsgNotif { +func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.ControlMessageType, err error, count uint64, topicType CtrlMsgTopicType) *InvCtrlMsgNotif { return &InvCtrlMsgNotif{ PeerID: peerID, Error: err, From fd316b0b74329a0bb1830df4313121d6d8c1da7e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:23:05 -0500 Subject: [PATCH 43/70] Update network/p2p/consumers.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/consumers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index a030ef39f37..2213a9dcfda 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -58,7 +58,7 @@ func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.Cont Error: err, MsgType: ctlMsgType, Count: count, - IsClusterPrefixed: isClusterPrefixed, + TopicType: topicType, } } From 24efbdca3e458c6c8b1537475a72936a0daeb028 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:23:13 -0500 Subject: [PATCH 44/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index a632f5aacca..aba5723c059 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -289,7 +289,7 @@ func (c *ControlMsgValidationInspector) checkPubsubMessageSender(message *pubsub // - DuplicateTopicErr: if there are any duplicate topics in the list of grafts // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. -func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, bool) { +func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) From 0bde7b1deb088a6a501fad44472efbae08e807d4 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:23:21 -0500 Subject: [PATCH 45/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index aba5723c059..d1aa0aeadd3 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -294,7 +294,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), false + return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType } tracker.set(topic.String()) err, isClusterPrefixed := c.validateTopic(from, topic, activeClusterIDS) From e50d2d015e342c09a58729b71de454034da9ed95 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:27:33 -0500 Subject: [PATCH 46/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index d1aa0aeadd3..88665fd0e93 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -297,7 +297,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType } tracker.set(topic.String()) - err, isClusterPrefixed := c.validateTopic(from, topic, activeClusterIDS) + err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { return err, isClusterPrefixed } From a477e1fa9521c898c90e0a5587c20d52ef35ce1a Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:27:43 -0500 Subject: [PATCH 47/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 88665fd0e93..7178a3c28bb 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -702,7 +702,7 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe // handle cluster prefixed topics if channels.IsClusterChannel(channel) { - return c.validateClusterPrefixedTopic(from, topic, activeClusterIds), true + return c.validateClusterPrefixedTopic(from, topic, activeClusterIds), p2p.CtrlMsgTopicTypeClusterPrefixed } // non cluster prefixed topic validation From 7978b8c98c981f52caa7f7ae89450a366e3dbca6 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:27:56 -0500 Subject: [PATCH 48/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 7178a3c28bb..db19b09fc67 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -299,7 +299,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { - return err, isClusterPrefixed + return err, ctrlMsgType } } return nil, false From 6f7824b5259047f1e7a2883e8def1b5703f6b975 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:28:03 -0500 Subject: [PATCH 49/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index db19b09fc67..e1ceeeb1156 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -708,7 +708,7 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe // non cluster prefixed topic validation err := channels.IsValidNonClusterFlowTopic(topic, c.sporkID) if err != nil { - return err, false + return err, p2p.CtrlMsgNonClusterTopicType } return nil, false } From 8535fb4dca9539f892336504ba793eaa002f2a0f Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:28:16 -0500 Subject: [PATCH 50/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index e1ceeeb1156..330a8be69c0 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -302,7 +302,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft return err, ctrlMsgType } } - return nil, false + return nil, p2p.CtrlMsgNonClusterTopicType } // inspectPruneMessages performs topic validation on all prunes in the control message using the provided validateTopic func while tracking duplicates. From 0cfd26c3a1196f843b82f0b12bcd68f8604684d8 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:28:28 -0500 Subject: [PATCH 51/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 330a8be69c0..5371700207f 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -315,7 +315,7 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // or any duplicate message ids found inside a single iHave. // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. -func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, bool) { +func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) From 04a25f089bc78d71b03bef44dd26f0f577b88fac Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:28:43 -0500 Subject: [PATCH 52/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 5371700207f..4a19b0bb483 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -323,7 +323,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), false } tracker.set(topic.String()) - err, isClusterPrefixed := c.validateTopic(from, topic, activeClusterIDS) + err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { return err, isClusterPrefixed } From 7067a1cd3750beabf6a6c5b407ccc995919d784f Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:28:57 -0500 Subject: [PATCH 53/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 4a19b0bb483..370048732a3 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -325,7 +325,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { - return err, isClusterPrefixed + return err, ctrlMsgType } } return nil, false From cb699abed2afcb1fb91c4c431b381422975673d2 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:29:09 -0500 Subject: [PATCH 54/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 370048732a3..7cdcad1412c 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -328,7 +328,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune return err, ctrlMsgType } } - return nil, false + return nil, p2p.CtrlMsgNonClusterTopicType } // inspectIHaveMessages performs topic validation on all ihaves in the control message using the provided validateTopic func while tracking duplicates. From bd0c3e2479560bd1845bfc92d31ee8b6f55765f4 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:29:22 -0500 Subject: [PATCH 55/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 7cdcad1412c..c97210660bf 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -341,7 +341,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune // or any duplicate message ids found inside a single iHave. // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. -func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihaves []*pubsub_pb.ControlIHave, activeClusterIDS flow.ChainIDList) (error, bool) { +func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihaves []*pubsub_pb.ControlIHave, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { if len(ihaves) == 0 { return nil, false } From 295aa59fd08743fa0afdb13ac4e77d40b3bdf76b Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:29:35 -0500 Subject: [PATCH 56/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index c97210660bf..f698d7eea00 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -343,7 +343,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihaves []*pubsub_pb.ControlIHave, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { if len(ihaves) == 0 { - return nil, false + return nil, p2p.CtrlMsgNonClusterTopicType } lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). From 208a0eb3e1ec59b85485273a87c6bcf0404dbd4e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:29:50 -0500 Subject: [PATCH 57/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index f698d7eea00..55a83c8c732 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -357,7 +357,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() if duplicateTopicTracker.isDuplicate(topic) { - return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), false + return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } duplicateTopicTracker.set(topic) err, isClusterPrefixed := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) From e21d9e5468b10cc6778f57fb4c4a38463b7f66c1 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:30:07 -0500 Subject: [PATCH 58/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 55a83c8c732..b02fb534669 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -360,7 +360,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } duplicateTopicTracker.set(topic) - err, isClusterPrefixed := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) + err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { return err, isClusterPrefixed } From 60a5dba3b570d2492b8529ae4c366626e7012380 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:30:18 -0500 Subject: [PATCH 59/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index b02fb534669..3445903ae99 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -362,7 +362,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave duplicateTopicTracker.set(topic) err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { - return err, isClusterPrefixed + return err, ctrlMsgType } for _, messageID := range messageIds { From 0fe0ce12a39276686a344ee27dabdc52e50e78e9 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:30:35 -0500 Subject: [PATCH 60/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 3445903ae99..673c8612739 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -367,7 +367,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave for _, messageID := range messageIds { if duplicateMessageIDTracker.isDuplicate(messageID) { - return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave), false + return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType } duplicateMessageIDTracker.set(messageID) } From ac7ad1c835ba108aa55c9b618d0e33f2b821ee35 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:30:47 -0500 Subject: [PATCH 61/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 673c8612739..34898b5c848 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -375,7 +375,7 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave lg.Debug(). Int("total_message_ids", totalMessageIds). Msg("ihave control message validation complete") - return nil, false + return nil, p2p.CtrlMsgNonClusterTopicType } // inspectIWantMessages inspects RPC iWant control messages. This func will sample the iWants and perform validation on each iWant in the sample. From 18af844ed7605e93b894016b9d5889b5b5094af0 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:30:58 -0500 Subject: [PATCH 62/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 34898b5c848..21a890c9223 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -694,7 +694,7 @@ func (c *ControlMsgValidationInspector) performSample(ctrlMsg p2pmsg.ControlMess // // This func returns an exception in case of unexpected bug or state corruption if cluster prefixed topic validation // fails due to unexpected error returned when getting the active cluster IDS. -func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) (error, bool) { +func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { channel, ok := channels.ChannelFromTopic(topic) if !ok { return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")), false From 952d536b7f9bb6c4f3e77eb515c8f34eb6fda1af Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:31:09 -0500 Subject: [PATCH 63/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 21a890c9223..cd8da8bf054 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -697,7 +697,7 @@ func (c *ControlMsgValidationInspector) performSample(ctrlMsg p2pmsg.ControlMess func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { channel, ok := channels.ChannelFromTopic(topic) if !ok { - return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")), false + return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")), p2p.CtrlMsgNonClusterTopicType } // handle cluster prefixed topics From 76b361a90d5c33a0520a66359cbc081f44ca4e52 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 14:31:20 -0500 Subject: [PATCH 64/70] Update network/p2p/inspector/validation/control_message_validation_inspector.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- .../validation/control_message_validation_inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index cd8da8bf054..da0e457dd56 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -710,7 +710,7 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe if err != nil { return err, p2p.CtrlMsgNonClusterTopicType } - return nil, false + return nil, p2p.CtrlMsgTopicTypeClusterPrefixed } // validateClusterPrefixedTopic validates cluster prefixed topics. From 0ddfcd21ae13fbb30fa08b71edbd258f86eb97b1 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 28 Nov 2023 22:02:31 +0200 Subject: [PATCH 65/70] Fixed test --- module/state_synchronization/indexer/indexer_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/module/state_synchronization/indexer/indexer_test.go b/module/state_synchronization/indexer/indexer_test.go index 126afb75274..14d959df4ce 100644 --- a/module/state_synchronization/indexer/indexer_test.go +++ b/module/state_synchronization/indexer/indexer_test.go @@ -228,9 +228,10 @@ func TestIndexer_Failure(t *testing.T) { // make sure the error returned is as expected expectedErr := fmt.Errorf( - "failed to index block data at height %d: could not index register payloads at height %d: error persisting data", - test.blocks[lastIndexedIndex].Header.Height+1, + "failed to index block data at height %d: %w", test.blocks[lastIndexedIndex].Header.Height+1, + fmt.Errorf( + "could not index register payloads at height %d: %w", test.blocks[lastIndexedIndex].Header.Height+1, fmt.Errorf("error persisting data")), ) _, cancel := context.WithCancel(context.Background()) From 880fbccc0798cab9fcce7dc2cf0e552185cdeace Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 22:53:19 -0500 Subject: [PATCH 66/70] update tests remove suite pattern causing flakiness --- network/p2p/consumers.go | 10 +- .../control_message_validation_inspector.go | 33 +- ...ntrol_message_validation_inspector_test.go | 819 +++++++++--------- network/p2p/scoring/registry.go | 2 +- utils/unittest/fixtures.go | 2 +- 5 files changed, 417 insertions(+), 449 deletions(-) diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index d0fba42e5f5..f2896dfcb86 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -25,7 +25,7 @@ type GossipSubInspectorNotifDistributor interface { } // CtrlMsgTopicType represents the type of the topic within a control message. -type CtrlMsgTopicType int +type CtrlMsgTopicType uint64 const ( // CtrlMsgNonClusterTopicType represents a non-cluster-prefixed topic. @@ -75,10 +75,10 @@ type InvCtrlMsgNotif struct { // - *InvCtlMsgNotif: invalid control message notification. func NewInvalidControlMessageNotification(peerID peer.ID, ctlMsgType p2pmsg.ControlMessageType, err error, count uint64, topicType CtrlMsgTopicType) *InvCtrlMsgNotif { return &InvCtrlMsgNotif{ - PeerID: peerID, - Error: err, - MsgType: ctlMsgType, - Count: count, + PeerID: peerID, + Error: err, + MsgType: ctlMsgType, + Count: count, TopicType: topicType, } } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index da0e457dd56..6247c8b1f92 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -221,27 +221,27 @@ func (c *ControlMsgValidationInspector) processInspectRPCReq(req *InspectRPCRequ for _, ctrlMsgType := range p2pmsg.ControlMessageTypes() { switch ctrlMsgType { case p2pmsg.CtrlMsgGraft: - err, isClusterPrefixed := c.inspectGraftMessages(req.Peer, req.rpc.GetControl().GetGraft(), activeClusterIDS) + err, topicType := c.inspectGraftMessages(req.Peer, req.rpc.GetControl().GetGraft(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgGraft, err, 1, isClusterPrefixed) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgGraft, err, 1, topicType) return nil } case p2pmsg.CtrlMsgPrune: - err, isClusterPrefixed := c.inspectPruneMessages(req.Peer, req.rpc.GetControl().GetPrune(), activeClusterIDS) + err, topicType := c.inspectPruneMessages(req.Peer, req.rpc.GetControl().GetPrune(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgPrune, err, 1, isClusterPrefixed) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgPrune, err, 1, topicType) return nil } case p2pmsg.CtrlMsgIWant: err := c.inspectIWantMessages(req.Peer, req.rpc.GetControl().GetIwant()) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIWant, err, 1, false) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIWant, err, 1, p2p.CtrlMsgNonClusterTopicType) return nil } case p2pmsg.CtrlMsgIHave: - err, isClusterPrefixed := c.inspectIHaveMessages(req.Peer, req.rpc.GetControl().GetIhave(), activeClusterIDS) + err, topicType := c.inspectIHaveMessages(req.Peer, req.rpc.GetControl().GetIhave(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIHave, err, 1, isClusterPrefixed) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.CtrlMsgIHave, err, 1, topicType) return nil } } @@ -250,7 +250,7 @@ func (c *ControlMsgValidationInspector) processInspectRPCReq(req *InspectRPCRequ // inspect rpc publish messages after all control message validation has passed err, errCount := c.inspectRpcPublishMessages(req.Peer, req.rpc.GetPublish(), activeClusterIDS) if err != nil { - c.logAndDistributeAsyncInspectErrs(req, p2pmsg.RpcPublishMessage, err, errCount, false) + c.logAndDistributeAsyncInspectErrs(req, p2pmsg.RpcPublishMessage, err, errCount, p2p.CtrlMsgNonClusterTopicType) return nil } @@ -320,7 +320,7 @@ func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prune for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), false + return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType } tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) @@ -699,7 +699,6 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe if !ok { return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")), p2p.CtrlMsgNonClusterTopicType } - // handle cluster prefixed topics if channels.IsClusterChannel(channel) { return c.validateClusterPrefixedTopic(from, topic, activeClusterIds), p2p.CtrlMsgTopicTypeClusterPrefixed @@ -710,7 +709,7 @@ func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channe if err != nil { return err, p2p.CtrlMsgNonClusterTopicType } - return nil, p2p.CtrlMsgTopicTypeClusterPrefixed + return nil, p2p.CtrlMsgNonClusterTopicType } // validateClusterPrefixedTopic validates cluster prefixed topics. @@ -728,12 +727,12 @@ func (c *ControlMsgValidationInspector) validateClusterPrefixedTopic(from peer.I lg := c.logger.With(). Str("from", p2plogging.PeerId(from)). Logger() - // reject messages from unstaked nodes for cluster prefixed topics + + // only staked nodes are expected to participate on cluster prefixed topics nodeID, err := c.getFlowIdentifier(from) if err != nil { return err } - if len(activeClusterIds) == 0 { // cluster IDs have not been updated yet _, incErr := c.tracker.Inc(nodeID) @@ -743,7 +742,7 @@ func (c *ControlMsgValidationInspector) validateClusterPrefixedTopic(from peer.I } // if the amount of messages received is below our hard threshold log the error and return nil. - if c.checkClusterPrefixHardThreshold(nodeID) { + if ok := c.checkClusterPrefixHardThreshold(nodeID); ok { lg.Warn(). Err(err). Str("topic", topic.String()). @@ -811,13 +810,13 @@ func (c *ControlMsgValidationInspector) checkClusterPrefixHardThreshold(nodeID f // - err: the error that occurred. // - count: the number of occurrences of the error. // - isClusterPrefixed: indicates if the errors occurred on a cluster prefixed topic. -func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *InspectRPCRequest, ctlMsgType p2pmsg.ControlMessageType, err error, count uint64, isClusterPrefixed bool) { +func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *InspectRPCRequest, ctlMsgType p2pmsg.ControlMessageType, err error, count uint64, topicType p2p.CtrlMsgTopicType) { lg := c.logger.With(). Err(err). Str("control_message_type", ctlMsgType.String()). Bool(logging.KeySuspicious, true). Bool(logging.KeyNetworkingSecurity, true). - Bool("is_cluster_prefixed", isClusterPrefixed). + Str("topic_type", topicType.String()). Uint64("error_count", count). Str("peer_id", p2plogging.PeerId(req.Peer)). Logger() @@ -828,7 +827,7 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In case IsErrUnstakedPeer(err): lg.Warn().Msg("control message received from unstaked peer") default: - distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, isClusterPrefixed)) + distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, topicType)) if distErr != nil { lg.Error(). Err(distErr). diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 09dd59276b1..60b1523dae9 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -3,15 +3,12 @@ package validation_test import ( "context" "fmt" + "math/rand" "testing" "time" pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" @@ -23,68 +20,12 @@ import ( "github.com/onflow/flow-go/network/p2p/inspector/validation" p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" - "github.com/onflow/flow-go/network/p2p/p2pconf" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) -type ControlMsgValidationInspectorSuite struct { - suite.Suite - sporkID flow.Identifier - config *p2pconf.GossipSubRPCValidationInspectorConfigs - distributor *mockp2p.GossipSubInspectorNotificationDistributor - params *validation.InspectorParams - rpcTracker *mockp2p.RpcControlTracking - idProvider *mockmodule.IdentityProvider - inspector *validation.ControlMsgValidationInspector - signalerCtx *irrecoverable.MockSignalerContext - cancel context.CancelFunc - defaultTopicOracle func() []string -} - -func TestControlMsgValidationInspector(t *testing.T) { - suite.Run(t, new(ControlMsgValidationInspectorSuite)) -} - -func (suite *ControlMsgValidationInspectorSuite) SetupTest() { - suite.sporkID = unittest.IdentifierFixture() - flowConfig, err := config.DefaultConfig() - require.NoError(suite.T(), err, "failed to get default flow config") - suite.config = &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs - distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(suite.T()) - p2ptest.MockInspectorNotificationDistributorReadyDoneAware(distributor) - suite.distributor = distributor - suite.idProvider = mockmodule.NewIdentityProvider(suite.T()) - rpcTracker := mockp2p.NewRpcControlTracking(suite.T()) - suite.rpcTracker = rpcTracker - params := &validation.InspectorParams{ - Logger: unittest.Logger(), - SporkID: suite.sporkID, - Config: &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs, - Distributor: distributor, - IdProvider: suite.idProvider, - HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), - InspectorMetrics: metrics.NewNoopCollector(), - RpcTracker: rpcTracker, - NetworkingType: network.PublicNetwork, - } - suite.params = params - inspector, err := validation.NewControlMsgValidationInspector(params) - require.NoError(suite.T(), err, "failed to create control message validation inspector fixture") - suite.inspector = inspector - suite.defaultTopicOracle = func() []string { - return []string{} - } - ctx, cancel := context.WithCancel(context.Background()) - suite.cancel = cancel - suite.signalerCtx = irrecoverable.NewMockSignalerContext(suite.T(), ctx) -} - -func (suite *ControlMsgValidationInspectorSuite) StopInspector() { - suite.cancel() - unittest.RequireCloseBefore(suite.T(), suite.inspector.Done(), 500*time.Millisecond, "inspector did not stop") -} - func TestNewControlMsgValidationInspector(t *testing.T) { t.Run("should create validation inspector without error", func(t *testing.T) { sporkID := unittest.IdentifierFixture() @@ -133,117 +74,123 @@ func TestNewControlMsgValidationInspector(t *testing.T) { // TestControlMessageValidationInspector_TruncateRPC verifies the expected truncation behavior of RPC control messages. // Message truncation for each control message type occurs when the count of control // messages exceeds the configured maximum sample size for that control message type. -func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationInspector_truncateRPC() { - suite.T().Run("truncateGraftMessages should truncate graft messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() +func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { + t.Run("truncateGraftMessages should truncate graft messages as expected", func(t *testing.T) { + graftPruneMessageMaxSampleSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.config.GraftPruneMessageMaxSampleSize = 100 - suite.inspector.Start(suite.signalerCtx) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + inspector.Start(signalerCtx) // topic validation not performed so we can use random strings - graftsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixtures(unittest.IdentifierListFixture(200).Strings()...)...)) - require.Greater(t, len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()), suite.config.GraftPruneMessageMaxSampleSize) + graftsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) + require.Greater(t, len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()), graftPruneMessageMaxSampleSize) graftsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixtures(unittest.IdentifierListFixture(50).Strings()...)...)) - require.Less(t, len(graftsLessThanMaxSampleSize.GetControl().GetGraft()), suite.config.GraftPruneMessageMaxSampleSize) + require.Less(t, len(graftsLessThanMaxSampleSize.GetControl().GetGraft()), graftPruneMessageMaxSampleSize) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, graftsGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, graftsLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, graftsGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, graftsLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with grafts greater than configured max sample size should be truncated to GraftPruneMessageMaxSampleSize - shouldBeTruncated := len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()) == suite.config.GraftPruneMessageMaxSampleSize + shouldBeTruncated := len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()) == graftPruneMessageMaxSampleSize // rpc with grafts less than GraftPruneMessageMaxSampleSize should not be truncated shouldNotBeTruncated := len(graftsLessThanMaxSampleSize.GetControl().GetGraft()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) - suite.T().Run("truncatePruneMessages should truncate prune messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("truncatePruneMessages should truncate prune messages as expected", func(t *testing.T) { + graftPruneMessageMaxSampleSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.config.GraftPruneMessageMaxSampleSize = 100 + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) //unittest.RequireCloseBefore(t, inspector.Ready(), 100*time.Millisecond, "inspector did not start") // topic validation not performed, so we can use random strings - prunesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(200).Strings()...)...)) - require.Greater(t, len(prunesGreaterThanMaxSampleSize.GetControl().GetPrune()), suite.config.GraftPruneMessageMaxSampleSize) + prunesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) + require.Greater(t, len(prunesGreaterThanMaxSampleSize.GetControl().GetPrune()), graftPruneMessageMaxSampleSize) prunesLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(50).Strings()...)...)) - require.Less(t, len(prunesLessThanMaxSampleSize.GetControl().GetPrune()), suite.config.GraftPruneMessageMaxSampleSize) + require.Less(t, len(prunesLessThanMaxSampleSize.GetControl().GetPrune()), graftPruneMessageMaxSampleSize) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, prunesGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, prunesLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, prunesGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, prunesLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with prunes greater than configured max sample size should be truncated to GraftPruneMessageMaxSampleSize - shouldBeTruncated := len(prunesGreaterThanMaxSampleSize.GetControl().GetPrune()) == suite.config.GraftPruneMessageMaxSampleSize + shouldBeTruncated := len(prunesGreaterThanMaxSampleSize.GetControl().GetPrune()) == graftPruneMessageMaxSampleSize // rpc with prunes less than GraftPruneMessageMaxSampleSize should not be truncated shouldNotBeTruncated := len(prunesLessThanMaxSampleSize.GetControl().GetPrune()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) - suite.T().Run("truncateIHaveMessages should truncate iHave messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("truncateIHaveMessages should truncate iHave messages as expected", func(t *testing.T) { + maxSampleSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.IHaveRPCInspectionConfig.MaxSampleSize = maxSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.config.IHaveRPCInspectionConfig.MaxSampleSize = 100 - suite.inspector.Start(suite.signalerCtx) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() + inspector.Start(signalerCtx) // topic validation not performed so we can use random strings - iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(200, unittest.IdentifierListFixture(200).Strings()...)...)) - require.Greater(t, len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHaveRPCInspectionConfig.MaxSampleSize) + iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(2000).Strings()...)...)) + require.Greater(t, len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()), maxSampleSize) iHavesLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(200, unittest.IdentifierListFixture(50).Strings()...)...)) - require.Less(t, len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()), suite.config.IHaveRPCInspectionConfig.MaxSampleSize) + require.Less(t, len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()), maxSampleSize) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, iHavesLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with iHaves greater than configured max sample size should be truncated to MaxSampleSize - shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == suite.config.IHaveRPCInspectionConfig.MaxSampleSize + shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == maxSampleSize // rpc with iHaves less than MaxSampleSize should not be truncated shouldNotBeTruncated := len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) - suite.T().Run("truncateIHaveMessageIds should truncate iHave message ids as expected", func(t *testing.T) { - //suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.SetupTest() - defer suite.StopInspector() + t.Run("truncateIHaveMessageIds should truncate iHave message ids as expected", func(t *testing.T) { + maxMessageIDSampleSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize = maxMessageIDSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() - suite.config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize = 100 - suite.inspector.Start(suite.signalerCtx) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() + inspector.Start(signalerCtx) // topic validation not performed so we can use random strings - iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(200, unittest.IdentifierListFixture(10).Strings()...)...)) + iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(10).Strings()...)...)) iHavesLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(50, unittest.IdentifierListFixture(10).Strings()...)...)) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, iHavesLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iHave := range iHavesGreaterThanMaxSampleSize.GetControl().GetIhave() { // rpc with iHaves message ids greater than configured max sample size should be truncated to MaxSampleSize - if len(iHave.GetMessageIDs()) != suite.config.IHaveRPCInspectionConfig.MaxMessageIDSampleSize { + if len(iHave.GetMessageIDs()) != maxMessageIDSampleSize { return false } } @@ -255,56 +202,61 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns } return true }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) - suite.T().Run("truncateIWantMessages should truncate iWant messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("truncateIWantMessages should truncate iWant messages as expected", func(t *testing.T) { + maxSampleSize := uint(100) + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.IWantRPCInspectionConfig.MaxSampleSize = maxSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.config.IWantRPCInspectionConfig.MaxSampleSize = 100 - suite.inspector.Start(suite.signalerCtx) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() + inspector.Start(signalerCtx) iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(200, 200)...)) - require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWantRPCInspectionConfig.MaxSampleSize) + require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), maxSampleSize) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(50, 200)...)) - require.Less(t, uint(len(iWantsLessThanMaxSampleSize.GetControl().GetIwant())), suite.config.IWantRPCInspectionConfig.MaxSampleSize) + require.Less(t, uint(len(iWantsLessThanMaxSampleSize.GetControl().GetIwant())), maxSampleSize) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, iWantsLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { // rpc with iWants greater than configured max sample size should be truncated to MaxSampleSize - shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(suite.config.IWantRPCInspectionConfig.MaxSampleSize) + shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(maxSampleSize) // rpc with iWants less than MaxSampleSize should not be truncated shouldNotBeTruncated := len(iWantsLessThanMaxSampleSize.GetControl().GetIwant()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) - suite.T().Run("truncateIWantMessageIds should truncate iWant message ids as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("truncateIWantMessageIds should truncate iWant message ids as expected", func(t *testing.T) { + maxMessageIDSampleSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.IWantRPCInspectionConfig.MaxMessageIDSampleSize = maxMessageIDSampleSize + }) // topic validation is ignored set any topic oracle - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() - suite.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize = 100 - suite.inspector.Start(suite.signalerCtx) - - iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 200)...)) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() + inspector.Start(signalerCtx) + + iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 2000)...)) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 50)...)) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) - require.NoError(t, suite.inspector.Inspect(from, iWantsLessThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) + require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iWant := range iWantsGreaterThanMaxSampleSize.GetControl().GetIwant() { // rpc with iWants message ids greater than configured max sample size should be truncated to MaxSampleSize - if len(iWant.GetMessageIDs()) != suite.config.IWantRPCInspectionConfig.MaxMessageIDSampleSize { + if len(iWant.GetMessageIDs()) != maxMessageIDSampleSize { return false } } @@ -316,29 +268,29 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns } return true }, time.Second, 500*time.Millisecond) + stopInspector(t, cancel, inspector) }) } // TestControlMessageValidationInspector_processInspectRPCReq verifies the correct behavior of control message validation. // It ensures that valid RPC control messages do not trigger erroneous invalid control message notifications, // while all types of invalid control messages trigger expected notifications. -func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationInspector_processInspectRPCReq() { - suite.T().Run("processInspectRPCReq should not disseminate any invalid notification errors for valid RPC's", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - defer suite.distributor.AssertNotCalled(t, "Distribute") +func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { + t.Run("processInspectRPCReq should not disseminate any invalid notification errors for valid RPC's", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, _ := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") topics := []string{ - fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID), - fmt.Sprintf("%s/%s", channels.PushBlocks, suite.sporkID), - fmt.Sprintf("%s/%s", channels.SyncCommittee, suite.sporkID), - fmt.Sprintf("%s/%s", channels.RequestChunks, suite.sporkID), + fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID), + fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID), + fmt.Sprintf("%s/%s", channels.SyncCommittee, sporkID), + fmt.Sprintf("%s/%s", channels.RequestChunks, sporkID), } // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return topics })) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) grafts := unittest.P2PRPCGraftFixtures(topics...) prunes := unittest.P2PRPCPruneFixtures(topics...) ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) @@ -357,25 +309,25 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns unittest.WithIHaves(ihaves...), unittest.WithIWants(iwants...), unittest.WithPubsubMessages(pubsubMsgs...)) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { id, ok := args[0].(string) require.True(t, ok) require.Contains(t, expectedMsgIds, id) }) from := unittest.PeerIdFixture(t) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("processInspectRPCReq should disseminate invalid control message notification for control messages with duplicate topics", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID) + t.Run("processInspectRPCReq should disseminate invalid control message notification for control messages with duplicate topics", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{duplicateTopic} })) // create control messages with duplicate topic @@ -386,31 +338,31 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns duplicateTopicGraftsRpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) duplicateTopicPrunesRpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) duplicateTopicIHavesRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(func(args mock.Arguments) { + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(func(args mock.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") require.Equal(t, from, notification.PeerID) require.Contains(t, []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave}, notification.MsgType) require.True(t, validation.IsDuplicateTopicErr(notification.Error)) }) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, duplicateTopicGraftsRpc)) - require.NoError(t, suite.inspector.Inspect(from, duplicateTopicPrunesRpc)) - require.NoError(t, suite.inspector.Inspect(from, duplicateTopicIHavesRpc)) + require.NoError(t, inspector.Inspect(from, duplicateTopicGraftsRpc)) + require.NoError(t, inspector.Inspect(from, duplicateTopicPrunesRpc)) + require.NoError(t, inspector.Inspect(from, duplicateTopicIHavesRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectGraftMessages should disseminate invalid control message notification for invalid graft messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectGraftMessages should disseminate invalid control message notification for invalid graft messages as expected", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t) // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, suite.sporkID) + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{unknownTopic, malformedTopic, invalidSporkIDTopic} })) unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) @@ -422,28 +374,28 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, unknownTopicReq)) - require.NoError(t, suite.inspector.Inspect(from, malformedTopicReq)) - require.NoError(t, suite.inspector.Inspect(from, invalidSporkIDTopicReq)) + require.NoError(t, inspector.Inspect(from, unknownTopicReq)) + require.NoError(t, inspector.Inspect(from, malformedTopicReq)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectPruneMessages should disseminate invalid control message notification for invalid prune messages as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectPruneMessages should disseminate invalid control message notification for invalid prune messages as expected", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t) // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, suite.sporkID) + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) unknownTopicPrune := unittest.P2PRPCPruneFixture(&unknownTopic) malformedTopicPrune := unittest.P2PRPCPruneFixture(&malformedTopic) invalidSporkIDTopicPrune := unittest.P2PRPCPruneFixture(&invalidSporkIDTopic) // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{unknownTopic, malformedTopic, invalidSporkIDTopic} })) unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(unknownTopicPrune)) @@ -451,25 +403,25 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, suite.inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, suite.inspector.Inspect(from, invalidSporkIDTopicRpc)) + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with invalid topics as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with invalid topics as expected", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t) // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, suite.sporkID) + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{unknownTopic, malformedTopic, invalidSporkIDTopic} })) unknownTopicIhave := unittest.P2PRPCIHaveFixture(&unknownTopic, unittest.IdentifierListFixture(5).Strings()...) @@ -481,23 +433,23 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, suite.inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, suite.inspector.Inspect(from, invalidSporkIDTopicRpc)) + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with duplicate message ids as expected", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), suite.sporkID) + t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with duplicate message ids as expected", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) // avoid unknown topics errors - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{validTopic} })) duplicateMsgID := unittest.IdentifierFixture() @@ -507,20 +459,20 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, duplicateMsgIDRpc)) + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when duplicate message ids exceeds the allowed threshold", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when duplicate message ids exceeds the allowed threshold", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) // oracle must be set even though iWant messages do not have topic IDs - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) duplicateMsgID := unittest.IdentifierFixture() duplicates := flow.IdentifierList{duplicateMsgID, duplicateMsgID} msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() @@ -529,349 +481,381 @@ func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationIns duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { id, ok := args[0].(string) require.True(t, ok) require.Contains(t, msgIds, id) }) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, duplicateMsgIDRpc)) + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - // set cache miss check size to 0 forcing the inspector to check the cache misses with only a single iWant - suite.config.CacheMissCheckSize = 0 - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - suite.config.IWantRPCInspectionConfig.CacheMissThreshold = .9 - msgIds := unittest.IdentifierListFixture(100).Strings() + t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { + cacheMissCheckSize := 1000 + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.CacheMissCheckSize = cacheMissCheckSize + // set high cache miss threshold to ensure we only disseminate notification when it is exceeded + params.Config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + }) // oracle must be set even though iWant messages do not have topic IDs - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(cacheMissCheckSize+1, 100)...)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { id, ok := args[0].(string) require.True(t, ok) - require.Contains(t, msgIds, id) + found := false + for _, iwant := range inspectMsgRpc.GetControl().GetIwant() { + for _, messageID := range iwant.GetMessageIDs() { + if id == messageID { + found = true + } + } + } + require.True(t, found) }) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // if size of iwants not greater than 10 cache misses will not be checked + params.Config.CacheMissCheckSize = 10 + // set high cache miss threshold to ensure we only disseminate notification when it is exceeded + params.Config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + }) // oracle must be set even though iWant messages do not have topic IDs - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - defer suite.distributor.AssertNotCalled(t, "Distribute") - // if size of iwants not greater than 10 cache misses will not be checked - suite.config.CacheMissCheckSize = 10 - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - suite.config.IWantRPCInspectionConfig.CacheMissThreshold = .9 + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + defer distributor.AssertNotCalled(t, "Distribute") + msgIds := unittest.IdentifierListFixture(100).Strings() inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) - suite.rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - suite.rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { id, ok := args[0].(string) require.True(t, ok) require.Contains(t, msgIds, id) }) from := unittest.PeerIdFixture(t) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 + t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.RpcMessageErrorThreshold = errThreshold + }) // create unknown topic - unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), suite.sporkID)).String() + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() // create malformed topic malformedTopic := channels.Topic("!@#$%^&**((").String() // a topics spork ID is considered invalid if it does not match the current spork ID invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() - // create 10 normal messages - pubsubMsgs := unittest.GossipSubMessageFixtures(10, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID)) - // add 5 invalid messages to force notification dissemination - pubsubMsgs = append(pubsubMsgs, []*pubsub_pb.Message{ + pubsubMsgs := unittest.GossipSubMessageFixtures(50, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) + // add 550 invalid messages to force notification dissemination + invalidMessageFixtures := []*pubsub_pb.Message{ {Topic: &unknownTopic}, {Topic: &malformedTopic}, - {Topic: &malformedTopic}, {Topic: &invalidSporkIDTopic}, - {Topic: &invalidSporkIDTopic}, - }...) + } + for i := 0; i < errThreshold+1; i++ { + pubsubMsgs = append(pubsubMsgs, invalidMessageFixtures[rand.Intn(len(invalidMessageFixtures))]) + } rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) topics := make([]string, len(pubsubMsgs)) for i, msg := range pubsubMsgs { topics[i] = *msg.Topic } // set topic oracle to return list of topics to avoid hasSubscription errors and force topic validation - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + require.NoError(t, inspector.SetTopicOracle(func() []string { return topics })) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + inspector.Start(signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 - pubsubMsgs := unittest.GossipSubMessageFixtures(5, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID)) + t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.RpcMessageErrorThreshold = errThreshold + }) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) from := unittest.PeerIdFixture(t) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - // set topic oracle to return list of topics excluding first topic sent - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when publish messages contain no topic", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - // 5 invalid pubsub messages will force notification dissemination - suite.config.RpcMessageErrorThreshold = 4 - - pubsubMsgs := unittest.GossipSubMessageFixtures(10, "") + t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when publish messages contain no topic", func(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // 5 invalid pubsub messages will force notification dissemination + params.Config.RpcMessageErrorThreshold = errThreshold + }) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) topics := make([]string, len(pubsubMsgs)) for i, msg := range pubsubMsgs { topics[i] = *msg.Topic } // set topic oracle to return list of topics excluding first topic sent - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should not inspect pubsub message sender on public networks", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("inspectRpcPublishMessages should not inspect pubsub message sender on public networks", func(t *testing.T) { + inspector, signalerCtx, cancel, _, _, sporkID, idProvider, _ := inspectorFixture(t) from := unittest.PeerIdFixture(t) - defer suite.idProvider.AssertNotCalled(t, "ByPeerID", from) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID) - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + defer idProvider.AssertNotCalled(t, "ByPeerID", from) + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{topic} })) pubsubMsgs := unittest.GossipSubMessageFixtures(10, topic, unittest.WithFrom(from)) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from unstaked peer", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - - // override the inspector and params, run the inspector in private mode - suite.params.NetworkingType = network.PrivateNetwork - var err error - suite.inspector, err = validation.NewControlMsgValidationInspector(suite.params) - require.NoError(suite.T(), err, "failed to create control message validation inspector fixture") - + t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from unstaked peer", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork + }) from := unittest.PeerIdFixture(t) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID) - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{topic} })) // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - suite.idProvider.On("ByPeerID", from).Return(nil, false).Times(501) + idProvider.On("ByPeerID", from).Return(nil, false).Times(501) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from ejected peer", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - - // override the inspector and params, run the inspector in private mode - suite.params.NetworkingType = network.PrivateNetwork - var err error - suite.inspector, err = validation.NewControlMsgValidationInspector(suite.params) - require.NoError(suite.T(), err, "failed to create control message validation inspector fixture") - + t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from ejected peer", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork + }) from := unittest.PeerIdFixture(t) id := unittest.IdentityFixture() id.Ejected = true - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, suite.sporkID) - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{topic} })) pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - suite.idProvider.On("ByPeerID", from).Return(id, true).Times(501) + idProvider.On("ByPeerID", from).Return(id, true).Times(501) rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - require.NoError(t, err, "failed to get inspect message request") - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, false) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, rpc)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) } // TestNewControlMsgValidationInspector_validateClusterPrefixedTopic ensures cluster prefixed topics are validated as expected. -func (suite *ControlMsgValidationInspectorSuite) TestNewControlMsgValidationInspector_validateClusterPrefixedTopic() { - suite.T().Run("validateClusterPrefixedTopic should not return an error for valid cluster prefixed topics", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - defer suite.distributor.AssertNotCalled(t, "Distribute") +func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testing.T) { + t.Run("validateClusterPrefixedTopic should not return an error for valid cluster prefixed topics", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") clusterID := flow.ChainID(unittest.IdentifierFixture().String()) - clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), sporkID)).String() + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{clusterPrefixedTopic} })) from := unittest.PeerIdFixture(t) - suite.idProvider.On("ByPeerID", from).Return(unittest.IdentityFixture(), true).Once() + idProvider.On("ByPeerID", from).Return(unittest.IdentityFixture(), true).Once() inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) - suite.inspector.ActiveClustersChanged(flow.ChainIDList{clusterID, flow.ChainID(unittest.IdentifierFixture().String()), flow.ChainID(unittest.IdentifierFixture().String())}) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + inspector.ActiveClustersChanged(flow.ChainIDList{clusterID, flow.ChainID(unittest.IdentifierFixture().String()), flow.ChainID(unittest.IdentifierFixture().String())}) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("validateClusterPrefixedTopic should not return error if cluster prefixed hard threshold not exceeded for unknown cluster ids", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - defer suite.distributor.AssertNotCalled(t, "Distribute") - require.NoError(t, suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - // set hard threshold to small number , ensure that a single unknown cluster prefix id does not cause a notification to be disseminated - suite.config.ClusterPrefixHardThreshold = 2 - defer suite.distributor.AssertNotCalled(t, "Distribute") + t.Run("validateClusterPrefixedTopic should not return error if cluster prefixed hard threshold not exceeded for unknown cluster ids", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // set hard threshold to small number , ensure that a single unknown cluster prefix id does not cause a notification to be disseminated + params.Config.ClusterPrefixHardThreshold = 2 + }) + defer distributor.AssertNotCalled(t, "Distribute") + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) clusterID := flow.ChainID(unittest.IdentifierFixture().String()) - clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() + clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), sporkID)).String() from := unittest.PeerIdFixture(t) inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) - suite.idProvider.On("ByPeerID", from).Return(unittest.IdentityFixture(), true).Once() - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + id := unittest.IdentityFixture() + idProvider.On("ByPeerID", from).Return(id, true).Once() + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) + stopInspector(t, cancel, inspector) }) - suite.T().Run("validateClusterPrefixedTopic should return an error when sender is unstaked", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() - defer suite.distributor.AssertNotCalled(t, "Distribute") + t.Run("validateClusterPrefixedTopic should return an error when sender is unstaked", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") clusterID := flow.ChainID(unittest.IdentifierFixture().String()) - clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), sporkID)).String() + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{clusterPrefixedTopic} })) from := unittest.PeerIdFixture(t) - suite.idProvider.On("ByPeerID", from).Return(nil, false).Once() + idProvider.On("ByPeerID", from).Return(nil, false).Once() inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) - suite.inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) + inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) - suite.inspector.Start(suite.signalerCtx) - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) - suite.T().Run("validateClusterPrefixedTopic should return error if cluster prefixed hard threshold exceeded for unknown cluster ids", func(t *testing.T) { - suite.SetupTest() - defer suite.StopInspector() + t.Run("validateClusterPrefixedTopic should return error if cluster prefixed hard threshold exceeded for unknown cluster ids", func(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // the 11th unknown cluster ID error should cause an error + params.Config.ClusterPrefixHardThreshold = 10 + }) clusterID := flow.ChainID(unittest.IdentifierFixture().String()) - clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), suite.sporkID)).String() - require.NoError(t, suite.inspector.SetTopicOracle(func() []string { + clusterPrefixedTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(clusterID), sporkID)).String() + require.NoError(t, inspector.SetTopicOracle(func() []string { return []string{clusterPrefixedTopic} })) - // the 11th unknown cluster ID error should cause an error - suite.config.ClusterPrefixHardThreshold = 10 from := unittest.PeerIdFixture(t) identity := unittest.IdentityFixture() - suite.idProvider.On("ByPeerID", from).Return(identity, true).Times(11) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsUnknownClusterIDErr, true) + idProvider.On("ByPeerID", from).Return(identity, true).Times(11) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsUnknownClusterIDErr, p2p.CtrlMsgTopicTypeClusterPrefixed) inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) - suite.inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) - suite.distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - suite.inspector.Start(suite.signalerCtx) + inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) for i := 0; i < 11; i++ { - require.NoError(t, suite.inspector.Inspect(from, inspectMsgRpc)) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) }) } // TestControlMessageValidationInspector_ActiveClustersChanged validates the expected update of the active cluster IDs list. -func (suite *ControlMsgValidationInspectorSuite) TestControlMessageValidationInspector_ActiveClustersChanged() { - suite.SetupTest() - defer suite.StopInspector() - defer suite.distributor.AssertNotCalled(suite.T(), "Distribute") +func TestControlMessageValidationInspector_ActiveClustersChanged(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, _ := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") identity := unittest.IdentityFixture() - suite.idProvider.On("ByPeerID", mock.AnythingOfType("peer.ID")).Return(identity, true).Times(5) + idProvider.On("ByPeerID", mock.AnythingOfType("peer.ID")).Return(identity, true).Times(5) activeClusterIds := make(flow.ChainIDList, 0) for _, id := range unittest.IdentifierListFixture(5) { activeClusterIds = append(activeClusterIds, flow.ChainID(id.String())) } - suite.inspector.ActiveClustersChanged(activeClusterIds) - require.NoError(suite.T(), suite.inspector.SetTopicOracle(suite.defaultTopicOracle)) - suite.inspector.Start(suite.signalerCtx) - from := unittest.PeerIdFixture(suite.T()) + inspector.ActiveClustersChanged(activeClusterIds) + require.NoError(t, inspector.SetTopicOracle(defaultTopicOracle)) + inspector.Start(signalerCtx) + from := unittest.PeerIdFixture(t) for _, id := range activeClusterIds { - topic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(id), suite.sporkID)).String() + topic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(id), sporkID)).String() rpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&topic))) - require.NoError(suite.T(), suite.inspector.Inspect(from, rpc)) + require.NoError(t, inspector.Inspect(from, rpc)) } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) + stopInspector(t, cancel, inspector) +} + +// invalidTopics returns 3 invalid topics. +// - unknown topic +// - malformed topic +// - topic with invalid spork ID +func invalidTopics(t *testing.T, sporkID flow.Identifier) (string, string, string) { + // create unknown topic + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() + // create malformed topic + malformedTopic := channels.Topic(unittest.RandomStringFixture(t, 100)).String() + // a topics spork ID is considered invalid if it does not match the current spork ID + invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() + return unknownTopic, malformedTopic, invalidSporkIDTopic +} + +// checkNotificationFunc returns util func used to ensure invalid control message notification disseminated contains expected information. +func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType p2pmsg.ControlMessageType, isExpectedErr func(err error) bool, topicType p2p.CtrlMsgTopicType) func(args mock.Arguments) { + return func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, topicType, notification.TopicType) + require.Equal(t, expectedPeerID, notification.PeerID) + require.Equal(t, expectedMsgType, notification.MsgType) + require.True(t, isExpectedErr(notification.Error)) + } } -// inspectorFixture returns a *ControlMsgValidationInspector fixture. -func inspectorFixture(t *testing.T, opts ...func(*validation.InspectorParams)) (*validation.ControlMsgValidationInspector, *mockp2p.GossipSubInspectorNotificationDistributor, *mockp2p.RpcControlTracking, *mockmodule.IdentityProvider, flow.Identifier, *p2pconf.GossipSubRPCValidationInspectorConfigs) { +func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorParams)) (*validation.ControlMsgValidationInspector, *irrecoverable.MockSignalerContext, context.CancelFunc, *mockp2p.GossipSubInspectorNotificationDistributor, *mockp2p.RpcControlTracking, flow.Identifier, *mockmodule.IdentityProvider, *validation.InspectorParams) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() - require.NoError(t, err, "failed to get default flow config") + require.NoError(t, err) distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) p2ptest.MockInspectorNotificationDistributorReadyDoneAware(distributor) idProvider := mockmodule.NewIdentityProvider(t) @@ -890,33 +874,18 @@ func inspectorFixture(t *testing.T, opts ...func(*validation.InspectorParams)) ( for _, opt := range opts { opt(params) } - inspector, err := validation.NewControlMsgValidationInspector(params) + validationInspector, err := validation.NewControlMsgValidationInspector(params) require.NoError(t, err, "failed to create control message validation inspector fixture") - return inspector, distributor, rpcTracker, idProvider, sporkID, &flowConfig.NetworkConfig.GossipSubRPCValidationInspectorConfigs + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + return validationInspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, idProvider, params } -// invalidTopics returns 3 invalid topics. -// - unknown topic -// - malformed topic -// - topic with invalid spork ID -func invalidTopics(t *testing.T, sporkID flow.Identifier) (string, string, string) { - // create unknown topic - unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() - // create malformed topic - malformedTopic := channels.Topic(unittest.RandomStringFixture(t, 100)).String() - // a topics spork ID is considered invalid if it does not match the current spork ID - invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() - return unknownTopic, malformedTopic, invalidSporkIDTopic +func stopInspector(t *testing.T, cancel context.CancelFunc, inspector *validation.ControlMsgValidationInspector) { + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 500*time.Millisecond, "inspector did not stop") } -// checkNotificationFunc returns util func used to ensure invalid control message notification disseminated contains expected information. -func checkNotificationFunc(t *testing.T, expectedPeerID peer.ID, expectedMsgType p2pmsg.ControlMessageType, isExpectedErr func(err error) bool, isClusterPrefixed bool) func(args mock.Arguments) { - return func(args mock.Arguments) { - notification, ok := args[0].(*p2p.InvCtrlMsgNotif) - require.True(t, ok) - require.Equal(t, isClusterPrefixed, notification.IsClusterPrefixed) - require.Equal(t, expectedPeerID, notification.PeerID) - require.Equal(t, expectedMsgType, notification.MsgType) - require.True(t, isExpectedErr(notification.Error)) - } +func defaultTopicOracle() []string { + return []string{} } diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 4aafcfb335b..e0745f46471 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -284,7 +284,7 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( } // reduce penalty for cluster prefixed topics allowing nodes that are potentially behind to catch up - if notification.IsClusterPrefixed { + if notification.TopicType == p2p.CtrlMsgTopicTypeClusterPrefixed { penalty *= r.penalty.ClusterPrefixedPenaltyReductionFactor } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 8da04f65fc9..8c11fc6b53b 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2620,7 +2620,7 @@ func P2PRPCPruneFixture(topic *string) *pubsub_pb.ControlPrune { } } -// P2PRPCIHaveFixtures returns n number of control message rpc iHave fixtures with m number of message ids each. +// P2PRPCIHaveFixtures returns n number of control message where n = len(topics) rpc iHave fixtures with m number of message ids each. func P2PRPCIHaveFixtures(m int, topics ...string) []*pubsub_pb.ControlIHave { n := len(topics) ihaves := make([]*pubsub_pb.ControlIHave, n) From 1749d789e61dff2384e7ad8ef332a9bda642bd01 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 28 Nov 2023 23:17:55 -0500 Subject: [PATCH 67/70] update cluster prefixed penalty calculations test to encompass all control message types --- network/p2p/scoring/registry_test.go | 125 ++++++++++++++++----------- 1 file changed, 75 insertions(+), 50 deletions(-) diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 51ba13057f3..e1b3170940b 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -428,60 +428,66 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { // the application-specific penalty should be reduced by the default reduction factor. This test verifies the accurate computation // of the application-specific score under these conditions. func TestPeerSpamPenaltyClusterPrefixed(t *testing.T) { - peerID := unittest.PeerIdFixture(t) - reg, spamRecords := newGossipSubAppSpecificScoreRegistry( - t, - withStakedIdentity(peerID), - withValidSubscriptions(peerID)) - - // initially, the spamRecords should not have the peer id. - assert.False(t, spamRecords.Has(peerID)) - - // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which - // is the default reward for a staked peer that has valid subscriptions. - score := reg.AppSpecificScoreFunc()(peerID) - assert.Equal(t, scoring.MaxAppSpecificReward, score) + ctlMsgTypes := p2pmsg.ControlMessageTypes() + peerIds := unittest.PeerIdFixtures(t, len(ctlMsgTypes)) + opts := make([]func(*scoring.GossipSubAppSpecificScoreRegistryConfig), 0) + for _, peerID := range peerIds { + opts = append(opts, withStakedIdentity(peerID), withValidSubscriptions(peerID)) + } + reg, spamRecords := newGossipSubAppSpecificScoreRegistry(t, opts...) + + for _, peerID := range peerIds { + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + score := reg.AppSpecificScoreFunc()(peerID) + assert.Equal(t, scoring.MaxAppSpecificReward, score) + } // Report consecutive misbehavior's for the specified peer ID. Two misbehavior's are reported concurrently: // 1. With IsClusterPrefixed set to false, ensuring the penalty applied to the application-specific score is not reduced. // 2. With IsClusterPrefixed set to true, reducing the penalty added to the overall app-specific score by the default reduction factor. - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ - PeerID: peerID, - MsgType: p2pmsg.CtrlMsgGraft, - IsClusterPrefixed: false, - }) - }() - go func() { - defer wg.Done() - reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ - PeerID: peerID, - MsgType: p2pmsg.CtrlMsgGraft, - IsClusterPrefixed: true, - }) - }() - unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") - - // expected penalty should be penaltyValueFixtures().Graft * (1 + clusterReductionFactor) - expectedPenalty := penaltyValueFixtures().Graft * (1 + penaltyValueFixtures().ClusterPrefixedPenaltyReductionFactor) - - // the penalty should now be updated in the spamRecords - record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. - assert.True(t, ok) - assert.NoError(t, err) - assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) - assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) - // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, - // and the peer should be deprived of the default reward for its valid staked role. - score = reg.AppSpecificScoreFunc()(peerID) - tolerance := 10e-3 // 0.1% - if expectedPenalty == 0 { - assert.Less(t, math.Abs(expectedPenalty), tolerance) - } else { - assert.Less(t, math.Abs(expectedPenalty-score)/expectedPenalty, tolerance) + for i, ctlMsgType := range ctlMsgTypes { + peerID := peerIds[i] + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: ctlMsgType, + TopicType: p2p.CtrlMsgNonClusterTopicType, + }) + }() + go func() { + defer wg.Done() + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: ctlMsgType, + TopicType: p2p.CtrlMsgTopicTypeClusterPrefixed, + }) + }() + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // expected penalty should be penaltyValueFixtures().Graft * (1 + clusterReductionFactor) + expectedPenalty := penaltyValueFixture(ctlMsgType) * (1 + penaltyValueFixtures().ClusterPrefixedPenaltyReductionFactor) + + // the penalty should now be updated in the spamRecords + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) + // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, + // and the peer should be deprived of the default reward for its valid staked role. + score := reg.AppSpecificScoreFunc()(peerID) + tolerance := 10e-3 // 0.1% + if expectedPenalty == 0 { + assert.Less(t, math.Abs(expectedPenalty), tolerance) + } else { + assert.Less(t, math.Abs(expectedPenalty-score)/expectedPenalty, tolerance) + } } } @@ -556,3 +562,22 @@ func penaltyValueFixtures() scoring.GossipSubCtrlMsgPenaltyValue { RpcPublishMessage: -10, } } + +// penaltyValueFixture returns the set penalty of the provided control message type returned from the fixture func penaltyValueFixtures. +func penaltyValueFixture(msgType p2pmsg.ControlMessageType) float64 { + penaltyValues := penaltyValueFixtures() + switch msgType { + case p2pmsg.CtrlMsgGraft: + return penaltyValues.Graft + case p2pmsg.CtrlMsgPrune: + return penaltyValues.Prune + case p2pmsg.CtrlMsgIHave: + return penaltyValues.IHave + case p2pmsg.CtrlMsgIWant: + return penaltyValues.IWant + case p2pmsg.RpcPublishMessage: + return penaltyValues.RpcPublishMessage + default: + return penaltyValues.ClusterPrefixedPenaltyReductionFactor + } +} From 43c323824b656c2b49065b42a769cd8f1d16bfcd Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 29 Nov 2023 00:06:23 -0500 Subject: [PATCH 68/70] Update control_message_validation_inspector_test.go --- .../validation/control_message_validation_inspector_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 05ad1beef02..79473adbafa 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -9,6 +9,9 @@ import ( pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" @@ -23,8 +26,6 @@ import ( mockp2p "github.com/onflow/flow-go/network/p2p/mock" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" ) func TestNewControlMsgValidationInspector(t *testing.T) { From 54e3bbb35281d3a1e7f3e68acb2572746d7e33b8 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 29 Nov 2023 01:38:29 -0500 Subject: [PATCH 69/70] Update control_message_validation_inspector_test.go --- .../validation/control_message_validation_inspector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index 79473adbafa..1040aacba44 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -11,7 +11,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" From d3c606860ff45c2c5f84553971db416c7dedb7d8 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 29 Nov 2023 01:47:51 -0500 Subject: [PATCH 70/70] Update validation_inspector_test.go --- .../rpc_inspector/validation_inspector_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 859dcfe37bb..f24c2472534 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -62,7 +62,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, channels.IsInvalidTopicErr(notification.Error)) switch notification.MsgType { @@ -197,7 +197,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.True(t, validation.IsDuplicateTopicErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) switch notification.MsgType { @@ -304,7 +304,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.True(t, validation.IsDuplicateTopicErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, @@ -416,7 +416,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { count.Inc() notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.True(t, notification.IsClusterPrefixed) + require.Equal(t, notification.TopicType, p2p.CtrlMsgTopicTypeClusterPrefixed) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, channels.IsUnknownClusterIDErr(notification.Error)) switch notification.MsgType { @@ -798,7 +798,7 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { return func(args mockery.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.CtrlMsgIWant, @@ -934,7 +934,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { return func(args mockery.Arguments) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) - require.False(t, notification.IsClusterPrefixed, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.RpcPublishMessage,