From c8807064416aff79a7c1586313280d302b4ac85f Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 24 Jan 2025 12:42:05 +0200 Subject: [PATCH 01/11] integration-tests/smoke/ccip: add re-org tests --- .github/e2e-tests.yml | 14 ++ .../changeset/testhelpers/test_environment.go | 21 +++ .../smoke/ccip/ccip_reorg_test.go | 168 ++++++++++++++++++ .../testsetups/ccip/test_helpers.go | 41 +++-- 4 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 integration-tests/smoke/ccip/ccip_reorg_test.go diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index e5c8b8505f3..fe9e5b39dc1 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -937,6 +937,20 @@ runner-test-matrix: # START: CCIPv1.6 tests + - id: smoke/ccip/ccip_reorg_test.go:* + path: integration-tests/smoke/ccip/ccip_reorg_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + CCIP_V16_TEST_ENV: docker + - id: smoke/ccip/ccip_token_price_updates_test.go:* path: integration-tests/smoke/ccip/ccip_token_price_updates_test.go test_env_type: docker diff --git a/deployment/ccip/changeset/testhelpers/test_environment.go b/deployment/ccip/changeset/testhelpers/test_environment.go index ca58a067667..13e25f9813f 100644 --- a/deployment/ccip/changeset/testhelpers/test_environment.go +++ b/deployment/ccip/changeset/testhelpers/test_environment.go @@ -39,6 +39,12 @@ const ( ENVTESTTYPE = "CCIP_V16_TEST_ENV" ) +type LogMessageToIgnore struct { + Msg string + Reason string + Level zapcore.Level +} + type TestConfigs struct { Type EnvType // set by env var CCIP_V16_TEST_ENV, defaults to Memory CreateJob bool @@ -59,6 +65,14 @@ type TestConfigs struct { NumOfRMNNodes int LinkPrice *big.Int WethPrice *big.Int + + // Test env related configs + + // LogMessagesToIgnore are log messages emitted by the chainlink node that cause + // the test to auto-fail if they were logged. + // In some tests we don't want this to happen where a failure is expected, e.g + // we are purposely re-orging beyond finality. + LogMessagesToIgnore []LogMessageToIgnore } func (tc *TestConfigs) Validate() error { @@ -102,6 +116,12 @@ func DefaultTestConfigs() *TestConfigs { type TestOps func(testCfg *TestConfigs) +func WithLogMessagesToIgnore(logMessages []LogMessageToIgnore) TestOps { + return func(testCfg *TestConfigs) { + testCfg.LogMessagesToIgnore = logMessages + } +} + func WithMultiCall3() TestOps { return func(testCfg *TestConfigs) { testCfg.IsMultiCall3 = true @@ -417,6 +437,7 @@ func NewEnvironmentWithJobsAndContracts(t *testing.T, tEnv TestEnvironment) Depl tc := tEnv.TestConfigs() e := NewEnvironment(t, tEnv) allChains := e.Env.AllChainSelectors() + t.Log("number of chains:", len(allChains)) mcmsCfg := make(map[uint64]commontypes.MCMSWithTimelockConfig) for _, c := range e.Env.AllChainSelectors() { diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go new file mode 100644 index 00000000000..b25600314a6 --- /dev/null +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -0,0 +1,168 @@ +package ccip + +import ( + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + chainsel "github.com/smartcontractkit/chain-selectors" + ctf_client "github.com/smartcontractkit/chainlink-testing-framework/lib/client" + "github.com/smartcontractkit/chainlink/deployment" + ccipcs "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + "github.com/smartcontractkit/chainlink/deployment/ccip/changeset/testhelpers" + testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" +) + +func Test_CCIPReorgBelowFinality(t *testing.T) { +} + +func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { + t.Skip("Flakey") + // This test sends a ccip message and re-orgs the chain + // after the message block has been finalized. + // The result should be that the plugin does not process + // messages from the re-orged chain anymore. + // However, it should gracefully process messages from non-reorged chains. + require.Equal( + t, + os.Getenv(testhelpers.ENVTESTTYPE), + string(testhelpers.Docker), + "Reorg tests are only supported in docker environments", + ) + + // This test sends a ccip message and re-orgs the chain + // prior to the message block being finalized. + e, _, tEnv := testsetups.NewIntegrationEnvironment( + t, + testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ + { + Msg: "Got very old block.", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Reorg greater than finality depth detected", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + }), + ) + + nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) + require.NoError(t, err) + + var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) + for _, n := range nodeInfos.NonBootstraps() { + nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) + } + + t.Log("nonBootstrapP2PIDs:", nonBootstrapP2PIDs) + + state, err := ccipcs.LoadOnchainState(e.Env) + require.NoError(t, err) + + allChains := e.Env.AllChainSelectors() + require.GreaterOrEqual(t, len(allChains), 2) + sourceChainSelector := allChains[0] + sourceChain, ok := chainsel.ChainBySelector(sourceChainSelector) + require.True(t, ok) + headTrackerService := fmt.Sprintf("EVM.%d.HeadTracker", sourceChain.EvmChainID) + logPollerService := fmt.Sprintf("EVM.%d.LogPoller", sourceChain.EvmChainID) + t.Log("head tracker service name:", headTrackerService, "log poller service name:", logPollerService) + destChainSelector := allChains[1] + + dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) + require.True(t, ok) + + chainSelToRPCURL := make(map[uint64]string) + for _, chain := range dockerEnv.GetDevEnvConfig().Chains { + require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + require.NoError(t, err) + + chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0] + } + + sourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[sourceChainSelector], nil) + + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceChainSelector, destChainSelector, false) + + // wait for log poller filters to get registered. + time.Sleep(30 * time.Second) + msgSentEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceChainSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + t.Log("msgSentEvent:", msgSentEvent) + time.Sleep(10 * time.Second) + // expectedSeqNum := map[testhelpers.SourceDestPair]uint64{ + // { + // SourceChainSelector: sourceChainSelector, + // DestChainSelector: destChainSelector, + // }: msgSentEvent.SequenceNumber, + // } + // expectedSeqNumExec := map[testhelpers.SourceDestPair][]uint64{ + // { + // SourceChainSelector: sourceChainSelector, + // DestChainSelector: destChainSelector, + // }: {msgSentEvent.SequenceNumber}, + // } + + // Run reorg above finality depth + t.Log("starting blockchain reorg on Simulated Geth chain") + const reorgDepth = 50 + err = sourceClient.GethSetHead(reorgDepth) + require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + + nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() + t.Logf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", len(nodeAPIs)-1) + require.Eventually(t, func() bool { + violatedResponses := make(map[string]struct{}) + for _, node := range nodeAPIs { + // skip bootstrap nodes, they won't have any logpoller filters + p2pKeys, err := node.MustReadP2PKeys() + require.NoError(t, err) + + t.Log("p2pKeys:", p2pKeys) + + require.GreaterOrEqual(t, len(p2pKeys.Data), 1) + if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { + t.Log("skipping bootstrap node w/ p2p id", p2pKeys.Data[0].Attributes.PeerID) + continue + } + + resp, _, err := node.Health() + require.NoError(t, err) + var violated bool + for _, d := range resp.Data { + if d.Attributes.Name == logPollerService && + d.Attributes.Output == "finality violated" && + d.Attributes.Status == "failing" { + violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} + violated = true + break + } + } + + if violated { + t.Log("node", p2pKeys.Data[0].Attributes.PeerID, "reported finality violation") + } else { + t.Log("node", p2pKeys.Data[0].Attributes.PeerID, "did not report finality violation, full response: %+v", resp) + } + } + + t.Logf("%d nodes reported finality violation", len(violatedResponses)) + return len(violatedResponses) == (len(nodeAPIs) - 1) + }, 3*time.Minute, 5*time.Second, "not all the nodes report finality violation") + t.Log("All nodes reported finality violation") +} diff --git a/integration-tests/testsetups/ccip/test_helpers.go b/integration-tests/testsetups/ccip/test_helpers.go index 59a9bdb47ed..2b9cf1e469e 100644 --- a/integration-tests/testsetups/ccip/test_helpers.go +++ b/integration-tests/testsetups/ccip/test_helpers.go @@ -61,6 +61,14 @@ type DeployedLocalDevEnvironment struct { devEnvCfg *devenv.EnvironmentConfig } +func (l *DeployedLocalDevEnvironment) GetCLClusterTestEnv() *test_env.CLClusterTestEnv { + return l.testEnv +} + +func (l *DeployedLocalDevEnvironment) GetDevEnvConfig() devenv.EnvironmentConfig { + return *l.devEnvCfg +} + func (l *DeployedLocalDevEnvironment) DeployedEnvironment() testhelpers.DeployedEnv { return l.DeployedEnv } @@ -76,7 +84,7 @@ func (l *DeployedLocalDevEnvironment) TestConfigs() *testhelpers.TestConfigs { func (l *DeployedLocalDevEnvironment) StartChains(t *testing.T) { lggr := logger.TestLogger(t) ctx := testcontext.Get(t) - envConfig, testEnv, cfg := CreateDockerEnv(t) + envConfig, testEnv, cfg := CreateDockerEnv(t, l.GenericTCConfig) l.devEnvTestCfg = cfg l.testEnv = testEnv l.devEnvCfg = envConfig @@ -323,7 +331,7 @@ func GenerateTestRMNConfig(t *testing.T, nRMNNodes int, tenv testhelpers.Deploye // CreateDockerEnv creates a new test environment with simulated private ethereum networks and job distributor // It returns the EnvironmentConfig which holds the chain config and JD config // The test environment is then used to start chainlink nodes -func CreateDockerEnv(t *testing.T) ( +func CreateDockerEnv(t *testing.T, v1_6TestConfig *testhelpers.TestConfigs) ( *devenv.EnvironmentConfig, *test_env.CLClusterTestEnv, tc.TestConfig, @@ -347,11 +355,12 @@ func CreateDockerEnv(t *testing.T) ( } // ignore critical CL node logs until they are fixed, as otherwise tests will fail - var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( - "No live RPC nodes available", - "CL nodes are started before simulated chains, so this is expected", - zapcore.DPanicLevel, - testreporters.WarnAboutAllowedMsgs_No), + var allowedMessages = []testreporters.AllowedLogMessage{ + testreporters.NewAllowedLogMessage( + "No live RPC nodes available", + "CL nodes are started before simulated chains, so this is expected", + zapcore.DPanicLevel, + testreporters.WarnAboutAllowedMsgs_No), testreporters.NewAllowedLogMessage( "Error stopping job service", "Possible lifecycle bug in chainlink: failed to close RMN home reader: has already been stopped: already stopped", @@ -362,7 +371,18 @@ func CreateDockerEnv(t *testing.T) ( "Possible lifecycle bug in chainlink.", zapcore.DPanicLevel, testreporters.WarnAboutAllowedMsgs_No), - ) + } + if v1_6TestConfig != nil { + for _, logMsg := range v1_6TestConfig.LogMessagesToIgnore { + allowedMessages = append(allowedMessages, testreporters.NewAllowedLogMessage( + logMsg.Msg, + logMsg.Reason, + logMsg.Level, + testreporters.WarnAboutAllowedMsgs_No, + )) + } + } + var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(allowedMessages...) builder := test_env.NewCLTestEnvBuilder(). WithTestConfig(&cfg). @@ -578,10 +598,7 @@ func FundNodes(t *testing.T, lggr zerolog.Logger, env *test_env.CLClusterTestEnv if receipt == nil { return fmt.Errorf("receipt is nil") } - txHash := "(none)" - if receipt != nil { - txHash = receipt.TxHash.String() - } + txHash := receipt.TxHash.String() lggr.Info(). Str("From", fromAddress.Hex()). Str("To", toAddr.String()). From 04e729d53a9879b49bc2fe3c385d564026fb7a13 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 24 Jan 2025 13:41:56 +0200 Subject: [PATCH 02/11] hopefully not flakey --- .../nodeclient/chainlink_models.go | 22 +++-- .../smoke/ccip/ccip_reorg_test.go | 90 +++++++++++-------- 2 files changed, 66 insertions(+), 46 deletions(-) diff --git a/deployment/environment/nodeclient/chainlink_models.go b/deployment/environment/nodeclient/chainlink_models.go index 1f4bbe4ebd0..84059b7df0d 100644 --- a/deployment/environment/nodeclient/chainlink_models.go +++ b/deployment/environment/nodeclient/chainlink_models.go @@ -33,17 +33,21 @@ type ResponseSlice struct { Data []map[string]interface{} } +type HealthCheck struct { + Name string `json:"name"` + Status string `json:"status"` + Output string `json:"output"` +} + +type HealthResponseDetail struct { + Type string `json:"type"` + ID string `json:"id"` + Attributes HealthCheck `json:"attributes"` +} + // HealthResponse is the generic model for services health statuses type HealthResponse struct { - Data []struct { - Type string `json:"type"` - ID string `json:"id"` - Attributes struct { - Name string `json:"name"` - Status string `json:"status"` - Output string `json:"output"` - } `json:"attributes"` - } `json:"data"` + Data []HealthResponseDetail `json:"data"` } // Response is the generic model that can be used for all Chainlink API responses diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index b25600314a6..ed0438f7a09 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -11,25 +11,33 @@ import ( "github.com/ethereum/go-ethereum/common" chainsel "github.com/smartcontractkit/chain-selectors" ctf_client "github.com/smartcontractkit/chainlink-testing-framework/lib/client" + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink/deployment" ccipcs "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset/testhelpers" + "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" ) -func Test_CCIPReorgBelowFinality(t *testing.T) { +func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { } -func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { - t.Skip("Flakey") - // This test sends a ccip message and re-orgs the chain - // after the message block has been finalized. - // The result should be that the plugin does not process - // messages from the re-orged chain anymore. - // However, it should gracefully process messages from non-reorged chains. +func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { + +} + +func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { +} + +// This test sends a ccip message and re-orgs the chain +// after the message block has been finalized. +// The result should be that the plugin does not process +// messages from the re-orged chain anymore. +// However, it should gracefully process messages from non-reorged chains. +func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { require.Equal( t, os.Getenv(testhelpers.ENVTESTTYPE), @@ -37,6 +45,8 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { "Reorg tests are only supported in docker environments", ) + l := logging.GetTestLogger(t) + // This test sends a ccip message and re-orgs the chain // prior to the message block being finalized. e, _, tEnv := testsetups.NewIntegrationEnvironment( @@ -52,6 +62,11 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { Reason: "We are expecting a re-org beyond finality", Level: zapcore.DPanicLevel, }, + { + Msg: "Failed to poll and save logs due to finality violation, retrying later", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, }), ) @@ -63,7 +78,7 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) } - t.Log("nonBootstrapP2PIDs:", nonBootstrapP2PIDs) + l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) state, err := ccipcs.LoadOnchainState(e.Env) require.NoError(t, err) @@ -73,9 +88,8 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { sourceChainSelector := allChains[0] sourceChain, ok := chainsel.ChainBySelector(sourceChainSelector) require.True(t, ok) - headTrackerService := fmt.Sprintf("EVM.%d.HeadTracker", sourceChain.EvmChainID) logPollerService := fmt.Sprintf("EVM.%d.LogPoller", sourceChain.EvmChainID) - t.Log("head tracker service name:", headTrackerService, "log poller service name:", logPollerService) + l.Info().Msgf("log poller service name: %s", logPollerService) destChainSelector := allChains[1] dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) @@ -95,6 +109,7 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceChainSelector, destChainSelector, false) // wait for log poller filters to get registered. + l.Info().Msg("waiting for log poller filters to get registered") time.Sleep(30 * time.Second) msgSentEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceChainSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), @@ -103,29 +118,17 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, }) - t.Log("msgSentEvent:", msgSentEvent) - time.Sleep(10 * time.Second) - // expectedSeqNum := map[testhelpers.SourceDestPair]uint64{ - // { - // SourceChainSelector: sourceChainSelector, - // DestChainSelector: destChainSelector, - // }: msgSentEvent.SequenceNumber, - // } - // expectedSeqNumExec := map[testhelpers.SourceDestPair][]uint64{ - // { - // SourceChainSelector: sourceChainSelector, - // DestChainSelector: destChainSelector, - // }: {msgSentEvent.SequenceNumber}, - // } + l.Info().Msgf("sent CCIP message, msgSentEvent: %+v", msgSentEvent) // Run reorg above finality depth - t.Log("starting blockchain reorg on Simulated Geth chain") const reorgDepth = 50 + l.Info().Int("reorgDepth", reorgDepth).Msgf("starting blockchain reorg on Simulated Geth chain") err = sourceClient.GethSetHead(reorgDepth) require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() - t.Logf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", len(nodeAPIs)-1) + nonBootstrapCount := len(nodeAPIs) - 1 + l.Info().Msgf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", nonBootstrapCount) require.Eventually(t, func() bool { violatedResponses := make(map[string]struct{}) for _, node := range nodeAPIs { @@ -133,36 +136,49 @@ func Test_CCIPReorgGreaterThanFinalityOnSource(t *testing.T) { p2pKeys, err := node.MustReadP2PKeys() require.NoError(t, err) - t.Log("p2pKeys:", p2pKeys) + l.Debug().Msgf("got p2pKeys from node API: %+v", p2pKeys) require.GreaterOrEqual(t, len(p2pKeys.Data), 1) if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { - t.Log("skipping bootstrap node w/ p2p id", p2pKeys.Data[0].Attributes.PeerID) + l.Info().Msgf("skipping bootstrap node w/ p2p id %s", p2pKeys.Data[0].Attributes.PeerID) continue } resp, _, err := node.Health() require.NoError(t, err) - var violated bool for _, d := range resp.Data { if d.Attributes.Name == logPollerService && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" { violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} - violated = true break } } - if violated { - t.Log("node", p2pKeys.Data[0].Attributes.PeerID, "reported finality violation") + if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { + l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) } else { - t.Log("node", p2pKeys.Data[0].Attributes.PeerID, "did not report finality violation, full response: %+v", resp) + l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", + p2pKeys.Data[0].Attributes.PeerID, + getLogPollerHealth(logPollerService, resp.Data), + ) } } - t.Logf("%d nodes reported finality violation", len(violatedResponses)) - return len(violatedResponses) == (len(nodeAPIs) - 1) + l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) + return len(violatedResponses) == nonBootstrapCount }, 3*time.Minute, 5*time.Second, "not all the nodes report finality violation") - t.Log("All nodes reported finality violation") + l.Info().Msg("All nodes reported finality violation") + + // TODO: assert that the plugin does not commit the message? +} + +func getLogPollerHealth(logPollerService string, healthResponses []nodeclient.HealthResponseDetail) nodeclient.HealthCheck { + for _, d := range healthResponses { + if d.Attributes.Name == logPollerService { + return d.Attributes + } + } + + return nodeclient.HealthCheck{} } From 31b59574ee59f1d41065f6402bed14d7fa43ff11 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 24 Jan 2025 15:43:27 +0200 Subject: [PATCH 03/11] fixes --- integration-tests/smoke/ccip/ccip_reorg_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index ed0438f7a09..d5f98c61a9e 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -10,6 +10,9 @@ import ( "github.com/ethereum/go-ethereum/common" chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + ctf_client "github.com/smartcontractkit/chainlink-testing-framework/lib/client" "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink/deployment" @@ -18,8 +21,6 @@ import ( "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" ) func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { @@ -101,7 +102,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) require.NoError(t, err) - chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0] + chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal } sourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[sourceChainSelector], nil) From 72e04b4e9a139a8ebd23763839eb2249ca0e1173 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 24 Jan 2025 17:17:24 +0200 Subject: [PATCH 04/11] assert a commit from non-reorged chain --- .../smoke/ccip/ccip_reorg_test.go | 71 ++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index d5f98c61a9e..9b9c4aad836 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" @@ -85,13 +86,19 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { require.NoError(t, err) allChains := e.Env.AllChainSelectors() - require.GreaterOrEqual(t, len(allChains), 2) - sourceChainSelector := allChains[0] - sourceChain, ok := chainsel.ChainBySelector(sourceChainSelector) + require.GreaterOrEqual(t, len(allChains), 3) + reorgSourceSelector := allChains[0] + reorgSourceChain, ok := chainsel.ChainBySelector(reorgSourceSelector) require.True(t, ok) - logPollerService := fmt.Sprintf("EVM.%d.LogPoller", sourceChain.EvmChainID) - l.Info().Msgf("log poller service name: %s", logPollerService) - destChainSelector := allChains[1] + noreorgSourceSelector := allChains[1] + noreorgSourceChain, ok := chainsel.ChainBySelector(noreorgSourceSelector) + require.True(t, ok) + reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", reorgSourceChain.EvmChainID) + noreorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", noreorgSourceChain.EvmChainID) + l.Info(). + Msgf("reorging log poller service name: %s, no reorg log poller service name: %s", + reorgLogPollerService, noreorgLogPollerService) + destChainSelector := allChains[2] dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) require.True(t, ok) @@ -105,26 +112,39 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal } - sourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[sourceChainSelector], nil) + reorgSourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[reorgSourceSelector], nil) - testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceChainSelector, destChainSelector, false) + // setup lanes from s1 and s2 to destChainSelector + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, reorgSourceSelector, destChainSelector, false) + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, noreorgSourceSelector, destChainSelector, false) // wait for log poller filters to get registered. l.Info().Msg("waiting for log poller filters to get registered") - time.Sleep(30 * time.Second) - msgSentEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceChainSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + time.Sleep(15 * time.Second) + reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + l.Info().Msgf("sent CCIP message that will get re-orged, msg id: %x", reorgingMsgEvent.Message.Header.MessageId) + msgEvent := testhelpers.TestSendRequest(t, e.Env, state, noreorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), Data: []byte("hello world"), TokenAmounts: nil, FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, }) - l.Info().Msgf("sent CCIP message, msgSentEvent: %+v", msgSentEvent) + l.Info().Msgf("sent CCIP message that will not get re-orged, msgSentEvent: %x", msgEvent.Message.Header.MessageId) // Run reorg above finality depth const reorgDepth = 50 - l.Info().Int("reorgDepth", reorgDepth).Msgf("starting blockchain reorg on Simulated Geth chain") - err = sourceClient.GethSetHead(reorgDepth) + l.Info(). + Int("reorgDepth", reorgDepth). + Uint64("sourceChainSelector", reorgSourceSelector). + Msg("starting blockchain reorg on Simulated Geth chain") + err = reorgSourceClient.GethSetHead(reorgDepth) require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() @@ -148,7 +168,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { resp, _, err := node.Health() require.NoError(t, err) for _, d := range resp.Data { - if d.Attributes.Name == logPollerService && + if d.Attributes.Name == reorgLogPollerService && d.Attributes.Output == "finality violated" && d.Attributes.Status == "failing" { violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} @@ -161,7 +181,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { } else { l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", p2pKeys.Data[0].Attributes.PeerID, - getLogPollerHealth(logPollerService, resp.Data), + getLogPollerHealth(reorgLogPollerService, resp.Data), ) } } @@ -171,7 +191,26 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { }, 3*time.Minute, 5*time.Second, "not all the nodes report finality violation") l.Info().Msg("All nodes reported finality violation") - // TODO: assert that the plugin does not commit the message? + // expect the commit to still go through on the non-reorged source chain. + testhelpers.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[noreorgSourceSelector], + e.Env.Chains[destChainSelector], + state.Chains[destChainSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit + ) + + // Works but super slow. + // testhelpers.ConfirmExecWithSeqNrs( + // t, + // e.Env.Chains[noreorgSourceSelector], + // e.Env.Chains[destChainSelector], + // state.Chains[destChainSelector].OffRamp, + // nil, // startBlock + // []uint64{1}, // expectedSeqNrs + // ) } func getLogPollerHealth(logPollerService string, healthResponses []nodeclient.HealthResponseDetail) nodeclient.HealthCheck { From b2631b7a9da47c95a859978b08dd63c1e0c6f837 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 24 Jan 2025 20:23:00 +0200 Subject: [PATCH 05/11] spin up 3 nets --- .github/e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index fe9e5b39dc1..5f7434c10fd 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -947,7 +947,7 @@ runner-test-matrix: test_cmd: cd integration-tests/smoke/ccip && go test ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json pyroscope_env: ci-smoke-ccipv1_6-evm-simulated test_env_vars: - E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3 E2E_JD_VERSION: 0.6.0 CCIP_V16_TEST_ENV: docker From f726c391e272b475758ce7c8ddb73c293522e799 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 27 Jan 2025 15:01:05 +0200 Subject: [PATCH 06/11] less than finality test --- .github/e2e-tests.yml | 18 +- .../changeset/testhelpers/test_environment.go | 10 + .../smoke/ccip/ccip_reorg_test.go | 185 ++++++++++++++++++ integration-tests/testconfig/ccip/ccip.toml | 168 +++++++++++++++- .../ccip/ccipv1_6_reorg_below_finality.toml | 13 ++ integration-tests/testconfig/testconfig.go | 22 ++- .../testconfig/testconfig_utils.go | 8 +- .../testsetups/ccip/test_helpers.go | 4 +- 8 files changed, 420 insertions(+), 8 deletions(-) create mode 100644 integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 5f7434c10fd..bbbb2582f4a 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -937,14 +937,28 @@ runner-test-matrix: # START: CCIPv1.6 tests - - id: smoke/ccip/ccip_reorg_test.go:* + - id: smoke/ccip/ccip_reorg_test.go:LessThanFinalityTests path: integration-tests/smoke/ccip/ccip_reorg_test.go test_env_type: docker runs_on: ubuntu-latest triggers: - PR E2E Core Tests - Nightly E2E Tests - test_cmd: cd integration-tests/smoke/ccip && go test ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json + test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_BelowFinality_OnSource|Test_CCIPReorg_BelowFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY + E2E_JD_VERSION: 0.6.0 + CCIP_V16_TEST_ENV: docker + + - id: smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests + path: integration-tests/smoke/ccip/ccip_reorg_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_GreaterThanFinality_OnSource|Test_CCIPReorg_GreaterThanFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json pyroscope_env: ci-smoke-ccipv1_6-evm-simulated test_env_vars: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3 diff --git a/deployment/ccip/changeset/testhelpers/test_environment.go b/deployment/ccip/changeset/testhelpers/test_environment.go index 440d92e8196..44a3b943f3c 100644 --- a/deployment/ccip/changeset/testhelpers/test_environment.go +++ b/deployment/ccip/changeset/testhelpers/test_environment.go @@ -74,6 +74,10 @@ type TestConfigs struct { // In some tests we don't want this to happen where a failure is expected, e.g // we are purposely re-orging beyond finality. LogMessagesToIgnore []LogMessageToIgnore + + // ExtraConfigTomls contains the filenames of additional toml files to be loaded + // to potentially override default configs. + ExtraConfigTomls []string } func (tc *TestConfigs) Validate() error { @@ -123,6 +127,12 @@ func WithLogMessagesToIgnore(logMessages []LogMessageToIgnore) TestOps { } } +func WithExtraConfigTomls(extraTomls []string) TestOps { + return func(testCfg *TestConfigs) { + testCfg.ExtraConfigTomls = extraTomls + } +} + func WithMultiCall3() TestOps { return func(testCfg *TestConfigs) { testCfg.IsMultiCall3 = true diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index 9b9c4aad836..4607580b955 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -8,9 +8,12 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/onsi/gomega" chainsel "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" @@ -25,6 +28,188 @@ import ( ) func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { + require.Equal( + t, + os.Getenv(testhelpers.ENVTESTTYPE), + string(testhelpers.Docker), + "Reorg tests are only supported in docker environments", + ) + + l := logging.GetTestLogger(t) + + // This test sends a ccip message and re-orgs the chain + // prior to the message block being finalized. + e, _, tEnv := testsetups.NewIntegrationEnvironment( + t, + testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ + { + Msg: "Got very old block.", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Reorg greater than finality depth detected", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Failed to poll and save logs due to finality violation, retrying later", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + }), + testhelpers.WithExtraConfigTomls([]string{ + "ccipv1_6_reorg_below_finality.toml", + }), + ) + + nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) + require.NoError(t, err) + + var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) + for _, n := range nodeInfos.NonBootstraps() { + nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) + } + + l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) + + state, err := ccipcs.LoadOnchainState(e.Env) + require.NoError(t, err) + + allChains := e.Env.AllChainSelectors() + require.GreaterOrEqual(t, len(allChains), 2) + reorgSourceSelector := allChains[0] + reorgSourceChain, ok := chainsel.ChainBySelector(reorgSourceSelector) + require.True(t, ok) + reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", reorgSourceChain.EvmChainID) + l.Info(). + Msgf("reorging log poller service name: %s", reorgLogPollerService) + destChainSelector := allChains[1] + + dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) + require.True(t, ok) + + chainSelToRPCURL := make(map[uint64]string) + for _, chain := range dockerEnv.GetDevEnvConfig().Chains { + require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + require.NoError(t, err) + + chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal + } + + reorgSourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[reorgSourceSelector], nil) + + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, reorgSourceSelector, destChainSelector, false) + + // wait for log poller filters to get registered. + l.Info().Msg("waiting for log poller filters to get registered") + time.Sleep(15 * time.Second) + reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + l.Info().Msgf("sent CCIP message that will get re-orged before getting finalized, msg id: %x, block num: %d, block hash: %x", + reorgingMsgEvent.Message.Header.MessageId, + reorgingMsgEvent.Raw.BlockNumber, + reorgingMsgEvent.Raw.BlockHash, + ) + + const reorgDepth = 5 + var minChainBlockNumberBeforeReorg = reorgingMsgEvent.Raw.BlockNumber + reorgDepth + // let reorgDepth blocks pass by before re-orging the message so it doesn't get "lost". + // Wait for chain to progress + require.Eventually(t, func() bool { + bn, err := reorgSourceClient.BlockNumber() + require.NoError(t, err) + l.Info(). + Int64("blockNumber", bn). + Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). + Msg("Waiting for chain to progress above target block number") + return bn >= int64(minChainBlockNumberBeforeReorg) + }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") + + // Run reorg below finality depth + l.Info(). + Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). + Int("reorgDepth", reorgDepth). + Uint64("sourceChainSelector", reorgSourceSelector). + Msg("starting blockchain reorg on Simulated Geth chain") + err = reorgSourceClient.GethSetHead(reorgDepth) + require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + + bnAfterReorg, err := reorgSourceClient.BlockNumber() + require.NoError(t, err, "error getting block number after reorg") + + l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") + + // assert that the message is still in the chain + it, err := state.Chains[reorgSourceSelector].OnRamp.FilterCCIPMessageSent(&bind.FilterOpts{ + Start: 0, + Context: tests.Context(t), + }, []uint64{destChainSelector}, []uint64{reorgingMsgEvent.Message.Header.SequenceNumber}) + require.NoError(t, err) + require.True(t, it.Next(), "message not found in the chain") + require.Equal(t, reorgingMsgEvent.Message.Header.MessageId, it.Event.Message.Header.MessageId) + l. + Info(). + Str("blockHashAfterReorg", it.Event.Raw.BlockHash.String()). + Str("blockHashBeforeReorg", reorgingMsgEvent.Raw.BlockHash.String()). + Msg("message blockhash after re-org") + + nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() + nonBootstrapCount := len(nodeAPIs) - 1 + l.Info().Msgf("waiting for %d non-bootstrap nodes to NOT report finality violation on the logpoller, since re-org is less than finality", nonBootstrapCount) + gomega.NewWithT(t).Consistently(func() bool { + violatedResponses := make(map[string]struct{}) + for _, node := range nodeAPIs { + // skip bootstrap nodes, they won't have any logpoller filters + p2pKeys, err := node.MustReadP2PKeys() + require.NoError(t, err) + + require.GreaterOrEqual(t, len(p2pKeys.Data), 1) + if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { + continue + } + + resp, _, err := node.Health() + require.NoError(t, err) + for _, d := range resp.Data { + if d.Attributes.Name == reorgLogPollerService && + d.Attributes.Output == "finality violated" && + d.Attributes.Status == "failing" { + violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} + break + } + } + + if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { + l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) + } else { + l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", + p2pKeys.Data[0].Attributes.PeerID, + getLogPollerHealth(reorgLogPollerService, resp.Data), + ) + } + } + + l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) + return len(violatedResponses) == 0 + }, time.Minute, 10*time.Second).Should(gomega.BeTrue()) + + // expect the commit to still go through on the non-reorged source chain. + testhelpers.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[reorgSourceSelector], + e.Env.Chains[destChainSelector], + state.Chains[destChainSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit + ) } func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 13188e6a134..c1f7b0f1c11 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -83,6 +83,84 @@ evm_supports_eip1559 = true evm_default_gas_limit = 6000000 evm_finality_depth = 1 +[Network.EVMNetworks.SIMULATED_1_DEEPER_FINALITY] +evm_name = 'chain-1337' +evm_chain_id = 1337 +evm_keys = [ + "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + "7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + "8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + "92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + "4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + "dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + "2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", + "f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897" +] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 50000 +evm_transaction_timeout = '2m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[Network.EVMNetworks.SIMULATED_2_DEEPER_FINALITY] +evm_name = 'chain-2337' +evm_chain_id = 2337 +evm_keys = [ + "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + "7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + "8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + "92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + "4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + "dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + "2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", + "f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897" +] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 50000 +evm_transaction_timeout = '2m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + +[Network.EVMNetworks.SIMULATED_3_DEEPER_FINALITY] +evm_name = 'chain-3337' +evm_chain_id = 3337 +evm_keys = [ + "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + "7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + "8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + "92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + "4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + "dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + "2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", + "f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897" +] +evm_simulated = true +client_implementation = 'Ethereum' +evm_chainlink_transaction_limit = 50000 +evm_transaction_timeout = '2m' +evm_minimum_confirmations = 1 +evm_gas_estimation_buffer = 1000 +evm_supports_eip1559 = true +evm_default_gas_limit = 6000000 +evm_finality_depth = 10 + [NodeConfig] BaseConfigTOML = """ [Feature] @@ -237,6 +315,94 @@ addresses_to_fund = [ "0xBcd4042DE499D14e55001CcbB24a551F3b954096" ] +[CCIP.PrivateEthereumNetworks.SIMULATED_1_DEEPER_FINALITY] +# either eth1 or eth2 (for post-Merge); for eth2 Prysm is used for consensus layer. +ethereum_version = "eth1" +# geth, besu, erigon or nethermind +execution_layer = "geth" +# eth2-only, if set to true environment startup will wait until at least 1 epoch has been finalised +wait_for_finalization=false + +[CCIP.PrivateEthereumNetworks.SIMULATED_1_DEEPER_FINALITY.EthereumChainConfig] +# eth2-only, the lower the value the faster the block production (3 is minimum) +seconds_per_slot = 3 +# eth2-only, the lower the value the faster the epoch finalisation (2 is minimum) +slots_per_epoch = 2 +# eht2-only, the lower tha value the faster the chain starts (10 is minimum) +genesis_delay = 15 +# eth2-only, number of validators +validator_count = 4 +chain_id = 1337 +# address that should be founded in genesis wih ETH +addresses_to_fund = [ + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", + "0x90f79bf6eb2c4f870365e785982e1f101e93b906", + "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", + "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", + "0x976ea74026e726554db657fa54763abd0c3a0aa9", + "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", + "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", + "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", + "0xBcd4042DE499D14e55001CcbB24a551F3b954096" +] + + +#[CCIP.Env.PrivateEthereumNetworks.SIMULATED_1_DEEPER_THAN_FINALITY.CustomDockerImages] +# custom docker image that will be used for execution layer client. It has to be one of: hyperledger/besu, nethermind/nethermind, thorax/erigon or ethereum/client-go. +# instead of using a specific tag you can also use "latest_available" to use latest published tag in Github or "latest_stable" to use latest stable release from Github +# (if corresponding Docker image on Docker Hub has not been published environment creation will fail). +#execution_layer="hyperledger/besu:latest_stable" + +[CCIP.PrivateEthereumNetworks.SIMULATED_2_DEEPER_FINALITY] +ethereum_version = "eth1" +execution_layer = "geth" + +[CCIP.PrivateEthereumNetworks.SIMULATED_2_DEEPER_FINALITY.EthereumChainConfig] +seconds_per_slot = 3 +slots_per_epoch = 2 +genesis_delay = 15 +validator_count = 4 +chain_id = 2337 +addresses_to_fund = [ + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", + "0x90f79bf6eb2c4f870365e785982e1f101e93b906", + "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", + "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", + "0x976ea74026e726554db657fa54763abd0c3a0aa9", + "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", + "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", + "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", + "0xBcd4042DE499D14e55001CcbB24a551F3b954096" +] + +[CCIP.PrivateEthereumNetworks.SIMULATED_3_DEEPER_FINALITY] +ethereum_version = "eth1" +execution_layer = "geth" + +[CCIP.PrivateEthereumNetworks.SIMULATED_3_DEEPER_FINALITY.EthereumChainConfig] +seconds_per_slot = 3 +slots_per_epoch = 2 +genesis_delay = 15 +validator_count = 4 +chain_id = 3337 +addresses_to_fund = [ + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "0x70997970C51812dc3A010C7d01b50e0d17dc79C8", + "0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc", + "0x90f79bf6eb2c4f870365e785982e1f101e93b906", + "0x15d34aaf54267db7d7c367839aaf71a00a2c6a65", + "0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc", + "0x976ea74026e726554db657fa54763abd0c3a0aa9", + "0x14dc79964da2c08b23698b3d3cc7ca32193d9955", + "0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f", + "0xa0ee7a142d267c1f36714e4a8f75612f20a79720", + "0xBcd4042DE499D14e55001CcbB24a551F3b954096" +] + [Seth] # Seth specific configuration, no need for generating ephemeral addresses for ccip-tests. ephemeral_addresses_number = 0 @@ -254,4 +420,4 @@ TestDuration = "5h" # destination chain selectors to send messages to EnabledDestionationChains = [3379446385462418246] # Directory where we receive environment configuration from crib -CribEnvDirectory = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" \ No newline at end of file +CribEnvDirectory = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" diff --git a/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml b/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml new file mode 100644 index 00000000000..6cf0b8809d8 --- /dev/null +++ b/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml @@ -0,0 +1,13 @@ +[NodeConfig.ChainConfigTOMLByChainID] +# Chain-specific EVMNode config TOML for chainlink nodes; applicable to all EVM node configs in chainlink TOML. It takes precedence +# over CommonChainConfigTOML and Chainlink Node's defaults. Passing blockchain nodes URLs here will have no effect. +#! FinalityDepth overridden to 10. +1337 = """ +AutoCreateKey = true +FinalityDepth = 10 +MinContractPayment = 0 +[GasEstimator] +PriceMax = '200 gwei' +LimitDefault = 6000000 +FeeCapDefault = '200 gwei' +""" diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 3ec61f42607..cd56bb58741 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -293,7 +293,22 @@ const ( Base64OverrideEnvVarName = k8s_config.EnvBase64ConfigOverride ) -func GetConfig(configurationNames []string, product Product) (TestConfig, error) { +// GetConfig returns a TestConfig struct with the given configuration names +// and product. It reads the configuration from the default.toml, .toml, +// and overrides.toml files. It also reads the configuration from the +// environment variables. +// If extraFileNames are provided, it will read the configuration from those files +// as well. +// If the Base64OverrideEnvVarName environment variable is set, it will override +// the configuration with the base64 encoded TOML config. +// If the configuration is embedded, it will read the configuration from the +// embedded files. +// If the configuration is not embedded, it will read the configuration from +// the file system. +// If the configuration is not found, it will return an error. +// If the configuration is found, it will validate the configuration and return +// the TestConfig struct. +func GetConfig(configurationNames []string, product Product, extraFileNames ...string) (TestConfig, error) { logger := logging.GetTestLogger(nil) for idx, configurationName := range configurationNames { @@ -313,6 +328,9 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) fmt.Sprintf("%s.toml", product), "overrides.toml", } + // add extra file names to the list + // no-op if nothing is provided. + fileNames = append(fileNames, extraFileNames...) testConfig := TestConfig{} testConfig.ConfigurationNames = configurationNames @@ -352,7 +370,7 @@ func GetConfig(configurationNames []string, product Product) (TestConfig, error) } else if err != nil { return TestConfig{}, errors.Wrapf(err, "error looking for file %s", filePath) } - logger.Debug().Str("location", filePath).Msgf("Found config file %s", fileName) + logger.Info().Str("location", filePath).Msgf("Found config file %s", fileName) content, err := readFile(filePath) if err != nil { diff --git a/integration-tests/testconfig/testconfig_utils.go b/integration-tests/testconfig/testconfig_utils.go index 8d41ed55be9..709e1971235 100644 --- a/integration-tests/testconfig/testconfig_utils.go +++ b/integration-tests/testconfig/testconfig_utils.go @@ -57,8 +57,11 @@ selected_networks=` return fmt.Errorf("%s\n%s", errStr, intro) } -func GetChainAndTestTypeSpecificConfig(testType string, product Product) (TestConfig, error) { - config, err := GetConfig([]string{testType}, product) +// GetChainAndTestTypeSpecificConfig returns a TestConfig with the chain and test type specific configuration. +// extraFileNames are optional and can be used to specify additional config files to load +// in order to override certain config values (e.g NodeConfig.ChainConfigTOMLByChainID). +func GetChainAndTestTypeSpecificConfig(testType string, product Product, extraFileNames ...string) (TestConfig, error) { + config, err := GetConfig([]string{testType}, product, extraFileNames...) if err != nil { return TestConfig{}, fmt.Errorf("error getting config: %w", err) } @@ -69,6 +72,7 @@ func GetChainAndTestTypeSpecificConfig(testType string, product Product) (TestCo fmt.Sprintf("%s-%s", config.GetNetworkConfig().SelectedNetworks[0], testType), }, product, + extraFileNames..., ) if err != nil { return TestConfig{}, fmt.Errorf("error getting config: %w", err) diff --git a/integration-tests/testsetups/ccip/test_helpers.go b/integration-tests/testsetups/ccip/test_helpers.go index e15e9d3bd7a..33b45a22ede 100644 --- a/integration-tests/testsetups/ccip/test_helpers.go +++ b/integration-tests/testsetups/ccip/test_helpers.go @@ -340,7 +340,9 @@ func CreateDockerEnv(t *testing.T, v1_6TestConfig *testhelpers.TestConfigs) ( require.NoError(t, gotenv.Load(".env"), "Error loading .env file") } - cfg, err := tc.GetChainAndTestTypeSpecificConfig("Smoke", tc.CCIP) + t.Logf("extra config tomls: %+v", v1_6TestConfig.ExtraConfigTomls) + + cfg, err := tc.GetChainAndTestTypeSpecificConfig("Smoke", tc.CCIP, v1_6TestConfig.ExtraConfigTomls...) require.NoError(t, err, "Error getting config") evmNetworks := networks.MustGetSelectedNetworkConfig(cfg.GetNetworkConfig()) From c53e07aed0f709ccfb492e0c7f356c153c49464a Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 27 Jan 2025 17:32:01 +0200 Subject: [PATCH 07/11] dest lower than finality test --- .../smoke/ccip/ccip_reorg_test.go | 196 +++++++++++++++--- .../Test_CCIPReorg_BelowFinality_OnDest.toml | 11 + ...Test_CCIPReorg_BelowFinality_OnSource.toml | 11 + .../ccip/ccipv1_6_reorg_below_finality.toml | 13 -- 4 files changed, 194 insertions(+), 37 deletions(-) create mode 100644 integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml create mode 100644 integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml delete mode 100644 integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index 4607580b955..aeeab2bdfb7 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -8,12 +8,11 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/onsi/gomega" chainsel "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" @@ -59,7 +58,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { }, }), testhelpers.WithExtraConfigTomls([]string{ - "ccipv1_6_reorg_below_finality.toml", + fmt.Sprintf("%s.toml", t.Name()), }), ) @@ -112,16 +111,17 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, }) - l.Info().Msgf("sent CCIP message that will get re-orged before getting finalized, msg id: %x, block num: %d, block hash: %x", - reorgingMsgEvent.Message.Header.MessageId, - reorgingMsgEvent.Raw.BlockNumber, - reorgingMsgEvent.Raw.BlockHash, - ) + l. + Info(). + Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). + Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). + Str("messageBlockHashBeforeReorg", reorgingMsgEvent.Raw.BlockHash.String()). + Msg("sent CCIP message that will get re-orged before getting finalized") - const reorgDepth = 5 - var minChainBlockNumberBeforeReorg = reorgingMsgEvent.Raw.BlockNumber + reorgDepth - // let reorgDepth blocks pass by before re-orging the message so it doesn't get "lost". - // Wait for chain to progress + const reorgDepth = 7 + var minChainBlockNumberBeforeReorg = reorgingMsgEvent.Raw.BlockNumber + reorgDepth - 1 + // let reorgDepth - 1 blocks pass by before re-orging the message. + // This will effectively rewind the chain to a block where the message didn't exist. require.Eventually(t, func() bool { bn, err := reorgSourceClient.BlockNumber() require.NoError(t, err) @@ -146,19 +146,20 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") - // assert that the message is still in the chain - it, err := state.Chains[reorgSourceSelector].OnRamp.FilterCCIPMessageSent(&bind.FilterOpts{ - Start: 0, - Context: tests.Context(t), - }, []uint64{destChainSelector}, []uint64{reorgingMsgEvent.Message.Header.SequenceNumber}) - require.NoError(t, err) - require.True(t, it.Next(), "message not found in the chain") - require.Equal(t, reorgingMsgEvent.Message.Header.MessageId, it.Event.Message.Header.MessageId) + reorgingMsgEvent = testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) l. Info(). - Str("blockHashAfterReorg", it.Event.Raw.BlockHash.String()). - Str("blockHashBeforeReorg", reorgingMsgEvent.Raw.BlockHash.String()). - Msg("message blockhash after re-org") + Str("blockHashAfterReorg", reorgingMsgEvent.Raw.BlockHash.String()). + Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). + Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). + Str("messageBlockHash", reorgingMsgEvent.Raw.BlockHash.String()). + Msgf("re-sent CCIP message after the re-org") nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() nonBootstrapCount := len(nodeAPIs) - 1 @@ -213,7 +214,154 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { } func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { + require.Equal( + t, + os.Getenv(testhelpers.ENVTESTTYPE), + string(testhelpers.Docker), + "Reorg tests are only supported in docker environments", + ) + + l := logging.GetTestLogger(t) + + // This test sends a ccip message and re-orgs the chain + // prior to the message block being finalized. + e, _, tEnv := testsetups.NewIntegrationEnvironment( + t, + testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ + { + Msg: "Got very old block.", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Reorg greater than finality depth detected", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Failed to poll and save logs due to finality violation, retrying later", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + }), + testhelpers.WithExtraConfigTomls([]string{ + fmt.Sprintf("%s.toml", t.Name()), + }), + ) + + nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) + require.NoError(t, err) + + var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) + for _, n := range nodeInfos.NonBootstraps() { + nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) + } + + l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) + + state, err := ccipcs.LoadOnchainState(e.Env) + require.NoError(t, err) + + allChains := e.Env.AllChainSelectors() + require.GreaterOrEqual(t, len(allChains), 2) + sourceSelector := allChains[0] + destSelector := allChains[1] + destChain, ok := chainsel.ChainBySelector(destSelector) + require.True(t, ok) + reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", destChain.EvmChainID) + l.Info(). + Msgf("reorging log poller service name: %s", reorgLogPollerService) + dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) + require.True(t, ok) + + chainSelToRPCURL := make(map[uint64]string) + for _, chain := range dockerEnv.GetDevEnvConfig().Chains { + require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + require.NoError(t, err) + + chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal + } + + reorgDestClient := ctf_client.NewRPCClient(chainSelToRPCURL[destSelector], nil) + + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destSelector, false) + + // wait for log poller filters to get registered. + l.Info().Msg("waiting for log poller filters to get registered") + time.Sleep(15 * time.Second) + reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceSelector, destSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + l. + Info(). + Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). + Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). + Str("messageBlockHash", reorgingMsgEvent.Raw.BlockHash.String()). + Msgf("sent CCIP message that whose commit report will re-org on dest before getting finalized") + + // expect the commit to still go through for the message. + reportEvent, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceSelector], + e.Env.Chains[destSelector], + state.Chains[destSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit + ) + require.NoError(t, err) + + l. + Info(). + Uint64("reportBlockNumber", reportEvent.Raw.BlockNumber). + Str("reportBlockHash", reportEvent.Raw.BlockHash.String()). + Msg("got commit report on dest, preparing to re-org it") + + // re-org the dest chain less than finality blocks. + const reorgDepth = 7 + var minChainBlockNumberBeforeReorg = reportEvent.Raw.BlockNumber + reorgDepth - 1 + // Wait for chain to progress + require.Eventually(t, func() bool { + bn, err := reorgDestClient.BlockNumber() + require.NoError(t, err) + l.Info(). + Int64("blockNumber", bn). + Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). + Msg("Waiting for chain to progress above target block number") + return bn >= int64(minChainBlockNumberBeforeReorg) + }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") + + // Run reorg below finality depth + l.Info(). + Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). + Int("reorgDepth", reorgDepth). + Uint64("sourceChainSelector", sourceSelector). + Msg("starting blockchain reorg on Simulated Geth chain") + err = reorgDestClient.GethSetHead(reorgDepth) + require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + + bnAfterReorg, err := reorgDestClient.BlockNumber() + require.NoError(t, err, "error getting block number after reorg") + + l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") + + // commit should be re-submitted after the re-org + reportEvent, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( + t, + e.Env.Chains[sourceSelector], + e.Env.Chains[destSelector], + state.Chains[destSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit + ) + require.NoError(t, err) } func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { @@ -235,7 +383,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { l := logging.GetTestLogger(t) // This test sends a ccip message and re-orgs the chain - // prior to the message block being finalized. + // after the message block is finalized. e, _, tEnv := testsetups.NewIntegrationEnvironment( t, testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ diff --git a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml new file mode 100644 index 00000000000..8dc6d2211ae --- /dev/null +++ b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml @@ -0,0 +1,11 @@ +[NodeConfig.ChainConfigTOMLByChainID] +#! FinalityDepth on 2337 overridden to 10. +2337 = """ +AutoCreateKey = true +FinalityDepth = 10 +MinContractPayment = 0 +[GasEstimator] +PriceMax = '200 gwei' +LimitDefault = 6000000 +FeeCapDefault = '200 gwei' +""" diff --git a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml new file mode 100644 index 00000000000..5798e7236d5 --- /dev/null +++ b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml @@ -0,0 +1,11 @@ +[NodeConfig.ChainConfigTOMLByChainID] +#! FinalityDepth on 1337 overridden to 10. +1337 = """ +AutoCreateKey = true +FinalityDepth = 10 +MinContractPayment = 0 +[GasEstimator] +PriceMax = '200 gwei' +LimitDefault = 6000000 +FeeCapDefault = '200 gwei' +""" diff --git a/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml b/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml deleted file mode 100644 index 6cf0b8809d8..00000000000 --- a/integration-tests/testconfig/ccip/ccipv1_6_reorg_below_finality.toml +++ /dev/null @@ -1,13 +0,0 @@ -[NodeConfig.ChainConfigTOMLByChainID] -# Chain-specific EVMNode config TOML for chainlink nodes; applicable to all EVM node configs in chainlink TOML. It takes precedence -# over CommonChainConfigTOML and Chainlink Node's defaults. Passing blockchain nodes URLs here will have no effect. -#! FinalityDepth overridden to 10. -1337 = """ -AutoCreateKey = true -FinalityDepth = 10 -MinContractPayment = 0 -[GasEstimator] -PriceMax = '200 gwei' -LimitDefault = 6000000 -FeeCapDefault = '200 gwei' -""" From 389bc1d0592cbfddfb8304e724a892e121b394bd Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 27 Jan 2025 20:48:23 +0200 Subject: [PATCH 08/11] dest finality violation, lint fixes --- .../smoke/ccip/ccip_reorg_test.go | 174 +++++++++++++++++- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index 7cbe1159735..a53d7cbb122 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -4,18 +4,22 @@ import ( "fmt" "os" "slices" + "strconv" "strings" "testing" "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/onsi/gomega" chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" + "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + ctf_client "github.com/smartcontractkit/chainlink-testing-framework/lib/client" "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink/deployment" @@ -58,7 +62,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { }, }), testhelpers.WithExtraConfigTomls([]string{ - fmt.Sprintf("%s.toml", t.Name()), + t.Name() + ".toml", }), ) @@ -91,7 +95,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { chainSelToRPCURL := make(map[uint64]string) for _, chain := range dockerEnv.GetDevEnvConfig().Chains { require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) require.NoError(t, err) chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal @@ -129,7 +133,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { Int64("blockNumber", bn). Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) + return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") // Run reorg below finality depth @@ -202,7 +206,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { }, time.Minute, 10*time.Second).Should(gomega.BeTrue()) // expect the commit to still go through on the non-reorged source chain. - testhelpers.ConfirmCommitWithExpectedSeqNumRange( + _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, reorgSourceSelector, e.Env.Chains[destChainSelector], @@ -211,6 +215,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) + require.NoError(t, err) } func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { @@ -245,7 +250,7 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { }, }), testhelpers.WithExtraConfigTomls([]string{ - fmt.Sprintf("%s.toml", t.Name()), + t.Name() + ".toml", }), ) @@ -278,7 +283,7 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { chainSelToRPCURL := make(map[uint64]string) for _, chain := range dockerEnv.GetDevEnvConfig().Chains { require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) require.NoError(t, err) chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal @@ -334,7 +339,7 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { Int64("blockNumber", bn). Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) + return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") // Run reorg below finality depth @@ -365,6 +370,154 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { } func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { + t.Skip("Not working") + require.Equal( + t, + os.Getenv(testhelpers.ENVTESTTYPE), + string(testhelpers.Docker), + "Reorg tests are only supported in docker environments", + ) + + l := logging.GetTestLogger(t) + + // This test sends a ccip message and re-orgs the chain + // after the message block is finalized. + e, _, tEnv := testsetups.NewIntegrationEnvironment( + t, + testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ + { + Msg: "Got very old block.", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Reorg greater than finality depth detected", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Failed to poll and save logs due to finality violation, retrying later", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + }), + ) + + nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) + require.NoError(t, err) + + var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) + for _, n := range nodeInfos.NonBootstraps() { + nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) + } + + l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) + + state, err := ccipcs.LoadOnchainState(e.Env) + require.NoError(t, err) + + allChains := e.Env.AllChainSelectors() + require.GreaterOrEqual(t, len(allChains), 2) + sourceSelector := allChains[0] + destChainSelector := allChains[1] + destChain, ok := chainsel.ChainBySelector(destChainSelector) + require.True(t, ok) + destLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", destChain.EvmChainID) + l. + Info(). + Msgf("reorging dest log poller service name: %s", destLogPollerService) + + dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) + require.True(t, ok) + + chainSelToRPCURL := make(map[uint64]string) + for _, chain := range dockerEnv.GetDevEnvConfig().Chains { + require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) + require.NoError(t, err) + + chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal + } + + reorgDestClient := ctf_client.NewRPCClient(chainSelToRPCURL[destChainSelector], nil) + + // setup lanes from s1 and s2 to destChainSelector + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destChainSelector, false) + + // wait for log poller filters to get registered. + l.Info().Msg("waiting for log poller filters to get registered") + time.Sleep(15 * time.Second) + reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), + Data: []byte("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }) + l.Info().Msgf("sent CCIP message that will get re-orged, msg id: %x", reorgingMsgEvent.Message.Header.MessageId) + + // Run reorg above finality depth + const reorgDepth = 50 + l.Info(). + Int("reorgDepth", reorgDepth). + Uint64("destChainSelector", destChainSelector). + Msg("starting blockchain reorg on Simulated Geth chain") + err = reorgDestClient.GethSetHead(reorgDepth) + require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + + nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() + nonBootstrapCount := len(nodeAPIs) - 1 + l.Info().Msgf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", nonBootstrapCount) + require.Eventually(t, func() bool { + violatedResponses := make(map[string]struct{}) + for _, node := range nodeAPIs { + // skip bootstrap nodes, they won't have any logpoller filters + p2pKeys, err := node.MustReadP2PKeys() + require.NoError(t, err) + + l.Debug().Msgf("got p2pKeys from node API: %+v", p2pKeys) + + require.GreaterOrEqual(t, len(p2pKeys.Data), 1) + if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { + l.Info().Msgf("skipping bootstrap node w/ p2p id %s", p2pKeys.Data[0].Attributes.PeerID) + continue + } + + resp, _, err := node.Health() + require.NoError(t, err) + for _, d := range resp.Data { + if d.Attributes.Name == destLogPollerService && + d.Attributes.Output == "finality violated" && + d.Attributes.Status == "failing" { + violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} + break + } + } + + if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { + l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) + } else { + l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", + p2pKeys.Data[0].Attributes.PeerID, + getLogPollerHealth(destLogPollerService, resp.Data), + ) + } + } + + l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) + return len(violatedResponses) == nonBootstrapCount + }, 2*time.Minute, 5*time.Second, "not all the nodes report finality violation") + l.Info().Msg("All nodes reported finality violation") + + // the commit should NOT go through on the re-orged dest chain. + // TODO: this is not a great way to assert not-happenings. + gomega.NewWithT(t).Consistently(func() bool { + it, err := state.Chains[destChainSelector].OffRamp.FilterCommitReportAccepted(&bind.FilterOpts{ + Start: 0, + }) + require.NoError(t, err) + return !it.Next() + }, tests.WaitTimeout(t), 10*time.Second).Should(gomega.BeTrue()) } // This test sends a ccip message and re-orgs the chain @@ -439,7 +592,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { chainSelToRPCURL := make(map[uint64]string) for _, chain := range dockerEnv.GetDevEnvConfig().Chains { require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(fmt.Sprintf("%d", chain.ChainID), chainsel.FamilyEVM) + details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) require.NoError(t, err) chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal @@ -525,7 +678,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { l.Info().Msg("All nodes reported finality violation") // expect the commit to still go through on the non-reorged source chain. - testhelpers.ConfirmCommitWithExpectedSeqNumRange( + _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, noreorgSourceSelector, e.Env.Chains[destChainSelector], @@ -534,6 +687,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) + require.NoError(t, err) // Works but super slow. // testhelpers.ConfirmExecWithSeqNrs( From dcc1439c77d9b044d25c886b49eebaafdb4a64a4 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Tue, 28 Jan 2025 15:50:13 +0200 Subject: [PATCH 09/11] violate finality on dest --- integration-tests/smoke/ccip/ccip_reorg_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index a53d7cbb122..fe296b53e79 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -18,7 +18,6 @@ import ( "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" ctf_client "github.com/smartcontractkit/chainlink-testing-framework/lib/client" "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" @@ -370,7 +369,6 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { } func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { - t.Skip("Not working") require.Equal( t, os.Getenv(testhelpers.ENVTESTTYPE), @@ -457,7 +455,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { l.Info().Msgf("sent CCIP message that will get re-orged, msg id: %x", reorgingMsgEvent.Message.Header.MessageId) // Run reorg above finality depth - const reorgDepth = 50 + const reorgDepth = 60 l.Info(). Int("reorgDepth", reorgDepth). Uint64("destChainSelector", destChainSelector). @@ -517,7 +515,7 @@ func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { }) require.NoError(t, err) return !it.Next() - }, tests.WaitTimeout(t), 10*time.Second).Should(gomega.BeTrue()) + }, 1*time.Minute, 10*time.Second).Should(gomega.BeTrue()) } // This test sends a ccip message and re-orgs the chain From 3c3529d4fb1ff2143e122c9af4b4276c0f088efc Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Wed, 29 Jan 2025 20:07:41 +0200 Subject: [PATCH 10/11] cleanup --- .github/e2e-tests.yml | 4 +- .../changeset/testhelpers/test_environment.go | 1 - .../nodeclient/chainlink_models.go | 2 + .../smoke/ccip/ccip_reorg_test.go | 40 ++----------------- .../Test_CCIPReorg_BelowFinality_OnDest.toml | 1 + ...Test_CCIPReorg_BelowFinality_OnSource.toml | 1 + integration-tests/testconfig/ccip/ccip.toml | 3 ++ 7 files changed, 12 insertions(+), 40 deletions(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 4ea5263095b..4b1424e226a 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -960,7 +960,7 @@ runner-test-matrix: triggers: - PR E2E Core Tests - Nightly E2E Tests - test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_BelowFinality_OnSource|Test_CCIPReorg_BelowFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json + test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_BelowFinality_OnSource|Test_CCIPReorg_BelowFinality_OnDest" ccip_reorg_test.go -timeout 25m -test.parallel=1 -count=1 -json pyroscope_env: ci-smoke-ccipv1_6-evm-simulated test_env_vars: E2E_TEST_SELECTED_NETWORK: SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY @@ -974,7 +974,7 @@ runner-test-matrix: triggers: - PR E2E Core Tests - Nightly E2E Tests - test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_GreaterThanFinality_OnSource|Test_CCIPReorg_GreaterThanFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json + test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_GreaterThanFinality_OnSource|Test_CCIPReorg_GreaterThanFinality_OnDest" ccip_reorg_test.go -timeout 25m -test.parallel=1 -count=1 -json pyroscope_env: ci-smoke-ccipv1_6-evm-simulated test_env_vars: E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3 diff --git a/deployment/ccip/changeset/testhelpers/test_environment.go b/deployment/ccip/changeset/testhelpers/test_environment.go index 75fde7c17d2..f415bc13ae2 100644 --- a/deployment/ccip/changeset/testhelpers/test_environment.go +++ b/deployment/ccip/changeset/testhelpers/test_environment.go @@ -487,7 +487,6 @@ func NewEnvironmentWithJobsAndContracts(t *testing.T, tEnv TestEnvironment) Depl tc := tEnv.TestConfigs() e := NewEnvironment(t, tEnv) allChains := e.Env.AllChainSelectors() - t.Log("number of chains:", len(allChains)) mcmsCfg := make(map[uint64]commontypes.MCMSWithTimelockConfig) for _, c := range e.Env.AllChainSelectors() { diff --git a/deployment/environment/nodeclient/chainlink_models.go b/deployment/environment/nodeclient/chainlink_models.go index 14372fd8e27..277b10d30cb 100644 --- a/deployment/environment/nodeclient/chainlink_models.go +++ b/deployment/environment/nodeclient/chainlink_models.go @@ -33,12 +33,14 @@ type ResponseSlice struct { Data []map[string]interface{} } +// HealthCheck corresponds to presenters.Check. type HealthCheck struct { Name string `json:"name"` Status string `json:"status"` Output string `json:"output"` } +// HealthResponseDetails is the generic model for services health statuses. type HealthResponseDetail struct { Type string `json:"type"` ID string `json:"id"` diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index fe296b53e79..5b84a1a9239 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -43,23 +43,6 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { // prior to the message block being finalized. e, _, tEnv := testsetups.NewIntegrationEnvironment( t, - testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ - { - Msg: "Got very old block.", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Reorg greater than finality depth detected", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Failed to poll and save logs due to finality violation, retrying later", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - }), testhelpers.WithExtraConfigTomls([]string{ t.Name() + ".toml", }), @@ -132,7 +115,7 @@ func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { Int64("blockNumber", bn). Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec + return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec // this will not overflow in the current test }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") // Run reorg below finality depth @@ -231,23 +214,6 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { // prior to the message block being finalized. e, _, tEnv := testsetups.NewIntegrationEnvironment( t, - testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ - { - Msg: "Got very old block.", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Reorg greater than finality depth detected", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Failed to poll and save logs due to finality violation, retrying later", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - }), testhelpers.WithExtraConfigTomls([]string{ t.Name() + ".toml", }), @@ -338,7 +304,7 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { Int64("blockNumber", bn). Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec + return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec // this will not overflow in the current test }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") // Run reorg below finality depth @@ -356,7 +322,7 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") // commit should be re-submitted after the re-org - reportEvent, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( + _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, sourceSelector, e.Env.Chains[destSelector], diff --git a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml index 8dc6d2211ae..6249d7951a6 100644 --- a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml +++ b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnDest.toml @@ -1,3 +1,4 @@ +#! These are intended to be overrides for the test Test_CCIPReorg_BelowFinality_OnDest. [NodeConfig.ChainConfigTOMLByChainID] #! FinalityDepth on 2337 overridden to 10. 2337 = """ diff --git a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml index 5798e7236d5..12dc6de4c81 100644 --- a/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml +++ b/integration-tests/testconfig/ccip/Test_CCIPReorg_BelowFinality_OnSource.toml @@ -1,3 +1,4 @@ +#! These are intended to be overrides for the test Test_CCIPReorg_BelowFinality_OnSource. [NodeConfig.ChainConfigTOMLByChainID] #! FinalityDepth on 1337 overridden to 10. 1337 = """ diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index c1f7b0f1c11..cdb14d45757 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -83,6 +83,9 @@ evm_supports_eip1559 = true evm_default_gas_limit = 6000000 evm_finality_depth = 1 +# SIMULATED_1_DEEPER_FINALITY, SIMULATED_2_DEEPER_FINALITY, SIMULATED_3_DEEPER_FINALITY +# are the same as SIMULATED_1, SIMULATED_2, SIMULATED_3, but with finality_depth = 10. +# These are primarily used in the reorg tests, but can technically be used in any tests. [Network.EVMNetworks.SIMULATED_1_DEEPER_FINALITY] evm_name = 'chain-1337' evm_chain_id = 1337 From c775042225c09a5f665cda87443d4d1720187a85 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Fri, 31 Jan 2025 17:15:21 +0200 Subject: [PATCH 11/11] refactor, skip some failing tests --- .../smoke/ccip/ccip_reorg_test.go | 886 ++++++++---------- .../testsetups/ccip/test_helpers.go | 4 +- 2 files changed, 376 insertions(+), 514 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_reorg_test.go b/integration-tests/smoke/ccip/ccip_reorg_test.go index 5b84a1a9239..df5c544d510 100644 --- a/integration-tests/smoke/ccip/ccip_reorg_test.go +++ b/integration-tests/smoke/ccip/ccip_reorg_test.go @@ -14,6 +14,8 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/onsi/gomega" chainsel "github.com/smartcontractkit/chain-selectors" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" @@ -26,257 +28,116 @@ import ( "github.com/smartcontractkit/chainlink/deployment/ccip/changeset/testhelpers" "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" ) -func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { - require.Equal( - t, - os.Getenv(testhelpers.ENVTESTTYPE), - string(testhelpers.Docker), - "Reorg tests are only supported in docker environments", - ) +var ( + logsToIgnoreOpt = testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ + { + Msg: "Got very old block.", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Reorg greater than finality depth detected", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + { + Msg: "Failed to poll and save logs due to finality violation, retrying later", + Reason: "We are expecting a re-org beyond finality", + Level: zapcore.DPanicLevel, + }, + }) +) - l := logging.GetTestLogger(t) +const ( + greaterThanFinalityReorgDepth = 60 + lessThanFinalityReorgDepth = 7 +) - // This test sends a ccip message and re-orgs the chain - // prior to the message block being finalized. - e, _, tEnv := testsetups.NewIntegrationEnvironment( - t, - testhelpers.WithExtraConfigTomls([]string{ - t.Name() + ".toml", - }), +func Test_CCIPReorg_BelowFinality_OnSource(t *testing.T) { + e, l, dockerEnv, nonBootstrapP2PIDs, state := setupReorgTest(t, + testhelpers.WithExtraConfigTomls([]string{t.Name() + ".toml"}), ) - nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) - require.NoError(t, err) - - var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) - for _, n := range nodeInfos.NonBootstraps() { - nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) - } - - l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) - - state, err := ccipcs.LoadOnchainState(e.Env) - require.NoError(t, err) - + // Chain setup allChains := e.Env.AllChainSelectors() require.GreaterOrEqual(t, len(allChains), 2) - reorgSourceSelector := allChains[0] - reorgSourceChain, ok := chainsel.ChainBySelector(reorgSourceSelector) - require.True(t, ok) - reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", reorgSourceChain.EvmChainID) - l.Info(). - Msgf("reorging log poller service name: %s", reorgLogPollerService) - destChainSelector := allChains[1] - - dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) - require.True(t, ok) - - chainSelToRPCURL := make(map[uint64]string) - for _, chain := range dockerEnv.GetDevEnvConfig().Chains { - require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) - require.NoError(t, err) - - chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal - } - - reorgSourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[reorgSourceSelector], nil) - - testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, reorgSourceSelector, destChainSelector, false) + sourceSelector := allChains[0] + destSelector := allChains[1] - // wait for log poller filters to get registered. - l.Info().Msg("waiting for log poller filters to get registered") - time.Sleep(15 * time.Second) - reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - l. - Info(). - Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). - Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). - Str("messageBlockHashBeforeReorg", reorgingMsgEvent.Raw.BlockHash.String()). - Msg("sent CCIP message that will get re-orged before getting finalized") - - const reorgDepth = 7 - var minChainBlockNumberBeforeReorg = reorgingMsgEvent.Raw.BlockNumber + reorgDepth - 1 - // let reorgDepth - 1 blocks pass by before re-orging the message. - // This will effectively rewind the chain to a block where the message didn't exist. - require.Eventually(t, func() bool { - bn, err := reorgSourceClient.BlockNumber() - require.NoError(t, err) - l.Info(). - Int64("blockNumber", bn). - Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). - Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec // this will not overflow in the current test - }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") + // Build RPC map and get clients + chainSelToRPCURL := buildChainSelectorToRPCURLMap(t, dockerEnv) + sourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[sourceSelector], nil) - // Run reorg below finality depth - l.Info(). - Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). - Int("reorgDepth", reorgDepth). - Uint64("sourceChainSelector", reorgSourceSelector). - Msg("starting blockchain reorg on Simulated Geth chain") - err = reorgSourceClient.GethSetHead(reorgDepth) - require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + // Setup CCIP lane + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destSelector, false) + waitForLogPollerFilters(l) - bnAfterReorg, err := reorgSourceClient.BlockNumber() - require.NoError(t, err, "error getting block number after reorg") + // Send initial message + msgBeforeReorg := sendCCIPMessage(t, e.Env, state, sourceSelector, destSelector, l) - l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") + // Wait and perform reorg + minBlock := msgBeforeReorg.Raw.BlockNumber + lessThanFinalityReorgDepth - 1 + waitForBlockNumber(t, sourceClient, minBlock, 1*time.Minute, 500*time.Millisecond, l) + performReorg(t, sourceClient, lessThanFinalityReorgDepth, l) - reorgingMsgEvent = testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - l. - Info(). - Str("blockHashAfterReorg", reorgingMsgEvent.Raw.BlockHash.String()). - Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). - Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). - Str("messageBlockHash", reorgingMsgEvent.Raw.BlockHash.String()). - Msgf("re-sent CCIP message after the re-org") + // Verify message consistency + msgAfterReorg := sendCCIPMessage(t, e.Env, state, sourceSelector, destSelector, l) + require.Equal(t, msgBeforeReorg.Message.Header.SequenceNumber, msgAfterReorg.Message.Header.SequenceNumber) + require.Equal(t, msgBeforeReorg.Message.Header.MessageId, msgAfterReorg.Message.Header.MessageId) + // Check node health nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() - nonBootstrapCount := len(nodeAPIs) - 1 - l.Info().Msgf("waiting for %d non-bootstrap nodes to NOT report finality violation on the logpoller, since re-org is less than finality", nonBootstrapCount) - gomega.NewWithT(t).Consistently(func() bool { - violatedResponses := make(map[string]struct{}) - for _, node := range nodeAPIs { - // skip bootstrap nodes, they won't have any logpoller filters - p2pKeys, err := node.MustReadP2PKeys() - require.NoError(t, err) - - require.GreaterOrEqual(t, len(p2pKeys.Data), 1) - if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { - continue - } - - resp, _, err := node.Health() - require.NoError(t, err) - for _, d := range resp.Data { - if d.Attributes.Name == reorgLogPollerService && - d.Attributes.Output == "finality violated" && - d.Attributes.Status == "failing" { - violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} - break - } - } - - if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { - l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) - } else { - l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", - p2pKeys.Data[0].Attributes.PeerID, - getLogPollerHealth(reorgLogPollerService, resp.Data), - ) - } - } - - l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) - return len(violatedResponses) == 0 - }, time.Minute, 10*time.Second).Should(gomega.BeTrue()) + checkFinalityViolations( + t, + nodeAPIs, + nonBootstrapP2PIDs, + getHeadTrackerService(t, sourceSelector), + getLogPollerService(t, sourceSelector), + l, + 0, // no nodes reporting finality violation + 1*time.Minute, // timeout + 10*time.Second, // interval + ) - // expect the commit to still go through on the non-reorged source chain. - _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( + // Verify commit + _, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, - reorgSourceSelector, - e.Env.Chains[destChainSelector], - state.Chains[destChainSelector].OffRamp, + sourceSelector, + e.Env.Chains[destSelector], + state.Chains[destSelector].OffRamp, nil, // startBlock ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) - require.NoError(t, err) + require.NoError(t, err, "Commit verification failed") } func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { - require.Equal( - t, - os.Getenv(testhelpers.ENVTESTTYPE), - string(testhelpers.Docker), - "Reorg tests are only supported in docker environments", - ) - - l := logging.GetTestLogger(t) - - // This test sends a ccip message and re-orgs the chain - // prior to the message block being finalized. - e, _, tEnv := testsetups.NewIntegrationEnvironment( - t, - testhelpers.WithExtraConfigTomls([]string{ - t.Name() + ".toml", - }), + e, l, dockerEnv, _, state := setupReorgTest(t, + testhelpers.WithExtraConfigTomls([]string{t.Name() + ".toml"}), ) - nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) - require.NoError(t, err) - - var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) - for _, n := range nodeInfos.NonBootstraps() { - nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) - } - - l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) - - state, err := ccipcs.LoadOnchainState(e.Env) - require.NoError(t, err) - allChains := e.Env.AllChainSelectors() require.GreaterOrEqual(t, len(allChains), 2) sourceSelector := allChains[0] destSelector := allChains[1] - destChain, ok := chainsel.ChainBySelector(destSelector) - require.True(t, ok) - reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", destChain.EvmChainID) - l.Info(). - Msgf("reorging log poller service name: %s", reorgLogPollerService) - - dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) - require.True(t, ok) - chainSelToRPCURL := make(map[uint64]string) - for _, chain := range dockerEnv.GetDevEnvConfig().Chains { - require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) - require.NoError(t, err) - - chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal - } - - reorgDestClient := ctf_client.NewRPCClient(chainSelToRPCURL[destSelector], nil) + // Chain setup + chainSelToRPCURL := buildChainSelectorToRPCURLMap(t, dockerEnv) + destClient := ctf_client.NewRPCClient(chainSelToRPCURL[destSelector], nil) + // Test setup testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destSelector, false) + waitForLogPollerFilters(l) - // wait for log poller filters to get registered. - l.Info().Msg("waiting for log poller filters to get registered") - time.Sleep(15 * time.Second) - reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceSelector, destSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destSelector].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - l. - Info(). - Str("messageID", hexutil.Encode(reorgingMsgEvent.Message.Header.MessageId[:])). - Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). - Str("messageBlockHash", reorgingMsgEvent.Raw.BlockHash.String()). - Msgf("sent CCIP message that whose commit report will re-org on dest before getting finalized") - - // expect the commit to still go through for the message. - reportEvent, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( + // Initial operation + msg := sendCCIPMessage(t, e.Env, state, sourceSelector, destSelector, l) + _, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, sourceSelector, e.Env.Chains[destSelector], @@ -285,43 +146,14 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) - require.NoError(t, err) + require.NoError(t, err, "Commit verification failed before reorg") - l. - Info(). - Uint64("reportBlockNumber", reportEvent.Raw.BlockNumber). - Str("reportBlockHash", reportEvent.Raw.BlockHash.String()). - Msg("got commit report on dest, preparing to re-org it") - - // re-org the dest chain less than finality blocks. - const reorgDepth = 7 - var minChainBlockNumberBeforeReorg = reportEvent.Raw.BlockNumber + reorgDepth - 1 - // Wait for chain to progress - require.Eventually(t, func() bool { - bn, err := reorgDestClient.BlockNumber() - require.NoError(t, err) - l.Info(). - Int64("blockNumber", bn). - Uint64("targetBlockNumber", minChainBlockNumberBeforeReorg). - Msg("Waiting for chain to progress above target block number") - return bn >= int64(minChainBlockNumberBeforeReorg) //nolint:gosec // this will not overflow in the current test - }, 1*time.Minute, 500*time.Millisecond, "timeout exceeded: chain did not progress above the target block number") + // Reorg execution + minBlock := msg.Raw.BlockNumber + lessThanFinalityReorgDepth - 1 + waitForBlockNumber(t, destClient, minBlock, 1*time.Minute, 500*time.Millisecond, l) + performReorg(t, destClient, lessThanFinalityReorgDepth, l) - // Run reorg below finality depth - l.Info(). - Uint64("messageBlockNumber", reorgingMsgEvent.Raw.BlockNumber). - Int("reorgDepth", reorgDepth). - Uint64("sourceChainSelector", sourceSelector). - Msg("starting blockchain reorg on Simulated Geth chain") - err = reorgDestClient.GethSetHead(reorgDepth) - require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") - - bnAfterReorg, err := reorgDestClient.BlockNumber() - require.NoError(t, err, "error getting block number after reorg") - - l.Info().Int64("blockNumberAfterReorg", bnAfterReorg).Msg("block number after reorg") - - // commit should be re-submitted after the re-org + // Post-reorg validation _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, sourceSelector, @@ -331,154 +163,64 @@ func Test_CCIPReorg_BelowFinality_OnDest(t *testing.T) { ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) - require.NoError(t, err) + require.NoError(t, err, "Commit verification failed after reorg") } func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { - require.Equal( - t, - os.Getenv(testhelpers.ENVTESTTYPE), - string(testhelpers.Docker), - "Reorg tests are only supported in docker environments", - ) - - l := logging.GetTestLogger(t) - - // This test sends a ccip message and re-orgs the chain - // after the message block is finalized. - e, _, tEnv := testsetups.NewIntegrationEnvironment( - t, - testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ - { - Msg: "Got very old block.", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Reorg greater than finality depth detected", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Failed to poll and save logs due to finality violation, retrying later", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - }), - ) - - nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) - require.NoError(t, err) - - var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) - for _, n := range nodeInfos.NonBootstraps() { - nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) - } - - l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) - - state, err := ccipcs.LoadOnchainState(e.Env) - require.NoError(t, err) + t.Skip("Not detecting finality violations correctly") + e, l, dockerEnv, nonBootstrapP2PIDs, state := setupReorgTest(t, logsToIgnoreOpt) allChains := e.Env.AllChainSelectors() require.GreaterOrEqual(t, len(allChains), 2) sourceSelector := allChains[0] - destChainSelector := allChains[1] - destChain, ok := chainsel.ChainBySelector(destChainSelector) - require.True(t, ok) - destLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", destChain.EvmChainID) + destSelector := allChains[1] l. Info(). - Msgf("reorging dest log poller service name: %s", destLogPollerService) - - dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) - require.True(t, ok) - - chainSelToRPCURL := make(map[uint64]string) - for _, chain := range dockerEnv.GetDevEnvConfig().Chains { - require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) - details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) - require.NoError(t, err) - - chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal - } - - reorgDestClient := ctf_client.NewRPCClient(chainSelToRPCURL[destChainSelector], nil) + Uint64("sourceSelector", sourceSelector). + Uint64("destSelector", destSelector). + Msg("Chain selectors") - // setup lanes from s1 and s2 to destChainSelector - testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destChainSelector, false) + // Chain setup + chainSelToRPCURL := buildChainSelectorToRPCURLMap(t, dockerEnv) + destClient := ctf_client.NewRPCClient(chainSelToRPCURL[destSelector], nil) - // wait for log poller filters to get registered. - l.Info().Msg("waiting for log poller filters to get registered") - time.Sleep(15 * time.Second) - reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, sourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - l.Info().Msgf("sent CCIP message that will get re-orged, msg id: %x", reorgingMsgEvent.Message.Header.MessageId) + // Test setup + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, sourceSelector, destSelector, false) + waitForLogPollerFilters(l) - // Run reorg above finality depth - const reorgDepth = 60 - l.Info(). - Int("reorgDepth", reorgDepth). - Uint64("destChainSelector", destChainSelector). - Msg("starting blockchain reorg on Simulated Geth chain") - err = reorgDestClient.GethSetHead(reorgDepth) - require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + // Initial operation + sendCCIPMessage(t, e.Env, state, sourceSelector, destSelector, l) + _, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( + t, + sourceSelector, + e.Env.Chains[destSelector], + state.Chains[destSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit + ) + require.NoError(t, err, "Commit verification failed before reorg") + // Deep reorg execution + performReorg(t, destClient, greaterThanFinalityReorgDepth, l) nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() - nonBootstrapCount := len(nodeAPIs) - 1 - l.Info().Msgf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", nonBootstrapCount) - require.Eventually(t, func() bool { - violatedResponses := make(map[string]struct{}) - for _, node := range nodeAPIs { - // skip bootstrap nodes, they won't have any logpoller filters - p2pKeys, err := node.MustReadP2PKeys() - require.NoError(t, err) - - l.Debug().Msgf("got p2pKeys from node API: %+v", p2pKeys) - - require.GreaterOrEqual(t, len(p2pKeys.Data), 1) - if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { - l.Info().Msgf("skipping bootstrap node w/ p2p id %s", p2pKeys.Data[0].Attributes.PeerID) - continue - } - - resp, _, err := node.Health() - require.NoError(t, err) - for _, d := range resp.Data { - if d.Attributes.Name == destLogPollerService && - d.Attributes.Output == "finality violated" && - d.Attributes.Status == "failing" { - violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} - break - } - } - if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { - l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) - } else { - l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", - p2pKeys.Data[0].Attributes.PeerID, - getLogPollerHealth(destLogPollerService, resp.Data), - ) - } - } - - l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) - return len(violatedResponses) == nonBootstrapCount - }, 2*time.Minute, 5*time.Second, "not all the nodes report finality violation") - l.Info().Msg("All nodes reported finality violation") + // Finality violation check + checkFinalityViolations( + t, + nodeAPIs, + nonBootstrapP2PIDs, + getHeadTrackerService(t, destSelector), + getLogPollerService(t, destSelector), + l, + len(nodeAPIs)-1, + tests.WaitTimeout(t), + 10*time.Second, + ) - // the commit should NOT go through on the re-orged dest chain. - // TODO: this is not a great way to assert not-happenings. + // Commit absence check gomega.NewWithT(t).Consistently(func() bool { - it, err := state.Chains[destChainSelector].OffRamp.FilterCommitReportAccepted(&bind.FilterOpts{ - Start: 0, - }) + it, err := state.Chains[destSelector].OffRamp.FilterCommitReportAccepted(&bind.FilterOpts{Start: 0}) require.NoError(t, err) return !it.Next() }, 1*time.Minute, 10*time.Second).Should(gomega.BeTrue()) @@ -490,186 +232,306 @@ func Test_CCIPReorg_GreaterThanFinality_OnDest(t *testing.T) { // messages from the re-orged chain anymore. // However, it should gracefully process messages from non-reorged chains. func Test_CCIPReorg_GreaterThanFinality_OnSource(t *testing.T) { - require.Equal( + t.Skip("Not detecting finality violations correctly") + e, l, dockerEnv, nonBootstrapP2PIDs, state := setupReorgTest(t, logsToIgnoreOpt) + + allChains := e.Env.AllChainSelectors() + require.GreaterOrEqual(t, len(allChains), 3) + reorgSource := allChains[0] + nonReorgSource := allChains[1] + destSelector := allChains[2] + + // Chain setup + chainSelToRPCURL := buildChainSelectorToRPCURLMap(t, dockerEnv) + reorgSourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[reorgSource], nil) + + // Multi-lane setup + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, reorgSource, destSelector, false) + testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, nonReorgSource, destSelector, false) + waitForLogPollerFilters(l) + + // Send messages from both sources + sendCCIPMessage(t, e.Env, state, reorgSource, destSelector, l) + sendCCIPMessage(t, e.Env, state, nonReorgSource, destSelector, l) + + // Deep reorg execution + performReorg(t, reorgSourceClient, greaterThanFinalityReorgDepth, l) + nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() + + // Finality check + checkFinalityViolations( t, - os.Getenv(testhelpers.ENVTESTTYPE), - string(testhelpers.Docker), - "Reorg tests are only supported in docker environments", + nodeAPIs, + nonBootstrapP2PIDs, + getHeadTrackerService(t, reorgSource), + getLogPollerService(t, reorgSource), + l, + len(nodeAPIs)-1, + tests.WaitTimeout(t), + 10*time.Second, ) - l := logging.GetTestLogger(t) - - // This test sends a ccip message and re-orgs the chain - // after the message block is finalized. - e, _, tEnv := testsetups.NewIntegrationEnvironment( + // Validate non-reorged source + _, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, - testhelpers.WithLogMessagesToIgnore([]testhelpers.LogMessageToIgnore{ - { - Msg: "Got very old block.", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Reorg greater than finality depth detected", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - { - Msg: "Failed to poll and save logs due to finality violation, retrying later", - Reason: "We are expecting a re-org beyond finality", - Level: zapcore.DPanicLevel, - }, - }), + nonReorgSource, + e.Env.Chains[destSelector], + state.Chains[destSelector].OffRamp, + nil, // startBlock + ccipocr3.NewSeqNumRange(1, 1), + false, // enforceSingleCommit ) + require.NoError(t, err, "Commit verification failed for non-reorged source") - nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) - require.NoError(t, err) + // Commit absence check on the reorged source + gomega.NewWithT(t).Consistently(func() bool { + it, err := state.Chains[destSelector].OffRamp.FilterCommitReportAccepted(&bind.FilterOpts{Start: 0}) + require.NoError(t, err) + var found bool + outer: + for it.Next() { + for _, mr := range it.Event.MerkleRoots { + if mr.SourceChainSelector == reorgSource { + found = true + break outer + } + } + } + return !found + }, 1*time.Minute, 10*time.Second).Should(gomega.BeTrue()) +} - var nonBootstrapP2PIDs = make([]string, 0, len(nodeInfos.NonBootstraps())) - for _, n := range nodeInfos.NonBootstraps() { - nonBootstrapP2PIDs = append(nonBootstrapP2PIDs, strings.TrimPrefix(n.PeerID.String(), "p2p_")) +func getLogPollerHealth(logPollerService string, healthResponses []nodeclient.HealthResponseDetail) nodeclient.HealthCheck { + for _, d := range healthResponses { + if d.Attributes.Name == logPollerService { + return d.Attributes + } } - l.Info().Msgf("nonBootstrapP2PIDs: %s", nonBootstrapP2PIDs) + return nodeclient.HealthCheck{} +} + +// Helper to initialize common test components +func setupReorgTest(t *testing.T, testOpts ...testhelpers.TestOps) ( + testhelpers.DeployedEnv, + logging.Logger, + *testsetups.DeployedLocalDevEnvironment, + []string, + ccipcs.CCIPOnChainState, +) { + require.Equal(t, os.Getenv(testhelpers.ENVTESTTYPE), string(testhelpers.Docker), + "Reorg tests are only supported in docker environments") + + l := logging.GetTestLogger(t) + e, _, tEnv := testsetups.NewIntegrationEnvironment(t, testOpts...) + dockerEnv := tEnv.(*testsetups.DeployedLocalDevEnvironment) + + nodeInfos, err := deployment.NodeInfo(e.Env.NodeIDs, e.Env.Offchain) + require.NoError(t, err) + nonBootstrapP2PIDs := getNonBootstrapP2PIDs(nodeInfos) state, err := ccipcs.LoadOnchainState(e.Env) require.NoError(t, err) - allChains := e.Env.AllChainSelectors() - require.GreaterOrEqual(t, len(allChains), 3) - reorgSourceSelector := allChains[0] - reorgSourceChain, ok := chainsel.ChainBySelector(reorgSourceSelector) - require.True(t, ok) - noreorgSourceSelector := allChains[1] - noreorgSourceChain, ok := chainsel.ChainBySelector(noreorgSourceSelector) - require.True(t, ok) - reorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", reorgSourceChain.EvmChainID) - noreorgLogPollerService := fmt.Sprintf("EVM.%d.LogPoller", noreorgSourceChain.EvmChainID) - l.Info(). - Msgf("reorging log poller service name: %s, no reorg log poller service name: %s", - reorgLogPollerService, noreorgLogPollerService) - destChainSelector := allChains[2] + return e, l, dockerEnv, nonBootstrapP2PIDs, state +} - dockerEnv, ok := tEnv.(*testsetups.DeployedLocalDevEnvironment) - require.True(t, ok) +// Extract non-bootstrap P2P IDs +func getNonBootstrapP2PIDs(nodeInfos deployment.Nodes) []string { + ids := make([]string, 0, len(nodeInfos.NonBootstraps())) + for _, n := range nodeInfos.NonBootstraps() { + ids = append(ids, strings.TrimPrefix(n.PeerID.String(), "p2p_")) + } + return ids +} + +// Build chain selector to RPC URL map +func buildChainSelectorToRPCURLMap(t *testing.T, dockerEnv *testsetups.DeployedLocalDevEnvironment) map[uint64]string { + devEnvConfig := dockerEnv.GetDevEnvConfig() + require.NotNil(t, devEnvConfig) chainSelToRPCURL := make(map[uint64]string) - for _, chain := range dockerEnv.GetDevEnvConfig().Chains { + for _, chain := range devEnvConfig.Chains { require.GreaterOrEqual(t, len(chain.HTTPRPCs), 1) details, err := chainsel.GetChainDetailsByChainIDAndFamily(strconv.FormatUint(chain.ChainID, 10), chainsel.FamilyEVM) require.NoError(t, err) - chainSelToRPCURL[details.ChainSelector] = chain.HTTPRPCs[0].Internal } + return chainSelToRPCURL +} - reorgSourceClient := ctf_client.NewRPCClient(chainSelToRPCURL[reorgSourceSelector], nil) +// Get log poller service name +func getLogPollerService(t *testing.T, chainSelector uint64) string { + chain, ok := chainsel.ChainBySelector(chainSelector) + require.True(t, ok) + return fmt.Sprintf("EVM.%d.LogPoller", chain.EvmChainID) +} - // setup lanes from s1 and s2 to destChainSelector - testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, reorgSourceSelector, destChainSelector, false) - testhelpers.AddLaneWithDefaultPricesAndFeeQuoterConfig(t, &e, state, noreorgSourceSelector, destChainSelector, false) +func getHeadTrackerService(t *testing.T, chainSelector uint64) string { + chain, ok := chainsel.ChainBySelector(chainSelector) + require.True(t, ok) + return fmt.Sprintf("EVM.%d.HeadTracker", chain.EvmChainID) +} - // wait for log poller filters to get registered. - l.Info().Msg("waiting for log poller filters to get registered") - time.Sleep(15 * time.Second) - reorgingMsgEvent := testhelpers.TestSendRequest(t, e.Env, state, reorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), - Data: []byte("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }) - l.Info().Msgf("sent CCIP message that will get re-orged, msg id: %x", reorgingMsgEvent.Message.Header.MessageId) - msgEvent := testhelpers.TestSendRequest(t, e.Env, state, noreorgSourceSelector, destChainSelector, false, router.ClientEVM2AnyMessage{ - Receiver: common.LeftPadBytes(state.Chains[destChainSelector].Receiver.Address().Bytes(), 32), +// Send CCIP message helper +func sendCCIPMessage( + t *testing.T, + env deployment.Environment, + state ccipcs.CCIPOnChainState, + sourceSelector, destSelector uint64, + l logging.Logger, +) *onramp.OnRampCCIPMessageSent { + msgEvent := testhelpers.TestSendRequest(t, env, state, sourceSelector, destSelector, false, router.ClientEVM2AnyMessage{ + Receiver: common.LeftPadBytes(state.Chains[destSelector].Receiver.Address().Bytes(), 32), Data: []byte("hello world"), TokenAmounts: nil, FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, }) - l.Info().Msgf("sent CCIP message that will not get re-orged, msgSentEvent: %x", msgEvent.Message.Header.MessageId) - // Run reorg above finality depth - const reorgDepth = 50 l.Info(). - Int("reorgDepth", reorgDepth). - Uint64("sourceChainSelector", reorgSourceSelector). - Msg("starting blockchain reorg on Simulated Geth chain") - err = reorgSourceClient.GethSetHead(reorgDepth) - require.NoError(t, err, "error starting blockchain reorg on Simulated Geth chain") + Str("messageID", hexutil.Encode(msgEvent.Message.Header.MessageId[:])). + Uint64("blockNumber", msgEvent.Raw.BlockNumber). + Str("blockHash", msgEvent.Raw.BlockHash.String()). + Msg("Sent CCIP message") - nodeAPIs := dockerEnv.GetCLClusterTestEnv().ClCluster.NodeAPIs() - nonBootstrapCount := len(nodeAPIs) - 1 - l.Info().Msgf("waiting for %d non-bootstrap nodes to report finality violation on the logpoller", nonBootstrapCount) - require.Eventually(t, func() bool { - violatedResponses := make(map[string]struct{}) + return msgEvent +} + +// Check finality violations helper +// Uses require.Eventually or gomega.Consistently based on expected count +func checkFinalityViolations( + t *testing.T, + nodeAPIs []*nodeclient.ChainlinkClient, + nonBootstrapP2PIDs []string, + headTrackerServiceName, + logPollerServiceName string, + l logging.Logger, + expectedCount int, + timeout time.Duration, + interval time.Duration, +) { + checkFunc := func() bool { + violated := 0 + headTrackerViolated := 0 + logPollerViolated := 0 for _, node := range nodeAPIs { - // skip bootstrap nodes, they won't have any logpoller filters p2pKeys, err := node.MustReadP2PKeys() require.NoError(t, err) - - l.Debug().Msgf("got p2pKeys from node API: %+v", p2pKeys) - - require.GreaterOrEqual(t, len(p2pKeys.Data), 1) - if !slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { - l.Info().Msgf("skipping bootstrap node w/ p2p id %s", p2pKeys.Data[0].Attributes.PeerID) + if len(p2pKeys.Data) == 0 || slices.Contains(nonBootstrapP2PIDs, p2pKeys.Data[0].Attributes.PeerID) { continue } resp, _, err := node.Health() require.NoError(t, err) for _, d := range resp.Data { - if d.Attributes.Name == reorgLogPollerService && - d.Attributes.Output == "finality violated" && - d.Attributes.Status == "failing" { - violatedResponses[p2pKeys.Data[0].Attributes.PeerID] = struct{}{} + isLogPollerFailing := d.Attributes.Name == logPollerServiceName && + d.Attributes.Output == commontypes.ErrFinalityViolated.Error() && + d.Attributes.Status == "failing" + isHeadTrackerFailing := d.Attributes.Name == headTrackerServiceName && + strings.Contains(d.Attributes.Output, "got very old block with number") && + d.Attributes.Status == "failing" + if isLogPollerFailing { + logPollerViolated++ + } + if isHeadTrackerFailing { + headTrackerViolated++ + } + if isLogPollerFailing || isHeadTrackerFailing { + violated++ break } } - - if _, ok := violatedResponses[p2pKeys.Data[0].Attributes.PeerID]; ok { - l.Info().Msgf("node %s reported finality violation", p2pKeys.Data[0].Attributes.PeerID) - } else { - l.Info().Msgf("node %s did not report finality violation, log poller response: %+v", - p2pKeys.Data[0].Attributes.PeerID, - getLogPollerHealth(reorgLogPollerService, resp.Data), - ) - } } - l.Info().Msgf("%d nodes reported finality violation", len(violatedResponses)) - return len(violatedResponses) == nonBootstrapCount - }, 3*time.Minute, 5*time.Second, "not all the nodes report finality violation") - l.Info().Msg("All nodes reported finality violation") + l. + Info(). + Int("violated", violated). + Int("expectedCount", expectedCount). + Int("headTrackerViolated", headTrackerViolated). + Int("logPollerViolated", logPollerViolated). + Msg("Checking finality violations") + return violated == expectedCount + } + if expectedCount > 0 { + require.Eventually(t, checkFunc, timeout, interval) + } else { + gomega.NewWithT(t).Consistently(checkFunc, timeout, interval).Should(gomega.BeTrue()) + } +} + +// waitForLogPollerFilters waits for log poller filters to be registered +func waitForLogPollerFilters(l logging.Logger) { + l.Info().Msg("Waiting for log poller filters to get registered") + time.Sleep(30 * time.Second) // Consider making duration configurable if needed +} - // expect the commit to still go through on the non-reorged source chain. - _, err = testhelpers.ConfirmCommitWithExpectedSeqNumRange( +// waitForBlockNumber waits until chain reaches target block number +func waitForBlockNumber( + t *testing.T, + client *ctf_client.RPCClient, + targetBlock uint64, + timeout time.Duration, + checkInterval time.Duration, + l logging.Logger, +) { + require.Eventually(t, func() bool { + bn, err := client.BlockNumber() + require.NoError(t, err) + l.Info(). + Int64("currentBlock", bn). + Uint64("targetBlock", targetBlock). + Msg("Waiting for chain progression") + return bn >= int64(targetBlock) + }, timeout, checkInterval, "Timeout waiting for block number") +} + +// performReorg executes a chain reorg and verifies its success +func performReorg( + t *testing.T, + client *ctf_client.RPCClient, + reorgDepth int, + l logging.Logger, +) { + l.Info(). + Int("reorgDepth", reorgDepth). + Msg("Starting blockchain reorg") + + // Get current block before reorg for verification + preReorgBlock, err := client.BlockNumber() + require.NoError(t, err) + + err = client.GethSetHead(reorgDepth) + require.NoError(t, err, "Failed to execute reorg") + + // Verify post-reorg state + postReorgBlock, err := client.BlockNumber() + require.NoError(t, err) + + l.Info(). + Int64("preReorgBlock", preReorgBlock). + Int64("postReorgBlock", postReorgBlock). + Msg("Reorg completed") + + require.Less(t, postReorgBlock, preReorgBlock, + "Block number should decrease after reorg") +} + +// confirmCommit verifies commit confirmation on destination chain +func confirmCommit( + t *testing.T, + sourceSelector uint64, + destChain deployment.Chain, + offRamp *offramp.OffRamp, +) { + _, err := testhelpers.ConfirmCommitWithExpectedSeqNumRange( t, - noreorgSourceSelector, - e.Env.Chains[destChainSelector], - state.Chains[destChainSelector].OffRamp, + sourceSelector, + destChain, + offRamp, nil, // startBlock ccipocr3.NewSeqNumRange(1, 1), false, // enforceSingleCommit ) - require.NoError(t, err) - - // Works but super slow. - // testhelpers.ConfirmExecWithSeqNrs( - // t, - // e.Env.Chains[noreorgSourceSelector], - // e.Env.Chains[destChainSelector], - // state.Chains[destChainSelector].OffRamp, - // nil, // startBlock - // []uint64{1}, // expectedSeqNrs - // ) -} - -func getLogPollerHealth(logPollerService string, healthResponses []nodeclient.HealthResponseDetail) nodeclient.HealthCheck { - for _, d := range healthResponses { - if d.Attributes.Name == logPollerService { - return d.Attributes - } - } - - return nodeclient.HealthCheck{} + require.NoError(t, err, "Commit verification failed") } diff --git a/integration-tests/testsetups/ccip/test_helpers.go b/integration-tests/testsetups/ccip/test_helpers.go index 2a06dc97275..87d73d2a82d 100644 --- a/integration-tests/testsetups/ccip/test_helpers.go +++ b/integration-tests/testsetups/ccip/test_helpers.go @@ -63,8 +63,8 @@ func (l *DeployedLocalDevEnvironment) GetCLClusterTestEnv() *test_env.CLClusterT return l.testEnv } -func (l *DeployedLocalDevEnvironment) GetDevEnvConfig() devenv.EnvironmentConfig { - return *l.devEnvCfg +func (l *DeployedLocalDevEnvironment) GetDevEnvConfig() *devenv.EnvironmentConfig { + return l.devEnvCfg } func (l *DeployedLocalDevEnvironment) DeployedEnvironment() testhelpers.DeployedEnv {