[CCIP-4171] integration-tests/smoke/ccip: add re-org tests

.github/e2e-tests.yml

@@ -937,6 +937,34 @@ runner-test-matrix:
 
   # START: CCIPv1.6 tests
 
+  - id: smoke/ccip/ccip_reorg_test.go:LessThanFinalityTests
+    path: integration-tests/smoke/ccip/ccip_reorg_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    triggers:
+      - PR E2E Core Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_BelowFinality_OnSource|Test_CCIPReorg_BelowFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json
+    pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
+    test_env_vars:
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY
+      E2E_JD_VERSION: 0.6.0
+      CCIP_V16_TEST_ENV: docker
+
+  - id: smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests
+    path: integration-tests/smoke/ccip/ccip_reorg_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    triggers:
+      - PR E2E Core Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/smoke/ccip && go test -run "Test_CCIPReorg_GreaterThanFinality_OnSource|Test_CCIPReorg_GreaterThanFinality_OnDest" ccip_reorg_test.go -timeout 18m -test.parallel=1 -count=1 -json
+    pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
+    test_env_vars:
+      E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3
+      E2E_JD_VERSION: 0.6.0
+      CCIP_V16_TEST_ENV: docker
+
   - id: smoke/ccip/ccip_token_price_updates_test.go:*
     path: integration-tests/smoke/ccip/ccip_token_price_updates_test.go
     test_env_type: docker

deployment/ccip/changeset/testhelpers/test_environment.go

@@ -40,6 +40,12 @@ const (
 	ENVTESTTYPE         = "CCIP_V16_TEST_ENV"
 )
 
+type LogMessageToIgnore struct {
+	Msg    string
+	Reason string
+	Level  zapcore.Level
+}
+
 type TestConfigs struct {
 	Type      EnvType // set by env var CCIP_V16_TEST_ENV, defaults to Memory
 	CreateJob bool
@@ -61,6 +67,18 @@ type TestConfigs struct {
 	LinkPrice                  *big.Int
 	WethPrice                  *big.Int
 	BlockTime                  time.Duration
+
+	// Test env related configs
+
+	// LogMessagesToIgnore are log messages emitted by the chainlink node that cause
+	// the test to auto-fail if they were logged.
+	// In some tests we don't want this to happen where a failure is expected, e.g
+	// we are purposely re-orging beyond finality.
+	LogMessagesToIgnore []LogMessageToIgnore
+
+	// ExtraConfigTomls contains the filenames of additional toml files to be loaded
+	// to potentially override default configs.
+	ExtraConfigTomls []string
 }
 
 func (tc *TestConfigs) Validate() error {
@@ -105,6 +123,18 @@ func DefaultTestConfigs() *TestConfigs {
 
 type TestOps func(testCfg *TestConfigs)
 
+func WithLogMessagesToIgnore(logMessages []LogMessageToIgnore) TestOps {
+	return func(testCfg *TestConfigs) {
+		testCfg.LogMessagesToIgnore = logMessages
+	}
+}
+
+func WithExtraConfigTomls(extraTomls []string) TestOps {
+	return func(testCfg *TestConfigs) {
+		testCfg.ExtraConfigTomls = extraTomls
+	}
+}
+
 func WithBlockTime(blockTime time.Duration) TestOps {
 	return func(testCfg *TestConfigs) {
 		testCfg.BlockTime = blockTime
@@ -457,6 +487,7 @@ func NewEnvironmentWithJobsAndContracts(t *testing.T, tEnv TestEnvironment) Depl
 	tc := tEnv.TestConfigs()
 	e := NewEnvironment(t, tEnv)
 	allChains := e.Env.AllChainSelectors()
+	t.Log("number of chains:", len(allChains))
 	mcmsCfg := make(map[uint64]commontypes.MCMSWithTimelockConfig)
 
 	for _, c := range e.Env.AllChainSelectors() {

deployment/environment/nodeclient/chainlink_models.go

@@ -33,17 +33,21 @@ type ResponseSlice struct {
 	Data []map[string]interface{}
 }
 
+type HealthCheck struct {
+	Name   string `json:"name"`
+	Status string `json:"status"`
+	Output string `json:"output"`
+}
+
+type HealthResponseDetail struct {
+	Type       string      `json:"type"`
+	ID         string      `json:"id"`
+	Attributes HealthCheck `json:"attributes"`
+}
+
 // HealthResponse is the generic model for services health statuses
 type HealthResponse struct {
-	Data []struct {
-		Type       string `json:"type"`
-		ID         string `json:"id"`
-		Attributes struct {
-			Name   string `json:"name"`
-			Status string `json:"status"`
-			Output string `json:"output"`
-		} `json:"attributes"`
-	} `json:"data"`
+	Data []HealthResponseDetail `json:"data"`
 }
 
 // Response is the generic model that can be used for all Chainlink API responses