NET-2954: Improve integration tests CI execution time

.circleci/config.yml

@@ -36,6 +36,9 @@ references:
     - "1.11.6"
     - "1.10.9"
     - "1.9.10"
+  consul-versions: &consul_versions
+    - "1.14"
+    - "1.15"
   images:
     # When updating the Go version, remember to also update the versions in the
     # workflows section for go-test-lib jobs.
@@ -887,7 +890,7 @@ jobs:
               -p=4 \
               -timeout=30m \
               -json \
-              ./... \
+              `go list ./... | grep -v upgrade` \
               --target-image consul \
               --target-version local \
               --latest-image consul \
@@ -906,6 +909,84 @@ jobs:
           path: *TEST_RESULTS_DIR
       - run: *notify-slack-failure
 
+  upgrade-integration-test:
+      machine:
+        image: *UBUNTU_CI_IMAGE
+        docker_layer_caching: true
+      parallelism: 3
+      resource_class: large
+      parameters:
+        consul-version:
+          type: enum
+          enum: *consul_versions
+      environment:
+        CONSUL_VERSION: << parameters.consul-version >>
+      steps:
+        - checkout
+        # Get go binary from workspace
+        - attach_workspace:
+            at: .
+        # Build the consul:local image from the already built binary
+        - run:
+            command: |
+              sudo rm -rf /usr/local/go
+              wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
+              sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
+            environment:
+              <<: *ENVIRONMENT
+        - run: *install-gotestsum
+        - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
+        - run:
+            name: Upgrade Integration Tests
+            command: |
+              mkdir -p /tmp/test-results/
+              cd ./test/integration/consul-container
+              docker run --rm consul:local consul version
+              gotestsum \
+                --raw-command \
+                --format=short-verbose \
+                --debug \
+                --rerun-fails=3 \
+                --packages="./..." \
+                -- \
+                go test \
+                -p=4 \
+                -tags "${GOTAGS}" \
+                -timeout=30m \
+                -json \
+                ./.../upgrade/ \
+                --target-image consul \
+                --target-version local \
+                --latest-image consul \
+                --latest-version $CONSUL_VERSION
+              ls -lrt
+            environment:
+              # this is needed because of incompatibility between RYUK container and circleci
+              GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml
+              GOTESTSUM_FORMAT: standard-verbose
+              COMPOSE_INTERACTIVE_NO_CLI: 1
+              # tput complains if this isn't set to something.
+              TERM: ansi
+        - store_test_results:
+            path: *TEST_RESULTS_DIR
+        - store_artifacts:
+            path: *TEST_RESULTS_DIR
+        - run: *notify-slack-failure
+
+  # Lints all *.dockerfile but don't fail at this time
+  dockerfile-lint:
+    docker:
+      - image: docker.mirror.hashicorp.services/hadolint/hadolint:latest-debian
+    steps:
+      - run: apt-get -qq update; apt-get -y install git # the hadolint container doesn't have git
+      - checkout
+      - run:
+          name: Dockefile lint
+          command: |
+            for file in $(find . -type f -name *.dockerfile); do
+              hadolint $file || true
+            done
+
   envoy-integration-test: &ENVOY_TESTS
     machine:
       image: *UBUNTU_CI_IMAGE
@@ -1210,6 +1291,13 @@ workflows:
       - compatibility-integration-test:
           requires:
             - dev-build
+      - upgrade-integration-test:
+          requires:
+            - dev-build
+          matrix:
+            parameters:
+              consul-version: *consul_versions
+
       - noop
   frontend:
     unless: << pipeline.parameters.trigger-load-test >>

test/integration/consul-container/test/upgrade/acl_node_test.go

@@ -46,10 +46,9 @@ func TestACL_Upgrade_Node_Token(t *testing.T) {
 		require.NoError(t, err)
 		libassert.CatalogNodeExists(t, client, cluster.Agents[1].GetAgentName())
 	}
-	for _, oldVersion := range UpgradeFromVersions {
-		t.Run(fmt.Sprintf("Upgrade from %s to %s", oldVersion, utils.TargetVersion),
-			func(t *testing.T) {
-				run(t, oldVersion, utils.TargetVersion)
-			})
-	}
+
+	t.Run(fmt.Sprintf("Upgrade from %s to %s", utils.LatestVersion, utils.TargetVersion),
+		func(t *testing.T) {
+			run(t, utils.LatestVersion, utils.TargetVersion)
+		})
 }

test/integration/consul-container/test/upgrade/fullstopupgrade_test.go

@@ -40,13 +40,11 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) {
 		},
 	)
 
-	for _, oldVersion := range UpgradeFromVersions {
-		tcs = append(tcs, testcase{
-			oldVersion:    oldVersion,
-			targetVersion: utils.TargetVersion,
-		},
-		)
-	}
+	tcs = append(tcs, testcase{
+		oldVersion:    utils.LatestVersion,
+		targetVersion: utils.TargetVersion,
+	},
+	)
 
 	run := func(t *testing.T, tc testcase) {
 		configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{

test/integration/consul-container/test/healthcheck/healthcheck_test.go → test/integration/consul-container/test/upgrade/healthcheck_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/go-version"
 	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/consul/api"
@@ -18,6 +19,12 @@ import (
 func TestTargetServersWithLatestGAClients(t *testing.T) {
 	t.Parallel()
 
+	fromVersion, err := version.NewVersion(utils.LatestVersion)
+	require.NoError(t, err)
+	if fromVersion.LessThan(utils.Version_1_14) {
+		return
+	}
+
 	const (
 		numServers = 3
 		numClients = 1

test/integration/consul-container/test/upgrade/ingress_gateway_test.go

@@ -270,17 +270,12 @@ func TestIngressGateway_UpgradeToTarget_fromLatest(t *testing.T) {
 		})
 	}
 
-	for _, oldVersion := range UpgradeFromVersions {
-		// copy to avoid lint loopclosure
-		oldVersion := oldVersion
-
-		t.Run(fmt.Sprintf("Upgrade from %s to %s", oldVersion, utils.TargetVersion),
-			func(t *testing.T) {
-				t.Parallel()
-				run(t, oldVersion, utils.TargetVersion)
-			})
-		time.Sleep(1 * time.Second)
-	}
+	t.Run(fmt.Sprintf("Upgrade from %s to %s", utils.LatestVersion, utils.TargetVersion),
+		func(t *testing.T) {
+			t.Parallel()
+			run(t, utils.LatestVersion, utils.TargetVersion)
+		})
+	time.Sleep(1 * time.Second)
 }
 
 func mappedHTTPGET(t *testing.T, uri string, mappedPort int, header http.Header) *http.Response {

test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go

@@ -12,7 +12,6 @@ import (
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
-	"github.com/hashicorp/consul/test/integration/consul-container/test/upgrade"
 	"github.com/stretchr/testify/require"
 )
 
@@ -386,13 +385,11 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 		tc.extraAssertion(staticClientProxy)
 	}
 
-	for _, oldVersion := range upgrade.UpgradeFromVersions {
-		for _, tc := range tcs {
-			t.Run(fmt.Sprintf("%s upgrade from %s to %s", tc.name, oldVersion, utils.TargetVersion),
-				func(t *testing.T) {
-					run(t, tc, oldVersion, utils.TargetVersion)
-				})
-		}
+	for _, tc := range tcs {
+		t.Run(fmt.Sprintf("%s upgrade from %s to %s", tc.name, utils.LatestVersion, utils.TargetVersion),
+			func(t *testing.T) {
+				run(t, tc, utils.LatestVersion, utils.TargetVersion)
+			})
 	}
 }
 

test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go

@@ -88,10 +88,8 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 	}
 
-	for _, oldVersion := range UpgradeFromVersions {
-		t.Run(fmt.Sprintf("Upgrade from %s to %s", oldVersion, utils.TargetVersion),
-			func(t *testing.T) {
-				run(t, oldVersion, utils.TargetVersion)
-			})
-	}
+	t.Run(fmt.Sprintf("Upgrade from %s to %s", utils.LatestVersion, utils.TargetVersion),
+		func(t *testing.T) {
+			run(t, utils.LatestVersion, utils.TargetVersion)
+		})
 }

test/integration/consul-container/test/upgrade/peering_http_test.go

@@ -30,6 +30,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		// common resources includes static-client in dialing cluster, and static-server in accepting cluster.
 		extraAssertion func(int)
 	}
+
 	tcs := []testcase{
 		// {
 		//  TODO: API changed from 1.13 to 1.14 in , PeerName to Peer
@@ -372,13 +373,11 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		tc.extraAssertion(appPort)
 	}
 
-	for _, oldVersion := range UpgradeFromVersions {
-		for _, tc := range tcs {
-			t.Run(fmt.Sprintf("%s upgrade from %s to %s", tc.name, oldVersion, utils.TargetVersion),
-				func(t *testing.T) {
-					run(t, tc, oldVersion, utils.TargetVersion)
-				})
-		}
+	for _, tc := range tcs {
+		t.Run(fmt.Sprintf("%s upgrade from %s to %s", tc.name, utils.LatestVersion, utils.TargetVersion),
+			func(t *testing.T) {
+				run(t, tc, utils.LatestVersion, utils.TargetVersion)
+			})
 	}
 }