Add an upgrade test for the new catalog

Currently this should be dormant and not executed. However its put in place to detect breaking changes in the future and show an example of how to do an upgrade test with integration tests structured like catalog v2.

GNUmakefile

@@ -73,6 +73,7 @@ CI_DEV_DOCKER_NAMESPACE?=hashicorpdev
 CI_DEV_DOCKER_IMAGE_NAME?=consul
 CI_DEV_DOCKER_WORKDIR?=bin/
 ################
+CONSUL_VERSION?=$(shell cat version/VERSION)
 
 TEST_MODCACHE?=1
 TEST_BUILDCACHE?=1
@@ -189,6 +190,8 @@ dev-docker: linux dev-build
 	@docker buildx use default && docker buildx build -t 'consul:local' -t '$(CONSUL_DEV_IMAGE)' \
        --platform linux/$(GOARCH) \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --load \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
 
@@ -209,6 +212,8 @@ remote-docker: check-remote-dev-image-env
 	@docker buildx use consul-builder && docker buildx build -t '$(REMOTE_DEV_IMAGE)' \
        --platform linux/amd64,linux/arm64 \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --push \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
 

test/integration/consul-container/libs/cluster/container.go

@@ -542,6 +542,10 @@ func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error {
 				continue
 			}
 		}
+
+		// if the pod is retained and therefore the IP then the grpc conn
+		// should handle reconnecting so there is no reason to close it.
+		c.closeGRPC()
 	}
 
 	var merr error
@@ -560,14 +564,17 @@ func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error {
 		c.pod = nil
 	}
 
+	return merr
+}
+
+func (c *consulContainerNode) closeGRPC() error {
 	if c.grpcConn != nil {
 		if err := c.grpcConn.Close(); err != nil {
-			merr = multierror.Append(merr, err)
+			return err
 		}
 		c.grpcConn = nil
 	}
-
-	return merr
+	return nil
 }
 
 func (c *consulContainerNode) DataDir() string {

test/integration/consul-container/libs/utils/docker.go

@@ -9,13 +9,28 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
+
+	"github.com/hashicorp/go-version"
 )
 
 // DockerExec simply shell out to the docker CLI binary on your host.
 func DockerExec(args []string, stdout io.Writer) error {
 	return cmdExec("docker", "docker", args, stdout, "")
 }
 
+// DockerImageVersion retrieves the value of the org.opencontainers.image.version label from the specified image.
+func DockerImageVersion(imageName string) (*version.Version, error) {
+	var b strings.Builder
+	err := cmdExec("docker", "docker", []string{"image", "inspect", "--format", `{{index .Config.Labels "org.opencontainers.image.version"}}`, imageName}, &b, "")
+	if err != nil {
+		return nil, err
+	}
+	output := b.String()
+
+	return version.NewVersion(strings.TrimSpace(output))
+}
+
 func cmdExec(name, binary string, args []string, stdout io.Writer, dir string) error {
 	if binary == "" {
 		panic("binary named " + name + " was not detected")

test/integration/consul-container/test/catalog/catalog_test.go

@@ -18,6 +18,7 @@ func TestCatalog(t *testing.T) {
 	cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
 		NumServers: 3,
 		BuildOpts:  &libcluster.BuildOptions{Datacenter: "dc1"},
+		Cmd:        `-hcl=experiments=["resource-apis"]`,
 	})
 
 	followers, err := cluster.Followers()

test/integration/consul-container/test/upgrade/catalog/catalog_test.go

@@ -0,0 +1,87 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package catalog
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/consul/internal/catalog/catalogtest"
+	"github.com/hashicorp/consul/proto-public/pbresource"
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/hashicorp/go-version"
+)
+
+var minCatalogResourceVersion = version.Must(version.NewVersion("v1.16.0"))
+
+const (
+	versionUndetermined = `
+Cannot determine the actual version the starting image represents.
+Scrutinze test failures to ensure that the starting version should
+actually be able to be used for creating the initial data set.
+	`
+)
+
+func maybeSkipUpgradeTest(t *testing.T, minVersion *version.Version) {
+	t.Helper()
+
+	image := utils.DockerImage(utils.GetLatestImageName(), utils.LatestVersion)
+	latestVersion, err := utils.DockerImageVersion(image)
+
+	if latestVersion != nil && latestVersion.LessThan(minVersion) {
+		t.Skipf("Upgrade test isn't applicable with version %q as the starting version", latestVersion.String())
+	}
+
+	if err != nil || latestVersion == nil {
+		t.Log(versionUndetermined)
+	}
+}
+
+// Test upgrade a cluster of latest version to the target version and ensure that the catalog still
+// functions properly. Note
+func TestCatalogUpgrade(t *testing.T) {
+	maybeSkipUpgradeTest(t, minCatalogResourceVersion)
+	t.Parallel()
+
+	const numServers = 1
+	buildOpts := &libcluster.BuildOptions{
+		ConsulImageName:      utils.GetLatestImageName(),
+		ConsulVersion:        utils.LatestVersion,
+		Datacenter:           "dc1",
+		InjectAutoEncryption: true,
+	}
+
+	cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{
+		NumServers:                1,
+		BuildOpts:                 buildOpts,
+		ApplyDefaultProxySettings: true,
+		Cmd:                       `-hcl=experiments=["resource-apis"]`,
+	})
+
+	client := cluster.APIClient(0)
+
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, numServers)
+
+	leader, err := cluster.Leader()
+	require.NoError(t, err)
+	rscClient := pbresource.NewResourceServiceClient(leader.GetGRPCConn())
+
+	// Initialize some data
+	catalogtest.PublishCatalogV1Alpha1IntegrationTestData(t, rscClient)
+
+	// upgrade the cluster to the Target version
+	t.Logf("initiating standard upgrade to version=%q", utils.TargetVersion)
+	err = cluster.StandardUpgrade(t, context.Background(), utils.GetTargetImageName(), utils.TargetVersion)
+
+	require.NoError(t, err)
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, numServers)
+
+	catalogtest.VerifyCatalogV1Alpha1IntegrationTestResults(t, rscClient)
+}