upgrade test: splitter and resolver config entry in peered cluster

test/integration/consul-container/libs/assert/service.go

@@ -49,9 +49,17 @@ func CatalogNodeExists(t *testing.T, c *api.Client, nodeName string) {
 	})
 }
 
+func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) {
+	doHTTPServiceEchoes(t, ip, port, path, nil)
+}
+
+func HTTPServiceEchoesResHeader(t *testing.T, ip string, port int, path string, expectedResHeader map[string]string) {
+	doHTTPServiceEchoes(t, ip, port, path, expectedResHeader)
+}
+
 // HTTPServiceEchoes verifies that a post to the given ip/port combination returns the data
 // in the response body. Optional path can be provided to differentiate requests.
-func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) {
+func doHTTPServiceEchoes(t *testing.T, ip string, port int, path string, expectedResHeader map[string]string) {
 	const phrase = "hello"
 
 	failer := func() *retry.Timer {
@@ -82,6 +90,24 @@ func HTTPServiceEchoes(t *testing.T, ip string, port int, path string) {
 		if !strings.Contains(string(body), phrase) {
 			r.Fatal("received an incorrect response ", string(body))
 		}
+
+		for k, v := range expectedResHeader {
+			if headerValues, ok := res.Header[k]; !ok {
+				r.Fatal("expected header not found", k)
+			} else {
+				found := false
+				for _, value := range headerValues {
+					if value == v {
+						found = true
+						break
+					}
+				}
+
+				if !found {
+					r.Fatalf("header %s value not match want %s got %s ", k, v, headerValues)
+				}
+			}
+		}
 	})
 }
 

test/integration/consul-container/libs/cluster/container.go

@@ -26,8 +26,9 @@ const bootLogLine = "Consul agent running"
 const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED"
 
 // Exposed ports info
-const MaxEnvoyOnNode = 10                 // the max number of Envoy sidecar can run along with the agent, base is 19000
-const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream
+const MaxEnvoyOnNode = 10                  // the max number of Envoy sidecar can run along with the agent, base is 19000
+const ServiceUpstreamLocalBindPort = 5000  // local bind Port of service's upstream
+const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams
 
 // consulContainerNode implements the Agent interface by running a Consul agent
 // in a container.
@@ -530,6 +531,7 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR
 
 	// Envoy upstream listener
 	pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", ServiceUpstreamLocalBindPort))
+	pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", ServiceUpstreamLocalBindPort2))
 
 	// Reserve the exposed ports for Envoy admin port, e.g., 19000 - 19009
 	basePort := 19000

test/integration/consul-container/libs/service/connect.go

@@ -14,7 +14,6 @@ import (
 	"github.com/hashicorp/consul/api"
 
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
-	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
@@ -23,7 +22,7 @@ type ConnectContainer struct {
 	ctx               context.Context
 	container         testcontainers.Container
 	ip                string
-	appPort           int
+	appPort           []int
 	externalAdminPort int
 	internalAdminPort int
 	mappedPublicPort  int
@@ -52,6 +51,10 @@ func (g ConnectContainer) Export(partition, peer string, client *api.Client) err
 }
 
 func (g ConnectContainer) GetAddr() (string, int) {
+	return g.ip, g.appPort[0]
+}
+
+func (g ConnectContainer) GetAddrs() (string, []int) {
 	return g.ip, g.appPort
 }
 
@@ -139,7 +142,7 @@ func (g ConnectContainer) GetStatus() (string, error) {
 // node. The container exposes port serviceBindPort and envoy admin port
 // (19000) by mapping them onto host ports. The container's name has a prefix
 // combining datacenter and name.
-func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID string, serviceBindPort int, node libcluster.Agent) (*ConnectContainer, error) {
+func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID string, serviceBindPorts []int, node cluster.Agent) (*ConnectContainer, error) {
 	nodeConfig := node.GetConfig()
 	if nodeConfig.ScratchDir == "" {
 		return nil, fmt.Errorf("node ScratchDir is required")
@@ -209,11 +212,19 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID
 	}
 
 	var (
-		appPortStr   = strconv.Itoa(serviceBindPort)
+		appPortStrs  []string
 		adminPortStr = strconv.Itoa(internalAdminPort)
 	)
 
-	info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{appPortStr, adminPortStr})
+	for _, port := range serviceBindPorts {
+		appPortStrs = append(appPortStrs, strconv.Itoa(port))
+	}
+
+	// expose the app ports and the envoy adminPortStr on the agent container
+	exposedPorts := make([]string, len(appPortStrs))
+	copy(exposedPorts, appPortStrs)
+	exposedPorts = append(exposedPorts, adminPortStr)
+	info, err := cluster.LaunchContainerOnNode(ctx, node, req, exposedPorts)
 	if err != nil {
 		return nil, err
 	}
@@ -222,14 +233,17 @@ func NewConnectService(ctx context.Context, sidecarServiceName string, serviceID
 		ctx:               ctx,
 		container:         info.Container,
 		ip:                info.IP,
-		appPort:           info.MappedPorts[appPortStr].Int(),
 		externalAdminPort: info.MappedPorts[adminPortStr].Int(),
 		internalAdminPort: internalAdminPort,
 		serviceName:       sidecarServiceName,
 	}
 
-	fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %d\n",
-		serviceID, out.appPort, serviceBindPort)
+	for _, port := range appPortStrs {
+		out.appPort = append(out.appPort, info.MappedPorts[port].Int())
+	}
+
+	fmt.Printf("NewConnectService: name %s, mapped App Port %d, service bind port %v\n",
+		serviceID, out.appPort, serviceBindPorts)
 	fmt.Printf("NewConnectService sidecar: name %s, mapped admin port %d, admin port %d\n",
 		sidecarServiceName, out.externalAdminPort, internalAdminPort)
 

test/integration/consul-container/libs/service/examples.go

@@ -64,6 +64,10 @@ func (g exampleContainer) GetAddr() (string, int) {
 	return g.ip, g.httpPort
 }
 
+func (g exampleContainer) GetAddrs() (string, []int) {
+	return "", nil
+}
+
 func (g exampleContainer) Restart() error {
 	return fmt.Errorf("Restart Unimplemented by ConnectContainer")
 }

test/integration/consul-container/libs/service/gateway.go

@@ -53,6 +53,10 @@ func (g gatewayContainer) GetAddr() (string, int) {
 	return g.ip, g.port
 }
 
+func (g gatewayContainer) GetAddrs() (string, []int) {
+	return "", nil
+}
+
 func (g gatewayContainer) GetLogs() (string, error) {
 	rc, err := g.container.Logs(context.Background())
 	if err != nil {

test/integration/consul-container/libs/service/helpers.go

@@ -61,7 +61,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		_ = serverService.Terminate()
 	})
 
-	serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", serviceOpts.ID), serviceOpts.ID, serviceOpts.HTTPPort, node) // bindPort not used
+	serverConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", serviceOpts.ID), serviceOpts.ID, []int{serviceOpts.HTTPPort}, node) // bindPort not used
 	if err != nil {
 		return nil, nil, err
 	}
@@ -117,7 +117,7 @@ func CreateAndRegisterStaticClientSidecar(
 	}
 
 	// Create a service and proxy instance
-	clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, libcluster.ServiceUpstreamLocalBindPort, node)
+	clientConnectProxy, err := NewConnectService(context.Background(), fmt.Sprintf("%s-sidecar", StaticClientServiceName), StaticClientServiceName, []int{libcluster.ServiceUpstreamLocalBindPort}, node)
 	if err != nil {
 		return nil, err
 	}

test/integration/consul-container/libs/service/service.go

@@ -12,6 +12,7 @@ type Service interface {
 	// Export a service to the peering cluster
 	Export(partition, peer string, client *api.Client) error
 	GetAddr() (string, int)
+	GetAddrs() (string, []int)
 	// GetAdminAddr returns the external admin address
 	GetAdminAddr() (string, int)
 	GetLogs() (string, error)