switch all client nodes in dc2 to dataplane [NET-4299]

.github/workflows/nightly-test-integrations.yml

@@ -309,13 +309,107 @@ jobs:
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
 
+  peering_commontopo-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    needs:
+      - setup
+      - dev-build
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read
+    strategy:
+      fail-fast: false
+    env:
+      ENVOY_VERSION: "1.24.6"
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
+      - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: 'go.mod'
+      - run: go env
+
+      # Get go binary from workspace
+      - name: fetch binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
+          path: .
+      - name: restore mode+x
+        run: chmod +x consul
+      - name: Build consul:local image
+        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Peering commonTopo Integration Tests
+        run: |
+          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
+          cd ./test-integ/peering_commontopo
+          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --raw-command \
+            --format=short-verbose \
+            --debug \
+            --packages="./..." \
+            -- \
+            go test \
+            -tags "${{ env.GOTAGS }}" \
+            -timeout=30m \
+            -json . \
+            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --target-version local \
+            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-version latest
+          ls -lrt
+        env:
+          # this is needed because of incompatibility between RYUK container and GHA
+          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
+          GOTESTSUM_FORMAT: standard-verbose
+          COMPOSE_INTERACTIVE_NO_CLI: 1
+          # tput complains if this isn't set to something.
+          TERM: ansi
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
+
+      - name: prepare datadog-ci
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
+          chmod +x /usr/local/bin/datadog-ci
+
+      - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
+          DD_ENV: ci
+        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
+
+
   test-integrations-success:
     needs: 
     - setup
     - dev-build
     - generate-envoy-job-matrices
     - envoy-integration-test
     - upgrade-integration-test
+    - peering_commontopo-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: ${{ always() }}
     steps:

test-integ/peering_commontopo/ac5_2_pq_failover_test.go

@@ -86,7 +86,10 @@ func (s *ac5_2PQFailoverSuite) setupDC(ct *commonTopo, clu, peerClu *topology.Cl
 		Service: NewFortioServiceWithDefaults(
 			clu.Datacenter,
 			serverSID,
-			nil,
+			func(s *topology.Service) {
+				s.EnvoyAdminPort = 0
+				s.DisableServiceMesh = true
+			},
 		),
 		Exports: []api.ServiceConsumer{{Peer: peer}},
 	}
@@ -149,7 +152,10 @@ func (s *ac5_2PQFailoverSuite) setupDC3(ct *commonTopo, clu, peer1, peer2 *topol
 		Service: NewFortioServiceWithDefaults(
 			clu.Datacenter,
 			serverSID,
-			nil,
+			func(s *topology.Service) {
+				s.EnvoyAdminPort = 0
+				s.DisableServiceMesh = true
+			},
 		),
 		Exports: func() []api.ServiceConsumer {
 			var consumers []api.ServiceConsumer

test-integ/peering_commontopo/ac7_1_rotate_gw_test.go

@@ -141,8 +141,12 @@ func (s *suiteRotateGW) setup(t *testing.T, ct *commonTopo) {
 
 	// add a second mesh gateway "new"
 	s.newMGWNodeName = fmt.Sprintf("new-%s-default-mgw", clu.Name)
+	nodeKind := topology.NodeKindClient
+	if clu.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
 	clu.Nodes = append(clu.Nodes, newTopologyMeshGatewaySet(
-		topology.NodeKindClient,
+		nodeKind,
 		"default",
 		s.newMGWNodeName,
 		1,

test-integ/peering_commontopo/asserter.go

@@ -41,6 +41,7 @@ type asserter struct {
 type sprawlLite interface {
 	HTTPClientForCluster(clusterName string) (*http.Client, error)
 	APIClientForNode(clusterName string, nid topology.NodeID, token string) (*api.Client, error)
+	APIClientForCluster(clusterName string, token string) (*api.Client, error)
 	Topology() *topology.Topology
 }
 
@@ -58,18 +59,12 @@ func (a *asserter) mustGetHTTPClient(t *testing.T, cluster string) *http.Client
 }
 
 func (a *asserter) mustGetAPIClient(t *testing.T, cluster string) *api.Client {
-	cl, err := a.apiClientFor(cluster)
+	clu := a.sp.Topology().Clusters[cluster]
+	cl, err := a.sp.APIClientForCluster(clu.Name, "")
 	require.NoError(t, err)
 	return cl
 }
 
-func (a *asserter) apiClientFor(cluster string) (*api.Client, error) {
-	clu := a.sp.Topology().Clusters[cluster]
-	// TODO: this always goes to the first client, but we might want to balance this
-	cl, err := a.sp.APIClientForNode(cluster, clu.FirstClient().ID(), "")
-	return cl, err
-}
-
 // httpClientFor returns a pre-configured http.Client that proxies requests
 // through the embedded squid instance in each LAN.
 //

test-integ/peering_commontopo/commontopo.go

@@ -52,6 +52,8 @@ type commonTopo struct {
 	services map[string]map[topology.ServiceID]struct{}
 }
 
+const agentlessDC = "dc2"
+
 func NewCommonTopo(t *testing.T) *commonTopo {
 	t.Helper()
 
@@ -84,11 +86,9 @@ func NewCommonTopo(t *testing.T) *commonTopo {
 	peerings = append(peerings, addPeerings(dc1, dc3)...)
 	peerings = append(peerings, addPeerings(dc2, dc3)...)
 
-	addMeshGateways(dc1, topology.NodeKindClient)
-	addMeshGateways(dc2, topology.NodeKindClient)
-	addMeshGateways(dc3, topology.NodeKindClient)
-	// TODO: consul-topology doesn't support this yet
-	// addMeshGateways(dc2, topology.NodeKindDataplane)
+	addMeshGateways(dc1)
+	addMeshGateways(dc2)
+	addMeshGateways(dc3)
 
 	setupGlobals(dc1)
 	setupGlobals(dc2)
@@ -131,7 +131,7 @@ func (ct *commonTopo) postLaunchChecks(t *testing.T) {
 	)
 
 	// check that exports line up as expected
-	for _, clu := range ct.Sprawl.Config().Clusters {
+	for _, clu := range ct.Sprawl.Topology().Clusters {
 		// expected exports per peer
 		type key struct {
 			peer      string
@@ -191,9 +191,6 @@ func LocalPeerName(clu *topology.Cluster, partition string) string {
 type serviceExt struct {
 	*topology.Service
 
-	// default NodeKindClient
-	NodeKind topology.NodeKind
-
 	Exports    []api.ServiceConsumer
 	Config     *api.ServiceConfigEntry
 	Intentions *api.ServiceIntentionsConfigEntry
@@ -227,8 +224,15 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 		return n
 	}
 
+	nodeKind := topology.NodeKindClient
+	// TODO: bug in deployer somewhere; it should guard against a KindDataplane node with
+	// DisableServiceMesh services on it; dataplane is only for service-mesh
+	if !svc.DisableServiceMesh && clu.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
+
 	node := &topology.Node{
-		Kind:      topology.NodeKindClient,
+		Kind:      nodeKind,
 		Name:      serviceHostnameString(clu.Datacenter, svc.ID),
 		Partition: svc.ID.Partition,
 		Addresses: []*topology.Address{
@@ -239,9 +243,6 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 		},
 		Cluster: clusterName,
 	}
-	if svc.NodeKind != "" {
-		node.Kind = svc.NodeKind
-	}
 	clu.Nodes = append(clu.Nodes, node)
 
 	// Export if necessary
@@ -265,7 +266,7 @@ func (ct *commonTopo) AddServiceNode(clu *topology.Cluster, svc serviceExt) *top
 }
 
 func (ct *commonTopo) APIClientForCluster(t *testing.T, clu *topology.Cluster) *api.Client {
-	cl, err := ct.Sprawl.APIClientForNode(clu.Name, clu.FirstClient().ID(), "")
+	cl, err := ct.Sprawl.APIClientForCluster(clu.Name, "")
 	require.NoError(t, err)
 	return cl
 }
@@ -372,10 +373,14 @@ func setupGlobals(clu *topology.Cluster) {
 
 // addMeshGateways adds a mesh gateway for every partition in the cluster.
 // Assumes that the LAN network name is equal to datacenter name.
-func addMeshGateways(c *topology.Cluster, kind topology.NodeKind) {
+func addMeshGateways(c *topology.Cluster) {
+	nodeKind := topology.NodeKindClient
+	if c.Datacenter == agentlessDC {
+		nodeKind = topology.NodeKindDataplane
+	}
 	for _, p := range c.Partitions {
 		c.Nodes = topology.MergeSlices(c.Nodes, newTopologyMeshGatewaySet(
-			kind,
+			nodeKind,
 			p.Name,
 			fmt.Sprintf("%s-%s-mgw", c.Name, p.Name),
 			1,

testing/deployer/sprawl/catalog.go

@@ -318,13 +318,45 @@ func serviceToCatalogRegistration(
 			Address: node.LocalAddress(),
 		},
 	}
+	if svc.IsMeshGateway {
+		reg.Service.Kind = api.ServiceKindMeshGateway
+		reg.Service.Proxy = &api.AgentServiceConnectProxyConfig{
+			Config: map[string]interface{}{
+				"envoy_gateway_no_default_bind":       true,
+				"envoy_gateway_bind_tagged_addresses": true,
+			},
+			MeshGateway: api.MeshGatewayConfig{
+				Mode: api.MeshGatewayModeLocal,
+			},
+		}
+	}
 	if node.HasPublicAddress() {
 		reg.TaggedAddresses = map[string]string{
 			"lan":      node.LocalAddress(),
 			"lan_ipv4": node.LocalAddress(),
 			"wan":      node.PublicAddress(),
 			"wan_ipv4": node.PublicAddress(),
 		}
+		// TODO: not sure what the difference is between these, but with just the
+		// top-level set, it appeared to not get set in either :/
+		reg.Service.TaggedAddresses = map[string]api.ServiceAddress{
+			"lan": {
+				Address: node.LocalAddress(),
+				Port:    svc.Port,
+			},
+			"lan_ipv4": {
+				Address: node.LocalAddress(),
+				Port:    svc.Port,
+			},
+			"wan": {
+				Address: node.PublicAddress(),
+				Port:    svc.Port,
+			},
+			"wan_ipv4": {
+				Address: node.PublicAddress(),
+				Port:    svc.Port,
+			},
+		}
 	}
 	if cluster.Enterprise {
 		reg.Partition = svc.ID.Partition

testing/deployer/sprawl/internal/build/docker.go

@@ -23,9 +23,10 @@ COPY --from=0 /bin/consul /bin/consul
 
 // FROM hashicorp/consul-dataplane:latest
 // COPY --from=busybox:uclibc /bin/sh /bin/sh
+// TODO: busybox:latest doesn't work, see https://hashicorp.slack.com/archives/C03EUN3QF1C/p1691784078972959
 const dockerfileDataplane = `
 ARG DATAPLANE_IMAGE
-FROM busybox:latest
+FROM busybox:1.34
 FROM ${DATAPLANE_IMAGE}
 COPY --from=0 /bin/busybox /bin/busybox
 USER 0:0

testing/deployer/sprawl/internal/tfgen/agent.go

@@ -13,14 +13,14 @@ import (
 	"github.com/hashicorp/consul/testing/deployer/topology"
 )
 
-func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
+func (g *Generator) generateAgentHCL(node *topology.Node) string {
 	if !node.IsAgent() {
-		return "", fmt.Errorf("not an agent")
+		panic("generateAgentHCL only applies to agents")
 	}
 
 	cluster, ok := g.topology.Clusters[node.Cluster]
 	if !ok {
-		return "", fmt.Errorf("no such cluster: %s", node.Cluster)
+		panic(fmt.Sprintf("no such cluster: %s", node.Cluster))
 	}
 
 	var b HCLBuilder
@@ -167,7 +167,7 @@ func (g *Generator) generateAgentHCL(node *topology.Node) (string, error) {
 		}
 	}
 
-	return b.String(), nil
+	return b.String()
 }
 
 type HCLBuilder struct {

testing/deployer/sprawl/internal/tfgen/dns.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"text/template"
 
 	"github.com/hashicorp/consul/testing/deployer/topology"
 	"github.com/hashicorp/consul/testing/deployer/util"
@@ -179,5 +178,3 @@ server IN A %s ; Consul server
 
 	return buf.Bytes()
 }
-
-var tfCorednsT = template.Must(template.ParseFS(content, "templates/container-coredns.tf.tmpl"))

testing/deployer/sprawl/internal/tfgen/docker.go

@@ -11,6 +11,9 @@ import (
 var invalidResourceName = regexp.MustCompile(`[^a-z0-9-]+`)
 
 func DockerImageResourceName(image string) string {
+	if image == "" {
+		panic(`image must not be ""`)
+	}
 	return invalidResourceName.ReplaceAllLiteralString(image, "-")
 }