From 1b98c7e3341cebc90afbfa98295f98990186be4a Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Tue, 20 Sep 2022 15:29:15 -0400 Subject: [PATCH] Review feedback from Iryna --- CONTRIBUTING.md | 2 +- acceptance/go.sum | 3 - .../tests/mesh-gateway/mesh_gateway_test.go | 294 ------------------ .../partitions/partitions_connect_test.go | 3 +- .../tests/partitions/partitions_sync_test.go | 3 +- .../peering_connect_namespaces_test.go | 8 +- .../tests/peering/peering_connect_test.go | 8 +- .../terminating_gateway_destinations_test.go | 6 +- .../main_test.go | 4 +- .../wan-federation/wan_federation_test.go | 185 +++++++++++ .../templates/mesh-gateway-deployment.yaml | 19 +- .../templates/mesh-gateway-service.yaml | 2 +- .../test/unit/mesh-gateway-clusterrole.bats | 2 - .../test/unit/mesh-gateway-deployment.bats | 75 ++--- .../test/unit/mesh-gateway-service.bats | 14 +- charts/consul/values.yaml | 8 - control-plane/connect-inject/annotations.go | 32 +- .../connect-inject/endpoints_controller.go | 64 ++-- .../endpoints_controller_ent_test.go | 106 ++++++- .../endpoints_controller_test.go | 144 ++++++++- control-plane/go.mod | 4 +- control-plane/go.sum | 6 - .../subcommand/connect-init/command.go | 15 +- .../subcommand/connect-init/command_test.go | 14 +- 24 files changed, 518 insertions(+), 503 deletions(-) delete mode 100644 acceptance/tests/mesh-gateway/mesh_gateway_test.go rename acceptance/tests/{mesh-gateway => wan-federation}/main_test.go (72%) create mode 100644 acceptance/tests/wan-federation/wan_federation_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec825cafd2..b65aa0eaed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -876,7 +876,7 @@ func TestExample(t *testing.T) { } ``` -Please see [mesh gateway tests](acceptance/tests/mesh-gateway/mesh_gateway_test.go) +Please see [wan federation tests](acceptance/tests/wan-federation/wan_federation_test.go) for an example of how to use write a test that uses multiple contexts. #### Writing Assertions diff --git a/acceptance/go.sum b/acceptance/go.sum index c4e8c9b20e..4097158e83 100644 --- a/acceptance/go.sum +++ b/acceptance/go.sum @@ -324,11 +324,8 @@ github.com/gruntwork-io/terratest v0.31.2 h1:xvYHA80MUq5kx670dM18HInewOrrQrAN+Xb github.com/gruntwork-io/terratest v0.31.2/go.mod h1:EEgJie28gX/4AD71IFqgMj6e99KP5mi81hEtzmDjxTo= github.com/hashicorp/consul-k8s/control-plane v0.0.0-20211207212234-aea9efea5638 h1:z68s6H6O3RjxDmNvou/2/3UBrsJkrMcNzI0IQN5scAM= github.com/hashicorp/consul-k8s/control-plane v0.0.0-20211207212234-aea9efea5638/go.mod h1:7ZeaiADGbvJDuoWAT8UKj6KCcLsFUk+34OkUGMVtdXg= -github.com/hashicorp/consul/api v1.10.1-0.20220726130109-a6a79d6811df h1:CCh336XUGPaMNqMkzRAeEL0BFin7LhX729xy3IxE5Y4= -github.com/hashicorp/consul/api v1.10.1-0.20220726130109-a6a79d6811df/go.mod h1:bcaw5CSZ7NE9qfOfKCI1xb7ZKjzu/MyvQkCLTfqLqxQ= github.com/hashicorp/consul/api v1.14.0 h1:Y64GIJ8hYTu+tuGekwO4G4ardXoiCivX9wv1iP/kihk= github.com/hashicorp/consul/api v1.14.0/go.mod h1:bcaw5CSZ7NE9qfOfKCI1xb7ZKjzu/MyvQkCLTfqLqxQ= -github.com/hashicorp/consul/sdk v0.10.0 h1:rGLEh2AWK4K0KCMvqWAz2EYxQqgciIfMagWZ0nVe5MI= github.com/hashicorp/consul/sdk v0.10.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4= github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= diff --git a/acceptance/tests/mesh-gateway/mesh_gateway_test.go b/acceptance/tests/mesh-gateway/mesh_gateway_test.go deleted file mode 100644 index 98125374af..0000000000 --- a/acceptance/tests/mesh-gateway/mesh_gateway_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package meshgateway - -import ( - "context" - "fmt" - "testing" - - "github.com/hashicorp/consul-k8s/acceptance/framework/consul" - "github.com/hashicorp/consul-k8s/acceptance/framework/environment" - "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" - "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" - "github.com/hashicorp/consul-k8s/acceptance/framework/logger" - "github.com/hashicorp/consul/api" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const StaticClientName = "static-client" - -// Test that Connect and wan federation over mesh gateways work in a default installation -// i.e. without ACLs because TLS is required for WAN federation over mesh gateways. -func TestMeshGatewayDefault(t *testing.T) { - env := suite.Environment() - cfg := suite.Config() - - primaryContext := env.DefaultContext(t) - secondaryContext := env.Context(t, environment.SecondaryContextName) - - primaryHelmValues := map[string]string{ - "global.datacenter": "dc1", - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", - "global.tls.enabled": "true", - "global.tls.httpsOnly": "false", - "global.federation.enabled": "true", - "global.federation.createFederationSecret": "true", - - "connectInject.enabled": "true", - "connectInject.replicas": "1", - "controller.enabled": "true", - - "meshGateway.enabled": "true", - "meshGateway.replicas": "1", - } - - if cfg.UseKind { - primaryHelmValues["meshGateway.service.type"] = "NodePort" - primaryHelmValues["meshGateway.service.nodePort"] = "30000" - } - - releaseName := helpers.RandomName() - - // Install the primary consul cluster in the default kubernetes context - primaryConsulCluster := consul.NewHelmCluster(t, primaryHelmValues, primaryContext, cfg, releaseName) - primaryConsulCluster.Create(t) - - // Get the federation secret from the primary cluster and apply it to secondary cluster - federationSecretName := fmt.Sprintf("%s-consul-federation", releaseName) - logger.Logf(t, "retrieving federation secret %s from the primary cluster and applying to the secondary", federationSecretName) - federationSecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(context.Background(), federationSecretName, metav1.GetOptions{}) - federationSecret.ResourceVersion = "" - require.NoError(t, err) - _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{}) - require.NoError(t, err) - - // Create secondary cluster - secondaryHelmValues := map[string]string{ - "global.datacenter": "dc2", - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", - "global.tls.enabled": "true", - "global.tls.httpsOnly": "false", - "global.tls.caCert.secretName": federationSecretName, - "global.tls.caCert.secretKey": "caCert", - "global.tls.caKey.secretName": federationSecretName, - "global.tls.caKey.secretKey": "caKey", - - "global.federation.enabled": "true", - - "server.extraVolumes[0].type": "secret", - "server.extraVolumes[0].name": federationSecretName, - "server.extraVolumes[0].load": "true", - "server.extraVolumes[0].items[0].key": "serverConfigJSON", - "server.extraVolumes[0].items[0].path": "config.json", - - "connectInject.enabled": "true", - "connectInject.replicas": "1", - "controller.enabled": "true", - - "meshGateway.enabled": "true", - "meshGateway.replicas": "1", - } - - if cfg.UseKind { - secondaryHelmValues["meshGateway.service.type"] = "NodePort" - secondaryHelmValues["meshGateway.service.nodePort"] = "30000" - } - - // Install the secondary consul cluster in the secondary kubernetes context - secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName) - secondaryConsulCluster.Create(t) - - if cfg.UseKind { - // This is a temporary workaround that seems to fix mesh gateway tests on kind 1.22.x. - // TODO (ishustava): we need to investigate this further and remove once we've found the issue. - k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "restart", fmt.Sprintf("sts/%s-consul-server", releaseName)) - k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "status", fmt.Sprintf("sts/%s-consul-server", releaseName)) - } - - primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, false) - secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, false) - - // Verify federation between servers - logger.Log(t, "verifying federation was successful") - helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, false) - - // Create a ProxyDefaults resource to configure services to use the mesh - // gateways. - logger.Log(t, "creating proxy-defaults config") - kustomizeDir := "../fixtures/bases/mesh-gateway" - k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { - k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir) - }) - - // Check that we can connect services over the mesh gateways - logger.Log(t, "creating static-server in dc2") - k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - - logger.Log(t, "creating static-client in dc1") - k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") - - logger.Log(t, "checking that connection is successful") - k8s.CheckStaticServerConnectionSuccessful(t, primaryContext.KubectlOptions(t), StaticClientName, "http://localhost:1234") -} - -// Test that Connect and wan federation over mesh gateways work in a secure installation, -// with ACLs and TLS with and without auto-encrypt enabled. -func TestMeshGatewaySecure(t *testing.T) { - cases := []struct { - name string - }{ - { - "with ACLs and TLS", - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - env := suite.Environment() - cfg := suite.Config() - - primaryContext := env.DefaultContext(t) - secondaryContext := env.Context(t, environment.SecondaryContextName) - - primaryHelmValues := map[string]string{ - "global.datacenter": "dc1", - "global.tls.enabled": "true", - - "global.acls.manageSystemACLs": "true", - "global.acls.createReplicationToken": "true", - - "global.federation.enabled": "true", - "global.federation.createFederationSecret": "true", - - "connectInject.enabled": "true", - "connectInject.replicas": "1", - "controller.enabled": "true", - - "meshGateway.enabled": "true", - "meshGateway.replicas": "1", - } - - if cfg.UseKind { - primaryHelmValues["meshGateway.service.type"] = "NodePort" - primaryHelmValues["meshGateway.service.nodePort"] = "30000" - } - - releaseName := helpers.RandomName() - - // Install the primary consul cluster in the default kubernetes context - primaryConsulCluster := consul.NewHelmCluster(t, primaryHelmValues, primaryContext, cfg, releaseName) - primaryConsulCluster.Create(t) - - // Get the federation secret from the primary cluster and apply it to secondary cluster - federationSecretName := fmt.Sprintf("%s-consul-federation", releaseName) - logger.Logf(t, "retrieving federation secret %s from the primary cluster and applying to the secondary", federationSecretName) - federationSecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(context.Background(), federationSecretName, metav1.GetOptions{}) - require.NoError(t, err) - federationSecret.ResourceVersion = "" - _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{}) - require.NoError(t, err) - - var k8sAuthMethodHost string - // When running on kind, the kube API address in kubeconfig will have a localhost address - // which will not work from inside the container. That's why we need to use the endpoints address instead - // which will point the node IP. - if cfg.UseKind { - // The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service. - kubernetesEndpoint, err := secondaryContext.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) - require.NoError(t, err) - k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port) - } else { - k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryContext.KubectlOptions(t)) - } - - // Create secondary cluster - secondaryHelmValues := map[string]string{ - "global.datacenter": "dc2", - - "global.tls.enabled": "true", - "global.tls.httpsOnly": "false", - "global.tls.caCert.secretName": federationSecretName, - "global.tls.caCert.secretKey": "caCert", - "global.tls.caKey.secretName": federationSecretName, - "global.tls.caKey.secretKey": "caKey", - - "global.acls.manageSystemACLs": "true", - "global.acls.replicationToken.secretName": federationSecretName, - "global.acls.replicationToken.secretKey": "replicationToken", - - "global.federation.enabled": "true", - "global.federation.k8sAuthMethodHost": k8sAuthMethodHost, - "global.federation.primaryDatacenter": "dc1", - - "server.extraVolumes[0].type": "secret", - "server.extraVolumes[0].name": federationSecretName, - "server.extraVolumes[0].load": "true", - "server.extraVolumes[0].items[0].key": "serverConfigJSON", - "server.extraVolumes[0].items[0].path": "config.json", - - "connectInject.enabled": "true", - "connectInject.replicas": "1", - "controller.enabled": "true", - - "meshGateway.enabled": "true", - "meshGateway.replicas": "1", - } - - if cfg.UseKind { - secondaryHelmValues["meshGateway.service.type"] = "NodePort" - secondaryHelmValues["meshGateway.service.nodePort"] = "30000" - } - - // Install the secondary consul cluster in the secondary kubernetes context - secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName) - secondaryConsulCluster.Create(t) - - if cfg.UseKind { - // This is a temporary workaround that seems to fix mesh gateway tests on kind 1.22.x. - // TODO (ishustava): we need to investigate this further and remove once we've found the issue. - k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "restart", fmt.Sprintf("sts/%s-consul-server", releaseName)) - k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "status", fmt.Sprintf("sts/%s-consul-server", releaseName)) - } - - primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, true) - secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, true) - - // Verify federation between servers - logger.Log(t, "verifying federation was successful") - helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, true) - - // Create a ProxyDefaults resource to configure services to use the mesh - // gateways. - logger.Log(t, "creating proxy-defaults config") - kustomizeDir := "../fixtures/bases/mesh-gateway" - k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { - k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), kustomizeDir) - }) - - // Check that we can connect services over the mesh gateways - logger.Log(t, "creating static-server in dc2") - k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - - logger.Log(t, "creating static-client in dc1") - k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") - - logger.Log(t, "creating intention") - _, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ - Kind: api.ServiceIntentions, - Name: "static-server", - Sources: []*api.SourceIntention{ - { - Name: StaticClientName, - Action: api.IntentionActionAllow, - }, - }, - }, nil) - require.NoError(t, err) - - logger.Log(t, "checking that connection is successful") - k8s.CheckStaticServerConnectionSuccessful(t, primaryContext.KubectlOptions(t), StaticClientName, "http://localhost:1234") - }) - } -} diff --git a/acceptance/tests/partitions/partitions_connect_test.go b/acceptance/tests/partitions/partitions_connect_test.go index 2556d5dd25..0d6a732683 100644 --- a/acceptance/tests/partitions/partitions_connect_test.go +++ b/acceptance/tests/partitions/partitions_connect_test.go @@ -25,6 +25,8 @@ const StaticClientNamespace = "ns2" // Test that Connect works in a default and ACLsEnabled installations for X-Partition and in-partition networking. func TestPartitions_Connect(t *testing.T) { + t.Skipf("currently unsupported in agentless") + env := suite.Environment() cfg := suite.Config() @@ -92,7 +94,6 @@ func TestPartitions_Connect(t *testing.T) { commonHelmValues := map[string]string{ "global.adminPartitions.enabled": "true", - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", "global.enableConsulNamespaces": "true", "global.tls.enabled": "true", diff --git a/acceptance/tests/partitions/partitions_sync_test.go b/acceptance/tests/partitions/partitions_sync_test.go index 5e3c6826f9..4baac35340 100644 --- a/acceptance/tests/partitions/partitions_sync_test.go +++ b/acceptance/tests/partitions/partitions_sync_test.go @@ -21,6 +21,8 @@ import ( // Test that Sync Catalog works in a default and ACLsEnabled installations for partitions. func TestPartitions_Sync(t *testing.T) { + t.Skipf("currently unsupported in agentless") + env := suite.Environment() cfg := suite.Config() @@ -87,7 +89,6 @@ func TestPartitions_Sync(t *testing.T) { commonHelmValues := map[string]string{ "global.adminPartitions.enabled": "true", - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", "global.enableConsulNamespaces": "true", "global.tls.enabled": "true", diff --git a/acceptance/tests/peering/peering_connect_namespaces_test.go b/acceptance/tests/peering/peering_connect_namespaces_test.go index 3c229cd4b6..fd24a322b6 100644 --- a/acceptance/tests/peering/peering_connect_namespaces_test.go +++ b/acceptance/tests/peering/peering_connect_namespaces_test.go @@ -27,6 +27,8 @@ const staticClientNamespace = "ns2" // Test that Connect works in installations for X-Peers networking. func TestPeering_ConnectNamespaces(t *testing.T) { + t.Skipf("currently unsupported in agentless") + env := suite.Environment() cfg := suite.Config() @@ -99,9 +101,9 @@ func TestPeering_ConnectNamespaces(t *testing.T) { commonHelmValues := map[string]string{ "global.peering.enabled": "true", "global.enableConsulNamespaces": "true", - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", - "global.tls.enabled": "true", - "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), + + "global.tls.enabled": "true", + "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), "global.acls.manageSystemACLs": strconv.FormatBool(c.ACLsEnabled), diff --git a/acceptance/tests/peering/peering_connect_test.go b/acceptance/tests/peering/peering_connect_test.go index 13dac5636c..6de682e4c4 100644 --- a/acceptance/tests/peering/peering_connect_test.go +++ b/acceptance/tests/peering/peering_connect_test.go @@ -22,6 +22,8 @@ import ( // Test that Connect works in installations for X-Peers networking. func TestPeering_Connect(t *testing.T) { + t.Skipf("currently unsupported in agentless") + env := suite.Environment() cfg := suite.Config() @@ -59,9 +61,9 @@ func TestPeering_Connect(t *testing.T) { commonHelmValues := map[string]string{ "global.peering.enabled": "true", - "global.tls.enabled": "true", - "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), - "global.image": "thisisnotashwin/consul@sha256:477091fe84cde79a68a37cc9cc69fb7a5ab35e647a0f5f2632451ace5ecc5e7c", + "global.tls.enabled": "true", + "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), + "global.acls.manageSystemACLs": strconv.FormatBool(c.ACLsEnabled), "connectInject.enabled": "true", diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go index d18d971743..2554cd4a5f 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go @@ -2,9 +2,6 @@ package terminatinggateway import ( "fmt" - "github.com/hashicorp/consul/api" - "github.com/hashicorp/go-version" - "github.com/stretchr/testify/require" "strconv" "testing" @@ -12,6 +9,9 @@ import ( "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/require" ) // Test that egress Destinations route through terminating gateways. diff --git a/acceptance/tests/mesh-gateway/main_test.go b/acceptance/tests/wan-federation/main_test.go similarity index 72% rename from acceptance/tests/mesh-gateway/main_test.go rename to acceptance/tests/wan-federation/main_test.go index fb8935441e..197a3181e8 100644 --- a/acceptance/tests/mesh-gateway/main_test.go +++ b/acceptance/tests/wan-federation/main_test.go @@ -1,4 +1,4 @@ -package meshgateway +package wanfederation import ( "fmt" @@ -16,7 +16,7 @@ func TestMain(m *testing.M) { if suite.Config().EnableMultiCluster { os.Exit(suite.Run()) } else { - fmt.Println("Skipping mesh gateway tests because -enable-multi-cluster is not set") + fmt.Println("Skipping wan federation tests because -enable-multi-cluster is not set") os.Exit(0) } } diff --git a/acceptance/tests/wan-federation/wan_federation_test.go b/acceptance/tests/wan-federation/wan_federation_test.go new file mode 100644 index 0000000000..b281b833d7 --- /dev/null +++ b/acceptance/tests/wan-federation/wan_federation_test.go @@ -0,0 +1,185 @@ +package wanfederation + +import ( + "context" + "fmt" + "strconv" + "testing" + + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const StaticClientName = "static-client" + +// Test that Connect and wan federation over mesh gateways work in a default installation +// i.e. without ACLs because TLS is required for WAN federation over mesh gateways. +func TestWANFederation(t *testing.T) { + cases := []struct { + name string + secure bool + }{ + { + name: "secure", + secure: true, + }, + { + name: "default", + secure: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + env := suite.Environment() + cfg := suite.Config() + + primaryContext := env.DefaultContext(t) + secondaryContext := env.Context(t, environment.SecondaryContextName) + + primaryHelmValues := map[string]string{ + "global.datacenter": "dc1", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": "false", + "global.federation.enabled": "true", + "global.federation.createFederationSecret": "true", + + "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), + "global.acls.createReplicationToken": strconv.FormatBool(c.secure), + + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + } + + if cfg.UseKind { + primaryHelmValues["meshGateway.service.type"] = "NodePort" + primaryHelmValues["meshGateway.service.nodePort"] = "30000" + } + + releaseName := helpers.RandomName() + + // Install the primary consul cluster in the default kubernetes context + primaryConsulCluster := consul.NewHelmCluster(t, primaryHelmValues, primaryContext, cfg, releaseName) + primaryConsulCluster.Create(t) + + // Get the federation secret from the primary cluster and apply it to secondary cluster + federationSecretName := fmt.Sprintf("%s-consul-federation", releaseName) + logger.Logf(t, "retrieving federation secret %s from the primary cluster and applying to the secondary", federationSecretName) + federationSecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(context.Background(), federationSecretName, metav1.GetOptions{}) + federationSecret.ResourceVersion = "" + require.NoError(t, err) + _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create secondary cluster + secondaryHelmValues := map[string]string{ + "global.datacenter": "dc2", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": "false", + "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), + "global.tls.caCert.secretName": federationSecretName, + "global.tls.caCert.secretKey": "caCert", + "global.tls.caKey.secretName": federationSecretName, + "global.tls.caKey.secretKey": "caKey", + + "global.federation.enabled": "true", + + "server.extraVolumes[0].type": "secret", + "server.extraVolumes[0].name": federationSecretName, + "server.extraVolumes[0].load": "true", + "server.extraVolumes[0].items[0].key": "serverConfigJSON", + "server.extraVolumes[0].items[0].path": "config.json", + + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + } + + var k8sAuthMethodHost string + // When running on kind, the kube API address in kubeconfig will have a localhost address + // which will not work from inside the container. That's why we need to use the endpoints address instead + // which will point the node IP. + if cfg.UseKind { + // The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service. + kubernetesEndpoint, err := secondaryContext.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) + require.NoError(t, err) + k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port) + } else { + k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryContext.KubectlOptions(t)) + } + + if c.secure { + secondaryHelmValues["global.acls.replicationToken.secretName"] = federationSecretName + secondaryHelmValues["global.acls.replicationToken.secretKey"] = "replicationToken" + secondaryHelmValues["global.federation.k8sAuthMethodHost"] = k8sAuthMethodHost + secondaryHelmValues["global.federation.primaryDatacenter"] = "dc1" + } + + if cfg.UseKind { + secondaryHelmValues["meshGateway.service.type"] = "NodePort" + secondaryHelmValues["meshGateway.service.nodePort"] = "30000" + } + + // Install the secondary consul cluster in the secondary kubernetes context + secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName) + secondaryConsulCluster.Create(t) + + primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, c.secure) + secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, c.secure) + + // Verify federation between servers + logger.Log(t, "verifying federation was successful") + helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, c.secure) + + // Create a ProxyDefaults resource to configure services to use the mesh + // gateways. + logger.Log(t, "creating proxy-defaults config") + kustomizeDir := "../fixtures/bases/mesh-gateway" + k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir) + }) + + // Check that we can connect services over the mesh gateways + logger.Log(t, "creating static-server in dc2") + k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + + logger.Log(t, "creating static-client in dc1") + k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + + if c.secure { + logger.Log(t, "creating intention") + _, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ + Kind: api.ServiceIntentions, + Name: "static-server", + Sources: []*api.SourceIntention{ + { + Name: StaticClientName, + Action: api.IntentionActionAllow, + }, + }, + }, nil) + require.NoError(t, err) + } + + logger.Log(t, "checking that connection is successful") + k8s.CheckStaticServerConnectionSuccessful(t, primaryContext.KubectlOptions(t), StaticClientName, "http://localhost:1234") + }) + } +} diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml index f20f93e195..287b9f1fa8 100644 --- a/charts/consul/templates/mesh-gateway-deployment.yaml +++ b/charts/consul/templates/mesh-gateway-deployment.yaml @@ -2,9 +2,7 @@ {{- if not .Values.connectInject.enabled }}{{ fail "connectInject.enabled must be true" }}{{ end -}} {{- if and .Values.global.acls.manageSystemACLs (ne .Values.meshGateway.consulServiceName "") (ne .Values.meshGateway.consulServiceName "mesh-gateway") }}{{ fail "if global.acls.manageSystemACLs is true, meshGateway.consulServiceName cannot be set" }}{{ end -}} {{- if .Values.meshGateway.globalMode }}{{ fail "meshGateway.globalMode is no longer supported; instead, you must migrate to CRDs (see www.consul.io/docs/k8s/crds/upgrade-to-crds)" }}{{ end -}} -{{- if .Values.global.lifecycleSidecarContainer }}{{ fail "global.lifecycleSidecarContainer has been renamed to global.consulSidecarContainer. Please set values using global.consulSidecarContainer." }}{{ end }} {{- if and .Values.global.adminPartitions.enabled (not .Values.global.enableConsulNamespaces) }}{{ fail "global.enableConsulNamespaces must be true if global.adminPartitions.enabled=true" }}{{ end }} -{{- if and (eq .Values.meshGateway.wanAddress.source "Service") (not .Values.meshGateway.service.enabled) }}{{ fail "if meshGateway.wanAddress.source=Service then meshGateway.service.enabled must be set to true" }}{{ end }} {{- if and (eq .Values.meshGateway.wanAddress.source "Static") (eq .Values.meshGateway.wanAddress.static "") }}{{ fail "if meshGateway.wanAddress.source=Static then meshGateway.wanAddress.static cannot be empty" }}{{ end }} {{- if and (eq .Values.meshGateway.wanAddress.source "Service") (eq .Values.meshGateway.service.type "NodePort") (not .Values.meshGateway.service.nodePort) }}{{ fail "if meshGateway.wanAddress.source=Service and meshGateway.service.type=NodePort, meshGateway.service.nodePort must be set" }}{{ end }} apiVersion: apps/v1 @@ -37,9 +35,10 @@ spec: annotations: "consul.hashicorp.com/connect-inject": "false" "consul.hashicorp.com/gateway-kind": "mesh" - "consul.hashicorp.com/mesh-gateway-source": "{{ .Values.meshGateway.wanAddress.source }}" + "consul.hashicorp.com/mesh-gateway-consul-service-name": "{{ .Values.meshGateway.consulServiceName }}" + "consul.hashicorp.com/mesh-gateway-wan-address-source": "{{ .Values.meshGateway.wanAddress.source }}" "consul.hashicorp.com/mesh-gateway-container-port": "{{ .Values.meshGateway.containerPort }}" - "consul.hashicorp.com/mesh-gateway-wan-address": "{{ .Values.meshGateway.wanAddress.static }}" + "consul.hashicorp.com/mesh-gateway-wan-address-static": "{{ .Values.meshGateway.wanAddress.static }}" {{- if eq .Values.meshGateway.wanAddress.source "Service" }} {{- if eq .Values.meshGateway.service.type "NodePort" }} "consul.hashicorp.com/mesh-gateway-wan-port": "{{ .Values.meshGateway.service.nodePort }}" @@ -114,14 +113,6 @@ spec: - name: mesh-gateway-init image: {{ .Values.global.imageK8S }} env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: @@ -146,7 +137,7 @@ spec: - | consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ -consul-api-timeout={{ .Values.global.consulAPITimeout }} \ - -gateway -gateway-kind="mesh-gateway" \ + -gateway-kind="mesh-gateway" \ -consul-node-name="k8s-service-mesh" \ -proxy-id-file=/consul/service/proxy-id \ {{- if .Values.global.adminPartitions.enabled }} @@ -177,7 +168,7 @@ spec: {{- end }} containers: - name: mesh-gateway - image: {{ .Values.global.imageConsulDataplane }} + image: {{ .Values.global.imageConsulDataplane | quote }} {{- if .Values.meshGateway.resources }} resources: {{- if eq (typeOf .Values.meshGateway.resources) "string" }} diff --git a/charts/consul/templates/mesh-gateway-service.yaml b/charts/consul/templates/mesh-gateway-service.yaml index 7bd7ec2acc..5fdceca8df 100644 --- a/charts/consul/templates/mesh-gateway-service.yaml +++ b/charts/consul/templates/mesh-gateway-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.meshGateway.enabled .Values.meshGateway.service.enabled }} +{{- if and .Values.meshGateway.enabled }} apiVersion: v1 kind: Service metadata: diff --git a/charts/consul/test/unit/mesh-gateway-clusterrole.bats b/charts/consul/test/unit/mesh-gateway-clusterrole.bats index da4d0bdb2c..3cb5826969 100644 --- a/charts/consul/test/unit/mesh-gateway-clusterrole.bats +++ b/charts/consul/test/unit/mesh-gateway-clusterrole.bats @@ -38,7 +38,6 @@ load _helpers -s templates/mesh-gateway-clusterrole.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=LoadBalancer' \ --set 'meshGateway.wanAddress.source=Service' \ . | tee /dev/stderr | @@ -66,7 +65,6 @@ load _helpers --set 'connectInject.enabled=true' \ --set 'global.acls.manageSystemACLs=true' \ --set 'global.enablePodSecurityPolicies=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=LoadBalancer' \ --set 'meshGateway.wanAddress.source=Service' \ . | tee /dev/stderr | diff --git a/charts/consul/test/unit/mesh-gateway-deployment.bats b/charts/consul/test/unit/mesh-gateway-deployment.bats index 5e4165f57a..f7e4749069 100755 --- a/charts/consul/test/unit/mesh-gateway-deployment.bats +++ b/charts/consul/test/unit/mesh-gateway-deployment.bats @@ -44,7 +44,7 @@ load _helpers --set 'connectInject.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.template.metadata.annotations | length' | tee /dev/stderr) - [ "${actual}" = "6" ] + [ "${actual}" = "7" ] } @test "meshGateway/Deployment: extra annotations can be set" { @@ -57,7 +57,7 @@ load _helpers key2: value2' \ . | tee /dev/stderr | yq -r '.spec.template.metadata.annotations | length' | tee /dev/stderr) - [ "${actual}" = "8" ] + [ "${actual}" = "9" ] } #-------------------------------------------------------------------- @@ -296,21 +296,6 @@ key2: value2' \ [ "${actual}" = "ClusterFirst" ] } -#-------------------------------------------------------------------- -# dataplane Image - -@test "meshGateway/Deployment: consul/dataplane image has default" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) - [[ "${actual}" =~ "ishustava/consul-dataplane:latest" ]] - -} - #-------------------------------------------------------------------- # resources @@ -492,11 +477,11 @@ key2: value2' \ [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[4].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + yq '[.env[2].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[4].value] | any(contains("http://release-name-consul-server.default.svc:8500"))' | tee /dev/stderr) + yq '[.env[2].value] | any(contains("http://release-name-consul-server.default.svc:8500"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | @@ -524,11 +509,11 @@ key2: value2' \ [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[5].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + yq '[.env[3].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[5].value] | any(contains("https://release-name-consul-server.default.svc:8501"))' | tee /dev/stderr) + yq '[.env[3].value] | any(contains("https://release-name-consul-server.default.svc:8501"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | @@ -567,15 +552,15 @@ key2: value2' \ [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[4].name] | any(contains("CONSUL_CACERT"))' | tee /dev/stderr) + yq '[.env[2].name] | any(contains("CONSUL_CACERT"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[5].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) + yq '[.env[3].name] | any(contains("CONSUL_HTTP_ADDR"))' | tee /dev/stderr) [ "${actual}" = "true" ] local actual=$(echo $object | - yq '[.env[5].value] | any(contains("https://release-name-consul-server.default.svc:8501"))' | tee /dev/stderr) + yq '[.env[3].value] | any(contains("https://release-name-consul-server.default.svc:8501"))' | tee /dev/stderr) echo $actual [ "${actual}" = "true" ] @@ -800,7 +785,7 @@ key2: value2' \ exp='consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ -consul-api-timeout=5s \ - -gateway -gateway-kind="mesh-gateway" \ + -gateway-kind="mesh-gateway" \ -consul-node-name="k8s-service-mesh" \ -proxy-id-file=/consul/service/proxy-id \ -service-name=mesh-gateway \ @@ -822,7 +807,7 @@ key2: value2' \ exp='consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ -consul-api-timeout=5s \ - -gateway -gateway-kind="mesh-gateway" \ + -gateway-kind="mesh-gateway" \ -consul-node-name="k8s-service-mesh" \ -proxy-id-file=/consul/service/proxy-id \ -acl-token-sink=/consul/service/acl-token \ @@ -849,7 +834,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8888" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeIP" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -869,7 +854,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeIP" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -889,7 +874,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeName" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -924,30 +909,16 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Static" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-static"]' | tee /dev/stderr) [ "${actual}" = "example.com" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } -@test "meshGateway/Deployment: mesh-gateway-init init container wanAddress.source=Service fails if service.enable is false" { - cd `chart_dir` - run helm template \ - -s templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'meshGateway.wanAddress.source=Service' \ - --set 'meshGateway.service.enabled=false' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "if meshGateway.wanAddress.source=Service then meshGateway.service.enabled must be set to true" ]] -} - @test "meshGateway/Deployment: mesh-gateway-init init container wanAddress.source=Service, type=LoadBalancer" { cd `chart_dir` local annotations=$(helm template \ @@ -956,7 +927,6 @@ key2: value2' \ --set 'connectInject.enabled=true' \ --set 'meshGateway.wanAddress.source=Service' \ --set 'meshGateway.wanAddress.port=ignored' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=LoadBalancer' \ . | tee /dev/stderr | yq -r '.spec.template.metadata.annotations ' | tee /dev/stderr) @@ -964,7 +934,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -978,7 +948,6 @@ key2: value2' \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'meshGateway.wanAddress.source=Service' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.nodePort=9999' \ --set 'meshGateway.service.type=NodePort' \ . | tee /dev/stderr | @@ -987,7 +956,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -1001,7 +970,6 @@ key2: value2' \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'meshGateway.wanAddress.source=Service' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=NodePort' \ . @@ -1017,7 +985,6 @@ key2: value2' \ --set 'connectInject.enabled=true' \ --set 'meshGateway.wanAddress.source=Service' \ --set 'meshGateway.wanAddress.port=ignored' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=ClusterIP' \ . | tee /dev/stderr | yq -r '.spec.template.metadata.annotations ' | tee /dev/stderr) @@ -1025,7 +992,7 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) @@ -1044,7 +1011,7 @@ key2: value2' \ exp='consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ -consul-api-timeout=5s \ - -gateway -gateway-kind="mesh-gateway" \ + -gateway-kind="mesh-gateway" \ -consul-node-name="k8s-service-mesh" \ -proxy-id-file=/consul/service/proxy-id \ -service-name=new-name \ @@ -1273,7 +1240,7 @@ key2: value2' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.consulCARole=carole' \ . | tee /dev/stderr | - yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role") | del(."consul.hashicorp.com/gateway-kind") | del(."consul.hashicorp.com/mesh-gateway-source") | del(."consul.hashicorp.com/mesh-gateway-container-port") | del(."consul.hashicorp.com/mesh-gateway-wan-address") | del(."consul.hashicorp.com/mesh-gateway-wan-port")' | tee /dev/stderr) + yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role") | del(."consul.hashicorp.com/gateway-kind") | del(."consul.hashicorp.com/mesh-gateway-wan-address-source") | del(."consul.hashicorp.com/mesh-gateway-container-port") | del(."consul.hashicorp.com/mesh-gateway-wan-address-static") | del(."consul.hashicorp.com/mesh-gateway-wan-port") | del(."consul.hashicorp.com/mesh-gateway-consul-service-name")' | tee /dev/stderr) [ "${actual}" = "{}" ] } diff --git a/charts/consul/test/unit/mesh-gateway-service.bats b/charts/consul/test/unit/mesh-gateway-service.bats index 60ca3a3503..acedeb22b3 100755 --- a/charts/consul/test/unit/mesh-gateway-service.bats +++ b/charts/consul/test/unit/mesh-gateway-service.bats @@ -20,13 +20,12 @@ load _helpers [ "${actual}" = "true" ] } -@test "meshGateway/Service: enabled with meshGateway.enabled=true meshGateway.service.enabled" { +@test "meshGateway/Service: enabled with meshGateway.enabled=true" { cd `chart_dir` local actual=$(helm template \ -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq 'length > 0' | tee /dev/stderr) [ "${actual}" = "true" ] @@ -41,7 +40,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.metadata.annotations' | tee /dev/stderr) [ "${actual}" = "null" ] @@ -53,7 +51,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.annotations=key: value' \ . | tee /dev/stderr | yq -r '.metadata.annotations.key' | tee /dev/stderr) @@ -69,7 +66,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.ports[0].port' | tee /dev/stderr) [ "${actual}" = "443" ] @@ -81,7 +77,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.port=8443' \ . | tee /dev/stderr | yq -r '.spec.ports[0].port' | tee /dev/stderr) @@ -97,7 +92,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.ports[0].targetPort' | tee /dev/stderr) [ "${actual}" = "8443" ] @@ -109,7 +103,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.containerPort=9443' \ . | tee /dev/stderr | yq -r '.spec.ports[0].targetPort' | tee /dev/stderr) @@ -125,7 +118,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.ports[0].nodePort' | tee /dev/stderr) [ "${actual}" = "null" ] @@ -137,7 +129,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.nodePort=8443' \ . | tee /dev/stderr | yq -r '.spec.ports[0].nodePort' | tee /dev/stderr) @@ -153,7 +144,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ . | tee /dev/stderr | yq -r '.spec.type' | tee /dev/stderr) [ "${actual}" = "LoadBalancer" ] @@ -165,7 +155,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.type=ClusterIP' \ . | tee /dev/stderr | yq -r '.spec.type' | tee /dev/stderr) @@ -181,7 +170,6 @@ load _helpers -s templates/mesh-gateway-service.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'meshGateway.service.enabled=true' \ --set 'meshGateway.service.additionalSpec=key: value' \ . | tee /dev/stderr | yq -r '.spec.key' | tee /dev/stderr) diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 754c4764ae..9167e8536d 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -636,11 +636,6 @@ global: # @default: hashicorp/consul-dataplane: imageConsulDataplane: "ishustava/consul-dataplane:latest@sha256:5ac25d6a159e360b9cbe9b12ea6be70cde8f4677cbf00ae4885e5f5fd9b1ad36" - # The name (and tag) of the consul-dataplane Docker image used for the - # connect-injected sidecar proxies and mesh, terminating, and ingress gateways. - # @default: hashicorp/consul-dataplane: - imageConsulDataplane: "ishustava/consul-dataplane:latest@sha256:5ac25d6a159e360b9cbe9b12ea6be70cde8f4677cbf00ae4885e5f5fd9b1ad36" - # Configuration for running this Helm chart on the Red Hat OpenShift platform. # This Helm chart currently supports OpenShift v4.x+. openshift: @@ -2426,9 +2421,6 @@ meshGateway: # The service option configures the Service that fronts the Gateway Deployment. service: - # Whether to create a Service or not. - enabled: true - # Type of service, ex. LoadBalancer, ClusterIP. type: LoadBalancer diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index a364361818..f11be62cdb 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -20,29 +20,31 @@ const ( // be set to a truthy or falsy value, as parseable by strconv.ParseBool. annotationInject = "consul.hashicorp.com/connect-inject" - // annotationGatewayKind is the key of the annotation that annotates pods + // annotationGatewayKind is the key of the annotation that indicates pods // that represent Consul Connect Gateways. This should be set to a // value that is either "mesh", "ingress" or "terminating". annotationGatewayKind = "consul.hashicorp.com/gateway-kind" - // annotationMeshGatewaySource is the key of the annotation that annotates pods - // that represent Consul Connect Gateways. This should be set to a - // truthy or falsy value, as parseable by strconv.ParseBool. + // annotationMeshGatewayConsulServiceName is the key of the annotation whose value + // is the service name with which the mesh gateway is registered. + annotationMeshGatewayConsulServiceName = "consul.hashicorp.com/mesh-gateway-consul-service-name" + + // annotationMeshGatewayContainerPort is the key of the annotation whose value is + // used as the port and also registered as the LAN port when the mesh-gateway + // service is registered. annotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" - // annotationMeshGatewaySource is the key of the annotation that annotates pods - // that represent Consul Connect Gateways. This should be set to a - // truthy or falsy value, as parseable by strconv.ParseBool. - annotationMeshGatewaySource = "consul.hashicorp.com/mesh-gateway-source" + // annotationMeshGatewaySource is the key of the annotation that determines which + // source to use to determine the wan address and wan port for the mesh-gateway + // service registration. + annotationMeshGatewaySource = "consul.hashicorp.com/mesh-gateway-wan-address-source" - // annotationGatewayKind is the key of the annotation that annotates pods - // that represent Consul Connect Gateways. This should be set to a - // truthy or falsy value, as parseable by strconv.ParseBool. - annotationMeshGatewayWANAddress = "consul.hashicorp.com/mesh-gateway-wan-address" + // annotationMeshGatewayWANAddress is the key of the annotation that when the source + // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. + annotationMeshGatewayWANAddress = "consul.hashicorp.com/mesh-gateway-wan-address-static" - // annotationGatewayKind is the key of the annotation that annotates pods - // that represent Consul Connect Gateways. This should be set to a - // truthy or falsy value, as parseable by strconv.ParseBool. + // annotationMeshGatewayWANPort is the key of the annotation whose value is the + // WAN port for the mesh-gateway service registration. annotationMeshGatewayWANPort = "consul.hashicorp.com/mesh-gateway-wan-port" // annotationInjectMountVolumes is the key of the annotation that controls whether diff --git a/control-plane/connect-inject/endpoints_controller.go b/control-plane/connect-inject/endpoints_controller.go index c5cf719447..f503877477 100644 --- a/control-plane/connect-inject/endpoints_controller.go +++ b/control-plane/connect-inject/endpoints_controller.go @@ -35,6 +35,7 @@ const ( kubernetesSuccessReasonMsg = "Kubernetes health checks passing" envoyPrometheusBindAddr = "envoy_prometheus_bind_addr" sidecarContainer = "consul-dataplane" + wildcardNamespace = "*" // clusterIPTaggedAddressName is the key for the tagged address to store the service's cluster IP and service port // in Consul. Note: This value should not be changed without a corresponding change in Consul. @@ -617,13 +618,19 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service return nil, err } + meshGatewayServiceName, ok := pod.Annotations[annotationMeshGatewayConsulServiceName] + if !ok { + return nil, fmt.Errorf("failed to read annontation %s from pod %s/%s", annotationMeshGatewayConsulServiceName, pod.Namespace, pod.Name) + } + service := &api.AgentService{ - Kind: api.ServiceKindMeshGateway, - ID: pod.Name, - Service: "mesh-gateway", - Port: port, - Address: pod.Status.PodIP, - Meta: meta, + Kind: api.ServiceKindMeshGateway, + ID: pod.Name, + Namespace: "default", + Service: meshGatewayServiceName, + Port: port, + Address: pod.Status.PodIP, + Meta: meta, TaggedAddresses: map[string]api.ServiceAddress{ "lan": { Address: pod.Status.PodIP, @@ -685,6 +692,9 @@ func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoi case corev1.ServiceTypeClusterIP: wanAddr = svc.Spec.ClusterIP case corev1.ServiceTypeLoadBalancer: + if len(svc.Status.LoadBalancer.Ingress) == 0 { + return "", 0, fmt.Errorf("failed to read ingress config for loadbalancer for service %s in namespace %s", endpoints.Name, endpoints.Namespace) + } for _, ingr := range svc.Status.LoadBalancer.Ingress { if ingr.IP != "" { wanAddr = ingr.IP @@ -753,9 +763,6 @@ func getHealthCheckStatusReason(healthCheckStatus, podName, podNamespace string) // them only if they are not in endpointsAddressesMap. If the map is nil, it will deregister all instances. If the map // has addresses, it will only deregister instances not in the map. func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace string, endpointsAddressesMap map[string]bool) error { - // We need to get services matching "k8s-service-name" and "k8s-namespace" metadata. - consulNamespace := r.consulNamespace(k8sSvcNamespace) - // Get services matching metadata. svcs, err := r.serviceInstancesForK8SServiceNameAndNamespace(k8sSvcName, k8sSvcNamespace) if err != nil { @@ -765,6 +772,7 @@ func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace stri // Deregister each service instance that matches the metadata. for _, svc := range svcs.Services { + // We need to get services matching "k8s-service-name" and "k8s-namespace" metadata. // If we selectively deregister, only deregister if the address is not in the map. Otherwise, deregister // every service instance. var serviceDeregistered bool @@ -775,7 +783,7 @@ func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace stri _, err = r.ConsulClient.Catalog().Deregister(&api.CatalogDeregistration{ Node: ConsulNodeName, ServiceID: svc.ID, - Namespace: consulNamespace, + Namespace: svc.Namespace, }, nil) if err != nil { r.Log.Error(err, "failed to deregister service instance", "id", svc.ID) @@ -788,7 +796,7 @@ func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace stri if _, err = r.ConsulClient.Catalog().Deregister(&api.CatalogDeregistration{ Node: ConsulNodeName, ServiceID: svc.ID, - Namespace: consulNamespace, + Namespace: svc.Namespace, }, nil); err != nil { r.Log.Error(err, "failed to deregister service instance", "id", svc.ID) return err @@ -798,7 +806,7 @@ func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace stri if r.AuthMethod != "" && serviceDeregistered { r.Log.Info("reconciling ACL tokens for service", "svc", svc.Service) - err = r.deleteACLTokensForServiceInstance(svc.Service, k8sSvcNamespace, svc.Meta[MetaKeyPodName]) + err = r.deleteACLTokensForServiceInstance(svc, k8sSvcNamespace, svc.Meta[MetaKeyPodName]) if err != nil { r.Log.Error(err, "failed to reconcile ACL tokens for service", "svc", svc.Service) return err @@ -812,15 +820,14 @@ func (r *EndpointsController) deregisterService(k8sSvcName, k8sSvcNamespace stri // deleteACLTokensForServiceInstance finds the ACL tokens that belongs to the service instance and deletes it from Consul. // It will only check for ACL tokens that have been created with the auth method this controller // has been configured with and will only delete tokens for the provided podName. -func (r *EndpointsController) deleteACLTokensForServiceInstance(serviceName, k8sNS, podName string) error { +func (r *EndpointsController) deleteACLTokensForServiceInstance(svc *api.AgentService, k8sNS, podName string) error { // Skip if podName is empty. if podName == "" { return nil } - consulNS := r.consulNamespace(k8sNS) tokens, _, err := r.ConsulClient.ACL().TokenList(&api.QueryOptions{ - Namespace: consulNS, + Namespace: svc.Namespace, }) if err != nil { return fmt.Errorf("failed to get a list of tokens from Consul: %s", err) @@ -829,10 +836,10 @@ func (r *EndpointsController) deleteACLTokensForServiceInstance(serviceName, k8s for _, token := range tokens { // Only delete tokens that: // * have been created with the auth method configured for this endpoints controller - // * have a single service identity whose service name is the same as 'serviceName' + // * have a single service identity whose service name is the same as 'svc.Service' if token.AuthMethod == r.AuthMethod && len(token.ServiceIdentities) == 1 && - token.ServiceIdentities[0].ServiceName == serviceName { + token.ServiceIdentities[0].ServiceName == svc.Service { tokenMeta, err := getTokenMetaFromDescription(token.Description) if err != nil { return fmt.Errorf("failed to parse token metadata: %s", err) @@ -843,16 +850,12 @@ func (r *EndpointsController) deleteACLTokensForServiceInstance(serviceName, k8s // If we can't find token's pod, delete it. if tokenPodName == podName { r.Log.Info("deleting ACL token for pod", "name", podName) - _, err = r.ConsulClient.ACL().TokenDelete(token.AccessorID, &api.WriteOptions{Namespace: consulNS}) - if err != nil { + if _, err := r.ConsulClient.ACL().TokenDelete(token.AccessorID, &api.WriteOptions{Namespace: svc.Namespace}); err != nil { return fmt.Errorf("failed to delete token from Consul: %s", err) } - } else if err != nil { - return err } } } - return nil } @@ -934,10 +937,17 @@ func getTokenMetaFromDescription(description string) (map[string]string, error) // serviceInstancesForK8SServiceNameAndNamespace calls Consul's ServicesWithFilter to get the list // of services instances that have the provided k8sServiceName and k8sServiceNamespace in their metadata. func (r *EndpointsController) serviceInstancesForK8SServiceNameAndNamespace(k8sServiceName, k8sServiceNamespace string) (*api.CatalogNodeServiceList, error) { + var ( + serviceList *api.CatalogNodeServiceList + err error + ) filter := fmt.Sprintf(`Meta[%q] == %q and Meta[%q] == %q and Meta[%q] == %q`, MetaKeyKubeServiceName, k8sServiceName, MetaKeyKubeNS, k8sServiceNamespace, MetaKeyManagedBy, managedByValue) - - serviceList, _, err := r.ConsulClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter, Namespace: r.consulNamespace(k8sServiceNamespace)}) + if r.EnableConsulNamespaces { + serviceList, _, err = r.ConsulClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter, Namespace: wildcardNamespace}) + } else { + serviceList, _, err = r.ConsulClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter}) + } return serviceList, err } @@ -1142,10 +1152,8 @@ func hasBeenInjected(pod corev1.Pod) bool { // isGateway checks the value of the gateway annotation and returns true if the Pod represents a Gateway. func isGateway(pod corev1.Pod) bool { - if anno, ok := pod.Annotations[annotationGatewayKind]; ok && anno != "" { - return true - } - return false + anno, ok := pod.Annotations[annotationGatewayKind] + return ok && anno != "" } // mapAddresses combines all addresses to a mapping of address to its health status. diff --git a/control-plane/connect-inject/endpoints_controller_ent_test.go b/control-plane/connect-inject/endpoints_controller_ent_test.go index 34b18d8c20..8a00bdef61 100644 --- a/control-plane/connect-inject/endpoints_controller_ent_test.go +++ b/control-plane/connect-inject/endpoints_controller_ent_test.go @@ -88,11 +88,12 @@ func TestReconcileCreateEndpointWithNamespaces(t *testing.T) { pod1 := createPodWithNamespace("pod1", testCase.SourceKubeNS, "1.2.3.4", true, true) pod2 := createPodWithNamespace("pod2", testCase.SourceKubeNS, "2.2.3.4", true, true) meshGateway := createGatewayWithNamespace("mesh-gateway", testCase.SourceKubeNS, "3.3.3.3", map[string]string{ - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: "mesh"}) + annotationMeshGatewayConsulServiceName: "mesh-gateway", + annotationMeshGatewaySource: "Static", + annotationMeshGatewayWANAddress: "2.3.4.5", + annotationMeshGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: "mesh"}) endpointWithAddresses := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "service-created", @@ -1533,6 +1534,68 @@ func TestReconcileDeleteEndpointWithNamespaces(t *testing.T) { }, enableACLs: true, }, + { + name: "mesh-gateway", + consulSvcName: "service-deleted", + initialConsulSvcs: []*api.AgentService{ + { + ID: "mesh-gateway", + Kind: api.ServiceKindMeshGateway, + Service: "mesh-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: ts.SourceKubeNS, + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "mesh-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + Namespace: "default", + }, + }, + enableACLs: false, + }, + { + name: "mesh-gateway with ACLs enabled", + consulSvcName: "service-deleted", + initialConsulSvcs: []*api.AgentService{ + { + ID: "mesh-gateway", + Kind: api.ServiceKindMeshGateway, + Service: "mesh-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: ts.SourceKubeNS, + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "mesh-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + Namespace: "default", + }, + }, + enableACLs: true, + }, } for _, tt := range cases { t.Run(fmt.Sprintf("%s:%s", name, tt.name), func(t *testing.T) { @@ -1575,7 +1638,8 @@ func TestReconcileDeleteEndpointWithNamespaces(t *testing.T) { require.NoError(t, err) // Create a token for it if ACLs are enabled. if tt.enableACLs { - if svc.Kind != api.ServiceKindConnectProxy { + switch svc.Kind { + case api.ServiceKindTypical: var writeOpts api.WriteOptions // When mirroring is enabled, the auth method will be created in the "default" Consul namespace. if ts.Mirror { @@ -1593,6 +1657,18 @@ func TestReconcileDeleteEndpointWithNamespaces(t *testing.T) { }, &writeOpts) require.NoError(t, err) + case api.ServiceKindMeshGateway: + var writeOpts api.WriteOptions + writeOpts.Namespace = "default" + test.SetupK8sAuthMethod(t, consulClient, svc.Service, svc.Meta[MetaKeyKubeNS]) + token, _, err = consulClient.ACL().Login(&api.ACLLoginParams{ + AuthMethod: test.AuthMethod, + BearerToken: test.ServiceAccountJWTToken, + Meta: map[string]string{ + TokenMetaPodNameKey: fmt.Sprintf("%s/%s", svc.Meta[MetaKeyKubeNS], svc.Meta[MetaKeyPodName]), + "component": svc.ID, + }, + }, &writeOpts) } } } @@ -1631,12 +1707,18 @@ func TestReconcileDeleteEndpointWithNamespaces(t *testing.T) { require.NoError(t, err) // After reconciliation, Consul should not have any instances of service-deleted. - serviceInstances, _, err := consulClient.Catalog().Service(tt.consulSvcName, "", &api.QueryOptions{Namespace: ts.ExpConsulNS}) - require.NoError(t, err) - require.Empty(t, serviceInstances) - proxyServiceInstances, _, err := consulClient.Catalog().Service(fmt.Sprintf("%s-sidecar-proxy", tt.consulSvcName), "", &api.QueryOptions{Namespace: ts.ExpConsulNS}) - require.NoError(t, err) - require.Empty(t, proxyServiceInstances) + if tt.consulSvcName == "mesh-gateway" { + gatewayInstances, _, err := consulClient.Catalog().Service(tt.consulSvcName, "", &api.QueryOptions{Namespace: "default"}) + require.NoError(t, err) + require.Empty(t, gatewayInstances) + } else { + serviceInstances, _, err := consulClient.Catalog().Service(tt.consulSvcName, "", &api.QueryOptions{Namespace: ts.ExpConsulNS}) + require.NoError(t, err) + require.Empty(t, serviceInstances) + proxyServiceInstances, _, err := consulClient.Catalog().Service(fmt.Sprintf("%s-sidecar-proxy", tt.consulSvcName), "", &api.QueryOptions{Namespace: ts.ExpConsulNS}) + require.NoError(t, err) + require.Empty(t, proxyServiceInstances) + } if tt.enableACLs { _, _, err = consulClient.ACL().TokenRead(token.AccessorID, nil) diff --git a/control-plane/connect-inject/endpoints_controller_test.go b/control-plane/connect-inject/endpoints_controller_test.go index 2a49a3c2f1..ef44d31d48 100644 --- a/control-plane/connect-inject/endpoints_controller_test.go +++ b/control-plane/connect-inject/endpoints_controller_test.go @@ -1192,11 +1192,12 @@ func TestReconcileCreateEndpoint(t *testing.T) { consulSvcName: "mesh-gateway", k8sObjects: func() []runtime.Object { gateway := createGatewayPod("mesh-gateway", "1.2.3.4", map[string]string{ - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: "mesh"}) + annotationMeshGatewayConsulServiceName: "mesh-gateway", + annotationMeshGatewaySource: "Static", + annotationMeshGatewayWANAddress: "2.3.4.5", + annotationMeshGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: "mesh"}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "mesh-gateway", @@ -1258,11 +1259,12 @@ func TestReconcileCreateEndpoint(t *testing.T) { consulSvcName: "mesh-gateway", k8sObjects: func() []runtime.Object { gateway := createGatewayPod("mesh-gateway", "1.2.3.4", map[string]string{ - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: "mesh"}) + annotationMeshGatewayConsulServiceName: "mesh-gateway", + annotationMeshGatewaySource: "Static", + annotationMeshGatewayWANAddress: "2.3.4.5", + annotationMeshGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: "mesh"}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "mesh-gateway", @@ -3313,6 +3315,67 @@ func TestReconcileDeleteEndpoint(t *testing.T) { }, enableACLs: true, }, + { + name: "Mesh Gateway", + consulSvcName: "service-deleted", + expectServicesToBeDeleted: true, + initialConsulSvcs: []*api.AgentService{ + { + ID: "mesh-gateway", + Kind: api.ServiceKindMeshGateway, + Service: "mesh-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "mesh-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + }, + }, + }, + { + name: "When ACLs are enabled, the mesh-gateway token should be deleted", + consulSvcName: "service-deleted", + expectServicesToBeDeleted: true, + initialConsulSvcs: []*api.AgentService{ + { + ID: "mesh-gateway", + Kind: api.ServiceKindMeshGateway, + Service: "mesh-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "mesh-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + }, + }, + enableACLs: true, + }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { @@ -3354,7 +3417,18 @@ func TestReconcileDeleteEndpoint(t *testing.T) { // Create a token for it if ACLs are enabled. if tt.enableACLs { test.SetupK8sAuthMethod(t, consulClient, svc.Service, "default") - if svc.Kind != api.ServiceKindConnectProxy { + switch svc.Kind { + case api.ServiceKindMeshGateway: + token, _, err = consulClient.ACL().Login(&api.ACLLoginParams{ + AuthMethod: test.AuthMethod, + BearerToken: test.ServiceAccountJWTToken, + Meta: map[string]string{ + "pod": fmt.Sprintf("%s/%s", svc.Meta[MetaKeyKubeNS], svc.Meta[MetaKeyPodName]), + "component": tt.consulSvcName, + }, + }, nil) + require.NoError(t, err) + case api.ServiceKindTypical: token, _, err = consulClient.ACL().Login(&api.ACLLoginParams{ AuthMethod: test.AuthMethod, BearerToken: test.ServiceAccountJWTToken, @@ -3362,7 +3436,6 @@ func TestReconcileDeleteEndpoint(t *testing.T) { "pod": fmt.Sprintf("%s/%s", svc.Meta[MetaKeyKubeNS], svc.Meta[MetaKeyPodName]), }, }, nil) - require.NoError(t, err) } } @@ -5587,7 +5660,7 @@ func Test_GetWANData(t *testing.T) { }, wanAddr: "test-loadbalancer-hostname", wanPort: 1234, - expErr: "failed to read annotation consul.hashicorp.com/mesh-gateway-source", + expErr: "failed to read annotation consul.hashicorp.com/mesh-gateway-wan-address-source", }, "no Service with Source=Service": { gatewayPod: corev1.Pod{ @@ -5666,6 +5739,51 @@ func Test_GetWANData(t *testing.T) { wanPort: 1234, expErr: "failed to parse WAN port from value not-a-valid-port", }, + "source=Service, serviceType=LoadBalancer no Ingress configured": { + gatewayPod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Annotations: map[string]string{ + annotationMeshGatewaySource: "Service", + annotationMeshGatewayWANAddress: "test-wan-address", + annotationMeshGatewayWANPort: "1234", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "test-nodename", + }, + Status: corev1.PodStatus{ + HostIP: "test-host-ip", + }, + }, + gatewayEndpoint: corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Namespace: "default", + }, + }, + k8sObjects: func() []runtime.Object { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ClusterIP: "test-cluster-ip", + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{}, + }, + }, + } + return []runtime.Object{service} + }, + wanAddr: "test-loadbalancer-hostname", + wanPort: 1234, + expErr: "failed to read ingress config for loadbalancer for service gateway in namespace default", + }, } for name, c := range cases { diff --git a/control-plane/go.mod b/control-plane/go.mod index 27b40011d7..6b05c2816c 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -7,7 +7,7 @@ require ( github.com/fsnotify/fsnotify v1.5.4 github.com/go-logr/logr v0.4.0 github.com/google/go-cmp v0.5.7 - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220831174802-b8af65262de8 github.com/hashicorp/consul/api v1.10.1-0.20220822180451-60c82757ea35 github.com/hashicorp/consul/sdk v0.11.0 github.com/hashicorp/go-discover v0.0.0-20200812215701-c4b85f6ed31f @@ -51,7 +51,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containernetworking/plugins v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 // indirect github.com/digitalocean/godo v1.10.0 // indirect @@ -69,7 +68,6 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect - github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220831174802-b8af65262de8 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect diff --git a/control-plane/go.sum b/control-plane/go.sum index bc15218321..e6df3cd145 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -138,8 +138,6 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/containernetworking/cni v1.1.1 h1:ky20T7c0MvKvbMOwS/FrlbNwjEoqJEUUYfsL4b0mc4k= github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE= -github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -310,8 +308,6 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -335,8 +331,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220824032443-9606072aec2a h1:cBwGTYV84M0W8PwHX6+PywY8NiP3dzA7JaZzs9+LdiA= -github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220824032443-9606072aec2a/go.mod h1:aw35GB76URgbtxaSSMxbOetbG7YEHHPkIX3/SkTBaWc= github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220831174802-b8af65262de8 h1:TQY0oKtLV15UNYWeSkTxi4McBIyLecsEtbc/VfxvbYA= github.com/hashicorp/consul-k8s/control-plane/cni v0.0.0-20220831174802-b8af65262de8/go.mod h1:aw35GB76URgbtxaSSMxbOetbG7YEHHPkIX3/SkTBaWc= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= diff --git a/control-plane/subcommand/connect-init/command.go b/control-plane/subcommand/connect-init/command.go index 0de7f0f633..05ea2f99ef 100644 --- a/control-plane/subcommand/connect-init/command.go +++ b/control-plane/subcommand/connect-init/command.go @@ -39,7 +39,6 @@ type Command struct { flagConsulServiceNamespace string // Consul destination namespace for the service. flagServiceAccountName string // Service account name. flagServiceName string // Service name. - flagGateway bool flagGatewayKind string flagLogLevel string flagLogJSON bool @@ -76,8 +75,7 @@ func (c *Command) init() { c.flagSet.StringVar(&c.flagACLTokenSink, "acl-token-sink", defaultTokenSinkFile, "File name where where ACL token should be saved.") c.flagSet.StringVar(&c.flagProxyIDFile, "proxy-id-file", defaultProxyIDFile, "File name where proxy's Consul service ID should be saved.") c.flagSet.BoolVar(&c.flagMultiPort, "multiport", false, "If the pod is a multi port pod.") - c.flagSet.BoolVar(&c.flagGateway, "gateway", false, "If the pod is a Consul gateway pod.") - c.flagSet.StringVar(&c.flagGatewayKind, "gateway-kind", "", "Name of the gateway that is being registered.") + c.flagSet.StringVar(&c.flagGatewayKind, "gateway-kind", "", "Kind of Consul gateway that is being registered.") c.flagSet.StringVar(&c.flagLogLevel, "log-level", "info", "Log verbosity level. Supported values (in order of detail) are \"trace\", "+ "\"debug\", \"info\", \"warn\", and \"error\".") @@ -128,8 +126,8 @@ func (c *Command) Run(args []string) int { if c.flagACLAuthMethod != "" { // loginMeta is the default metadata that we pass to the consul login API. var loginMeta map[string]string - if c.flagGateway { - loginMeta = map[string]string{"component": c.flagGatewayKind} + if c.flagGatewayKind != "" { + loginMeta = map[string]string{"component": c.flagGatewayKind, "pod": fmt.Sprintf("%s/%s", c.flagPodNamespace, c.flagPodName)} } else { loginMeta = map[string]string{"pod": fmt.Sprintf("%s/%s", c.flagPodNamespace, c.flagPodName)} } @@ -163,7 +161,7 @@ func (c *Command) Run(args []string) int { c.logger.Error("Unable to update client connection", "error", err) return 1 } - if c.flagGateway { + if c.flagGatewayKind != "" { err = backoff.Retry(c.getGatewayRegistration(consulClient), backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), c.serviceRegistrationPollingAttempts)) if err != nil { c.logger.Error("Timed out waiting for gateway registration", "error", err) @@ -324,15 +322,12 @@ func (c *Command) validateFlags() error { if c.flagPodNamespace == "" { return errors.New("-pod-namespace must be set") } - if c.flagACLAuthMethod != "" && c.flagServiceAccountName == "" && !c.flagGateway { + if c.flagACLAuthMethod != "" && c.flagServiceAccountName == "" && c.flagGatewayKind == "" { return errors.New("-service-account-name must be set when ACLs are enabled") } if c.flagConsulNodeName == "" { return errors.New("-consul-node-name must be set") } - if c.flagGateway && c.flagGatewayKind == "" { - return errors.New("-gateway-kind must be set if -gateway is set") - } if c.http.ConsulAPITimeout() <= 0 { return errors.New("-consul-api-timeout must be set to a value greater than 0") diff --git a/control-plane/subcommand/connect-init/command_test.go b/control-plane/subcommand/connect-init/command_test.go index 76df4d96a3..453ceebaf5 100644 --- a/control-plane/subcommand/connect-init/command_test.go +++ b/control-plane/subcommand/connect-init/command_test.go @@ -70,16 +70,6 @@ func TestRun_FlagValidation(t *testing.T) { }, expErr: "unknown log level: invalid", }, - { - flags: []string{ - "-pod-name", testPodName, - "-pod-namespace", testPodNamespace, - "-service-account-name", "foo", - "-gateway", - "-consul-node-name", "bar", - }, - expErr: "-gateway-kind must be set if -gateway is set", - }, } for _, c := range cases { t.Run(c.expErr, func(t *testing.T) { @@ -434,7 +424,6 @@ func TestRun_Gateways(t *testing.T) { // CONSUL_HTTP_ADDR when it processes the command template. flags := []string{"-pod-name", testGatewayName, "-pod-namespace", testPodNamespace, - "-gateway", "-gateway-kind", tt.gatewayKind, "-http-addr", fmt.Sprintf("%s://%s", cfg.Scheme, cfg.Address), "-proxy-id-file", proxyFile, @@ -466,7 +455,7 @@ func TestRun_Gateways(t *testing.T) { require.NoError(t, err) token, _, err := consulClient.ACL().TokenReadSelf(nil) require.NoError(t, err) - require.Equal(t, fmt.Sprintf(`token created via login: {"component":"%s"}`, tt.gatewayKind), token.Description) + require.Equal(t, fmt.Sprintf(`token created via login: {"component":"%s","pod":"%s/%s"}`, tt.gatewayKind, testPodNamespace, testGatewayName), token.Description) } // Validate contents of proxyFile. @@ -739,7 +728,6 @@ func TestRun_Gateways_Errors(t *testing.T) { } flags := []string{ "-http-addr", server.HTTPAddr, - "-gateway", "-gateway-kind", "mesh-gateway", "-pod-name", testPodName, "-pod-namespace", testPodNamespace,