From 21ac5b9f7641e00745124f11df4c57f196dbcedb Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Tue, 27 Sep 2022 15:03:35 -0400 Subject: [PATCH] Register Ingress Gateway with Endpoints Controller - Use consul-dataplane instead of envoy. --- .../ingress_gateway_namespaces_test.go | 22 +- .../ingress-gateway/ingress_gateway_test.go | 12 +- acceptance/tests/ingress-gateway/main_test.go | 7 +- .../templates/connect-inject-deployment.yaml | 3 + .../ingress-gateways-deployment.yaml | 537 ++++------- .../templates/mesh-gateway-deployment.yaml | 17 +- .../unit/ingress-gateways-deployment.bats | 889 +++--------------- .../test/unit/mesh-gateway-deployment.bats | 32 +- charts/consul/values.yaml | 12 - control-plane/connect-inject/annotations.go | 16 +- .../connect-inject/endpoints_controller.go | 78 +- .../endpoints_controller_ent_test.go | 140 ++- .../endpoints_controller_test.go | 490 ++++++++-- .../subcommand/connect-init/command.go | 12 +- .../subcommand/inject-connect/command.go | 8 + 15 files changed, 941 insertions(+), 1334 deletions(-) diff --git a/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go b/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go index d1e5757f91..6a7863adbb 100644 --- a/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go +++ b/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go @@ -31,10 +31,10 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { secure bool }{ { - false, + secure: false, }, { - true, + secure: true, }, } for _, c := range cases { @@ -44,6 +44,7 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { // Install the Helm chart without the ingress gateway first // so that we can create the namespace for it. + igName := "ingress-gateway" helmValues := map[string]string{ "connectInject.enabled": "true", "connectInject.consulNamespaces.consulDestinationNamespace": testNamespace, @@ -51,6 +52,11 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { "global.enableConsulNamespaces": "true", "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), "global.tls.enabled": strconv.FormatBool(c.secure), + + "ingressGateways.enabled": "true", + "ingressGateways.gateways[0].name": igName, + "ingressGateways.gateways[0].replicas": "1", + "ingressGateways.gateways[0].consulNamespace": testNamespace, } releaseName := helpers.RandomName() @@ -70,15 +76,6 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { require.NoError(t, err) } - igName := "ingress-gateway" - logger.Log(t, "upgrading with ingress gateways enabled") - consulCluster.Upgrade(t, map[string]string{ - "ingressGateways.enabled": "true", - "ingressGateways.gateways[0].name": igName, - "ingressGateways.gateways[0].replicas": "1", - "ingressGateways.gateways[0].consulNamespace": testNamespace, - }) - logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { @@ -158,9 +155,6 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { // Test we can connect through the ingress gateway when both // the ingress gateway and the connect service are in different namespaces. -// These tests currently only test non-secure and secure without auto-encrypt installations -// because in the case of namespaces there isn't a significant distinction in code between auto-encrypt -// and non-auto-encrypt secure installations, so testing just one is enough. func TestIngressGatewayNamespaceMirroring(t *testing.T) { cfg := suite.Config() if !cfg.EnableEnterprise { diff --git a/acceptance/tests/ingress-gateway/ingress_gateway_test.go b/acceptance/tests/ingress-gateway/ingress_gateway_test.go index a913d2c024..0aca33221d 100644 --- a/acceptance/tests/ingress-gateway/ingress_gateway_test.go +++ b/acceptance/tests/ingress-gateway/ingress_gateway_test.go @@ -18,24 +18,17 @@ const StaticClientName = "static-client" // Test that ingress gateways work in a default installation and a secure installation. func TestIngressGateway(t *testing.T) { cases := []struct { - secure bool - autoEncrypt bool + secure bool }{ { false, - false, - }, - { - true, - false, }, { true, - true, }, } for _, c := range cases { - name := fmt.Sprintf("secure: %t; auto-encrypt: %t", c.secure, c.autoEncrypt) + name := fmt.Sprintf("secure: %t", c.secure) t.Run(name, func(t *testing.T) { ctx := suite.Environment().DefaultContext(t) cfg := suite.Config() @@ -48,7 +41,6 @@ func TestIngressGateway(t *testing.T) { "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), "global.tls.enabled": strconv.FormatBool(c.secure), - "global.tls.autoEncrypt": strconv.FormatBool(c.autoEncrypt), } releaseName := helpers.RandomName() diff --git a/acceptance/tests/ingress-gateway/main_test.go b/acceptance/tests/ingress-gateway/main_test.go index 3196d5a977..bd927f96cb 100644 --- a/acceptance/tests/ingress-gateway/main_test.go +++ b/acceptance/tests/ingress-gateway/main_test.go @@ -1,7 +1,6 @@ package ingressgateway import ( - "fmt" "os" "testing" @@ -11,8 +10,6 @@ import ( var suite testsuite.Suite func TestMain(m *testing.M) { - fmt.Println("Skipping ingress gateway tests because it's not supported with agentless yet") - os.Exit(0) - //suite = testsuite.NewSuite(m) - //os.Exit(suite.Run()) + suite = testsuite.NewSuite(m) + os.Exit(suite.Run()) } diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index e31349b2c3..5771843738 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -120,6 +120,9 @@ spec: - "-ec" - | consul-k8s-control-plane inject-connect \ + {{- if .Values.global.federation.enabled }} + -enable-federation \ + {{- end }} -log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \ -log-json={{ .Values.global.logJSON }} \ -default-inject={{ .Values.connectInject.default }} \ diff --git a/charts/consul/templates/ingress-gateways-deployment.yaml b/charts/consul/templates/ingress-gateways-deployment.yaml index 25e96e4f73..61cafe59e0 100644 --- a/charts/consul/templates/ingress-gateways-deployment.yaml +++ b/charts/consul/templates/ingress-gateways-deployment.yaml @@ -63,7 +63,47 @@ spec: release: {{ $root.Release.Name }} component: ingress-gateway ingress-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }} + consul.hashicorp.com/connect-inject-managed-by: consul-k8s-endpoints-controller annotations: + "consul.hashicorp.com/connect-inject": "false" + "consul.hashicorp.com/gateway-kind": "ingress-gateway" + "consul.hashicorp.com/gateway-consul-service-name": "{{ .name }}" + {{- if $root.Values.global.enableConsulNamespaces }} + "consul.hashicorp.com/gateway-namespace": {{ (default $defaults.consulNamespace .consulNamespace) }} + {{- end }} + "consul.hashicorp.com/gateway-wan-address-source": "Service" + {{- $serviceType := (default $defaults.service.type $service.type) }} + {{- if (eq $serviceType "NodePort") }} + {{- if $service.ports }} + {{- $firstPort := first $service.ports}} + {{- if $firstPort.nodePort }} + "consul.hashicorp.com/gateway-wan-port": "{{ $firstPort.nodePort }}" + {{- else }}{{ fail "if ingressGateways .service.type=NodePort and defining ingressGateways.gateways.service.ports, the first port entry must include a nodePort" }} + {{- end }} + {{- else if $defaults.service.ports }} + {{- $firstDefaultPort := first $defaults.service.ports}} + {{- if $firstDefaultPort.nodePort }} + "consul.hashicorp.com/gateway-wan-port": "{{ $firstDefaultPort.nodePort }}" + {{- else }}{{ fail "if ingressGateways .service.type=NodePort and using ingressGateways.defaults.service.ports, the first port entry must include a nodePort" }} + {{- end }} + {{- else }}{{ fail "if ingressGateways .service.type=NodePort, the first port entry in either the defaults or specific gateway must include a nodePort" }} + {{- end }} + {{- else }} + {{- if $service.ports }} + {{- $firstPort := first $service.ports}} + {{- if $firstPort.port }} + "consul.hashicorp.com/gateway-wan-port": "{{ $firstPort.port }}" + {{- else }}{{ fail "if ingressGateways .service.type is not NodePort and defining ingressGateways.gateways.service.ports, the first port entry must include a port" }} + {{- end }} + {{- else if $defaults.service.ports }} + {{- $firstDefaultPort := first $defaults.service.ports}} + {{- if $firstDefaultPort.port }} + "consul.hashicorp.com/gateway-wan-port": "{{ $firstDefaultPort.port }}" + {{- else }}{{ fail "if ingressGateways .service.type is not NodePort and using ingressGateways.defaults.service.ports, the first port entry must include a port" }} + {{- end }} + {{- else }}{{ fail "if ingressGateways .service.type is not NodePort, the first port entry in either the defaults or specific gateway must include a port" }} + {{- end }} + {{- end }} {{- if (and $root.Values.global.secretsBackend.vault.enabled $root.Values.global.tls.enabled) }} "vault.hashicorp.com/agent-init-first": "true" "vault.hashicorp.com/agent-inject": "true" @@ -78,7 +118,6 @@ spec: {{ tpl $root.Values.global.secretsBackend.vault.agentAnnotations $root | nindent 8 | trim }} {{- end }} {{- end }} - "consul.hashicorp.com/connect-inject": "false" {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} "prometheus.io/scrape": "true" "prometheus.io/path": "/metrics" @@ -107,374 +146,172 @@ spec: terminationGracePeriodSeconds: {{ default $defaults.terminationGracePeriodSeconds .terminationGracePeriodSeconds }} serviceAccountName: {{ template "consul.fullname" $root }}-{{ .name }} volumes: - - name: consul-bin - emptyDir: {} + - name: consul-service + emptyDir: + medium: "Memory" + {{- if $root.Values.global.tls.enabled }} + {{- if not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots) }} + - name: consul-ca-cert + secret: + {{- if $root.Values.global.tls.caCert.secretName }} + secretName: {{ $root.Values.global.tls.caCert.secretName }} + {{- else }} + secretName: {{ template "consul.fullname" $root }}-ca-cert + {{- end }} + items: + - key: {{ default "tls.crt" $root.Values.global.tls.caCert.secretKey }} + path: tls.crt + {{- end }} + {{- end }} + initContainers: + # ingress-gateway-init registers the ingress gateway service with Consul. + - name: ingress-gateway-init + image: {{ $root.Values.global.imageK8S }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- include "consul.consulK8sConsulServerEnvVars" $root | nindent 8 }} + {{- if $root.Values.global.enableConsulNamespaces }} + - name: CONSUL_NAMESPACE + value: {{ (default $defaults.consulNamespace .consulNamespace) }} + {{- end }} + {{- if $root.Values.global.acls.manageSystemACLs }} + - name: CONSUL_LOGIN_AUTH_METHOD + value: {{ template "consul.fullname" $root }}-k8s-component-auth-method + - name: CONSUL_LOGIN_DATACENTER + value: {{ $root.Values.global.datacenter }} + - name: CONSUL_LOGIN_META + value: "component=ingress-gateway,pod=$(NAMESPACE)/$(POD_NAME)" + {{- end }} + command: + - "/bin/sh" + - "-ec" + - | + consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${NAMESPACE} \ + -gateway-kind="ingress-gateway" \ + -consul-node-name="k8s-service-mesh" \ + -proxy-id-file=/consul/service/proxy-id \ + -service-name={{ template "consul.fullname" $root }}-{{ .name }} \ + -log-level={{ default $root.Values.global.logLevel }} \ + -log-json={{ $root.Values.global.logJSON }} + volumeMounts: - name: consul-service - emptyDir: - medium: "Memory" + mountPath: /consul/service {{- if $root.Values.global.tls.enabled }} - {{- if not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots) }} - name: consul-ca-cert - secret: - {{- if $root.Values.global.tls.caCert.secretName }} - secretName: {{ $root.Values.global.tls.caCert.secretName }} - {{- else }} - secretName: {{ template "consul.fullname" $root }}-ca-cert - {{- end }} - items: - - key: {{ default "tls.crt" $root.Values.global.tls.caCert.secretKey }} - path: tls.crt - {{- end }} - {{- if $root.Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert - emptyDir: - medium: "Memory" + mountPath: /consul/tls/ca + readOnly: true {{- end }} + resources: + requests: + memory: "50Mi" + cpu: "50m" + limits: + memory: "50Mi" + cpu: "50m" + containers: + - name: ingress-gateway + image: {{ $root.Values.global.imageConsulDataplane | quote }} + {{- if (default $defaults.resources .resources) }} + resources: {{ toYaml (default $defaults.resources .resources) | nindent 10 }} {{- end }} - initContainers: - # We use the Envoy image as our base image so we use an init container to - # copy the Consul binary to a shared directory that can be used when - # starting Envoy. - - name: copy-consul-bin - image: {{ $root.Values.global.image | quote }} - command: - - cp - - /bin/consul - - /consul-bin/consul - volumeMounts: - - name: consul-bin - mountPath: /consul-bin - {{- $initContainer := .initCopyConsulContainer }} - {{- if (or $initContainer $defaults.initCopyConsulContainer) }} - {{- if (default $defaults.initCopyConsulContainer.resources $initContainer.resources) }} - resources: {{ toYaml (default $defaults.initCopyConsulContainer.resources $initContainer.resources) | nindent 12 }} - {{- end }} - {{- end }} - {{- if (and $root.Values.global.tls.enabled $root.Values.global.tls.enableAutoEncrypt) }} - {{- include "consul.getAutoEncryptClientCA" $root | nindent 8 }} + volumeMounts: + {{- if $root.Values.global.tls.enabled }} + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true {{- end }} - # ingress-gateway-init registers the ingress gateway service with Consul. - - name: ingress-gateway-init - image: {{ $root.Values.global.imageK8S }} - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if $root.Values.global.tls.enabled }} - - name: CONSUL_HTTP_ADDR - value: https://$(HOST_IP):8501 - - name: CONSUL_CACERT - value: /consul/tls/ca/tls.crt - {{- else }} - - name: CONSUL_HTTP_ADDR - value: http://$(HOST_IP):8500 - {{- end }} - command: - - "/bin/sh" - - "-ec" - - | - {{- if $root.Values.global.acls.manageSystemACLs }} - consul-k8s-control-plane acl-init \ - -component-name=ingress-gateway/{{ template "consul.fullname" $root }}-{{ .name }} \ - -acl-auth-method={{ template "consul.fullname" $root }}-k8s-component-auth-method \ - {{- if $root.Values.global.adminPartitions.enabled }} - -partition={{ $root.Values.global.adminPartitions.name }} \ - {{- end }} - -token-sink-file=/consul/service/acl-token \ - -consul-api-timeout={{ $root.Values.global.consulAPITimeout }} \ - -log-level={{ default $root.Values.global.logLevel }} \ - -log-json={{ $root.Values.global.logJSON }} - {{ end }} - - {{- $serviceType := (default $defaults.service.type $service.type) }} - {{- if (eq $serviceType "NodePort") }} - WAN_ADDR="${HOST_IP}" - {{- else if (or (eq $serviceType "ClusterIP") (eq $serviceType "LoadBalancer")) }} - consul-k8s-control-plane service-address \ - -log-level={{ $root.Values.global.logLevel }} \ - -log-json={{ $root.Values.global.logJSON }} \ - -k8s-namespace={{ $root.Release.Namespace }} \ - -name={{ template "consul.fullname" $root }}-{{ .name }} \ - -output-file=/tmp/address.txt - WAN_ADDR="$(cat /tmp/address.txt)" - {{- else }} - {{- fail "currently set ingressGateway value service.type is not supported" }} - {{- end }} - - {{- if (eq $serviceType "NodePort") }} - {{- if $service.ports }} - {{- $firstPort := first $service.ports}} - {{- if $firstPort.nodePort }} - WAN_PORT={{ $firstPort.nodePort }} - {{- else }}{{ fail "if ingressGateways .service.type=NodePort and defining ingressGateways.gateways.service.ports, the first port entry must include a nodePort" }} - {{- end }} - {{- else if $defaults.service.ports }} - {{- $firstDefaultPort := first $defaults.service.ports}} - {{- if $firstDefaultPort.nodePort }} - WAN_PORT={{ $firstDefaultPort.nodePort }} - {{- else }}{{ fail "if ingressGateways .service.type=NodePort and using ingressGateways.defaults.service.ports, the first port entry must include a nodePort" }} - {{- end }} - {{- else }}{{ fail "if ingressGateways .service.type=NodePort, the first port entry in either the defaults or specific gateway must include a nodePort" }} - {{- end }} - - {{- else }} - {{- if $service.ports }} - {{- $firstPort := first $service.ports}} - {{- if $firstPort.port }} - WAN_PORT={{ $firstPort.port }} - {{- else }}{{ fail "if ingressGateways .service.type is not NodePort and defining ingressGateways.gateways.service.ports, the first port entry must include a port" }} - {{- end }} - {{- else if $defaults.service.ports }} - {{- $firstDefaultPort := first $defaults.service.ports}} - {{- if $firstDefaultPort.port }} - WAN_PORT={{ $firstDefaultPort.port }} - {{- else }}{{ fail "if ingressGateways .service.type is not NodePort and using ingressGateways.defaults.service.ports, the first port entry must include a port" }} - {{- end }} - {{- else }}{{ fail "if ingressGateways .service.type is not NodePort, the first port entry in either the defaults or specific gateway must include a port" }} - {{- end }} - {{- end }} - - cat > /consul/service/service.hcl << EOF - service { - kind = "ingress-gateway" - name = "{{ .name }}" - id = "${POD_NAME}" - {{- if $root.Values.global.enableConsulNamespaces }} - namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}" - {{- end }} - {{- if $root.Values.global.adminPartitions.enabled }} - partition = "{{ $root.Values.global.adminPartitions.name }}" - {{- end }} - port = ${WAN_PORT} - address = "${WAN_ADDR}" - tagged_addresses { - lan { - address = "${POD_IP}" - port = 21000 - } - wan { - address = "${WAN_ADDR}" - port = ${WAN_PORT} - } - } - proxy { - config { - {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} - envoy_prometheus_bind_addr = "${POD_IP}:20200" - {{- end }} - envoy_gateway_no_default_bind = true - envoy_gateway_bind_addresses { - all-interfaces { - address = "0.0.0.0" - } - } - } - } - checks = [ - { - name = "Ingress Gateway Listening" - interval = "10s" - tcp = "${POD_IP}:21000" - deregister_critical_service_after = "6h" - } - ] - } - EOF - - /consul-bin/consul services register \ - {{- if $root.Values.global.acls.manageSystemACLs }} - -token-file=/consul/service/acl-token \ - {{- end }} - /consul/service/service.hcl - volumeMounts: - - name: consul-service - mountPath: /consul/service - - name: consul-bin - mountPath: /consul-bin - {{- if $root.Values.global.tls.enabled }} - {{- if $root.Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /bin/sh + - -ec + - | + consul-dataplane \ + -envoy-ready-bind-address=$POD_IP \ + -envoy-ready-bind-port=21000 \ + {{- if $root.Values.externalServers.enabled }} + -addresses={{ $root.Values.externalServers.hosts | first }} \ {{- else }} - - name: consul-ca-cert + -addresses="{{ template "consul.fullname" $root }}-server.{{ $root.Release.Namespace }}.svc" \ {{- end }} - mountPath: /consul/tls/ca - readOnly: true - {{- end }} - resources: - requests: - memory: "50Mi" - cpu: "50m" - limits: - memory: "50Mi" - cpu: "50m" - containers: - - name: ingress-gateway - image: {{ $root.Values.global.imageEnvoy | quote }} - {{- if (default $defaults.resources .resources) }} - resources: {{ toYaml (default $defaults.resources .resources) | nindent 12 }} - {{- end }} - volumeMounts: - - name: consul-bin - mountPath: /consul-bin - - name: consul-service - mountPath: /consul/service - readOnly: true - {{- if $root.Values.global.tls.enabled }} - {{- if $root.Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert - {{- else }} - - name: consul-ca-cert - {{- end }} - mountPath: /consul/tls/ca - readOnly: true - {{- end }} - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if $root.Values.global.acls.manageSystemACLs }} - - name: CONSUL_HTTP_TOKEN_FILE - value: "/consul/service/acl-token" - {{- end}} - {{- if $root.Values.global.tls.enabled }} - - name: CONSUL_HTTP_ADDR - value: https://$(HOST_IP):8501 - - name: CONSUL_GRPC_ADDR - value: https://$(HOST_IP):8502 - - name: CONSUL_CACERT - value: /consul/tls/ca/tls.crt + {{- if $root.Values.externalServers.enabled }} + -grpc-port={{ $root.Values.externalServers.grpcPort }} \ {{- else }} - - name: CONSUL_HTTP_ADDR - value: http://$(HOST_IP):8500 - - name: CONSUL_GRPC_ADDR - value: $(HOST_IP):8502 + -grpc-port=8502 \ {{- end }} - command: - - /consul-bin/consul - - connect - - envoy - - -gateway=ingress - - -proxy-id=$(POD_NAME) - - -address=$(POD_IP):21000 + -proxy-service-id=$POD_NAME \ + -service-node-name="k8s-service-mesh" \ {{- if $root.Values.global.enableConsulNamespaces }} - - -namespace={{ default $defaults.consulNamespace .consulNamespace }} - {{- end }} - {{- if $root.Values.global.adminPartitions.enabled }} - - -partition={{ $root.Values.global.adminPartitions.name }} + -service-namespace={{ (default $defaults.consulNamespace .consulNamespace) }} \ {{- end }} - livenessProbe: - tcpSocket: - port: 21000 - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 21000 - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - ports: - - name: gateway-health - containerPort: 21000 - {{- range $index, $allPorts := (default $defaults.service.ports $service.ports) }} - - name: gateway-{{ $index }} - containerPort: {{ $allPorts.port }} + {{- if and $root.Values.global.tls.enabled }} + {{- if (not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots)) }} + -ca-certs=/consul/tls/ca/tls.crt \ {{- end }} - lifecycle: - preStop: - exec: - command: - - "/bin/sh" - - "-ec" - - | - /consul-bin/consul services deregister \ - {{- if $root.Values.global.enableConsulNamespaces }} - -namespace={{ default $defaults.consulNamespace .consulNamespace }} \ - {{- end }} - {{- if $root.Values.global.adminPartitions.enabled }} - -partition={{ $root.Values.global.adminPartitions.name }} \ - {{- end }} - -id="${POD_NAME}" - {{- if $root.Values.global.acls.manageSystemACLs }} - - "/consul-bin/consul logout" - {{- end}} - - # consul-sidecar ensures the ingress gateway is always registered with - # the local Consul agent, even if it loses the initial registration. - - name: consul-sidecar - image: {{ $root.Values.global.imageK8S }} - volumeMounts: - - name: consul-service - mountPath: /consul/service - readOnly: true - - name: consul-bin - mountPath: /consul-bin - {{- if $root.Values.global.tls.enabled }} - {{- if $root.Values.global.tls.enableAutoEncrypt }} - - name: consul-auto-encrypt-ca-cert + {{- if and $root.Values.externalServers.enabled $root.Values.externalServers.tlsServerName }} + -tls-server-name={{$root.Values.externalServers.tlsServerName }} \ {{- else }} - - name: consul-ca-cert - {{- end }} - mountPath: /consul/tls/ca - readOnly: true + -tls-server-name=server.{{ $root.Values.global.datacenter }}.{{ $root.Values.global.domain }} \ {{- end }} - {{- if $root.Values.global.consulSidecarContainer }} - {{- if $root.Values.global.consulSidecarContainer.resources }} - resources: {{ toYaml $root.Values.global.consulSidecarContainer.resources | nindent 12 }} - {{- end }} - {{- end }} - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - {{- if $root.Values.global.tls.enabled }} - - name: CONSUL_HTTP_ADDR - value: https://$(HOST_IP):8501 - - name: CONSUL_CACERT - value: /consul/tls/ca/tls.crt {{- else }} - - name: CONSUL_HTTP_ADDR - value: http://$(HOST_IP):8500 + -tls-disabled \ {{- end }} - command: - - consul-k8s-control-plane - - consul-sidecar - - -log-level={{ $root.Values.global.logLevel }} - - -log-json={{ $root.Values.global.logJSON }} - - -service-config=/consul/service/service.hcl - - -consul-binary=/consul-bin/consul - - -consul-api-timeout={{ $root.Values.global.consulAPITimeout }} {{- if $root.Values.global.acls.manageSystemACLs }} - - -token-file=/consul/service/acl-token + -credential-type=login \ + -login-bearer-path=/var/run/secrets/kubernetes.io/serviceaccount/token \ + -login-meta=component=ingress-gateway \ + -login-meta=pod=${NAMESPACE}/${POD_NAME} \ + -login-method={{ template "consul.fullname" $root }}-k8s-component-auth-method \ + {{- if $root.Values.global.adminPartitions.enabled }} + -login-partition={{ $root.Values.global.adminPartitions.name }} \ + {{- end }} {{- end }} + {{- if $root.Values.global.adminPartitions.enabled }} + -service-partition={{ $root.Values.global.adminPartitions.name }} \ + {{- end }} + -log-level={{ default $root.Values.global.logLevel }} \ + -log-json={{ $root.Values.global.logJSON }} + livenessProbe: + tcpSocket: + port: 21000 + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 21000 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - name: gateway-health + containerPort: 21000 + {{- range $index, $allPorts := (default $defaults.service.ports $service.ports) }} + - name: gateway-{{ $index }} + containerPort: {{ $allPorts.port }} + {{- end }} {{- if (default $defaults.priorityClassName .priorityClassName) }} priorityClassName: {{ default $defaults.priorityClassName .priorityClassName | quote }} {{- end }} diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml index ce28fbbdd3..27f287b2fe 100644 --- a/charts/consul/templates/mesh-gateway-deployment.yaml +++ b/charts/consul/templates/mesh-gateway-deployment.yaml @@ -35,18 +35,18 @@ spec: annotations: "consul.hashicorp.com/connect-inject": "false" "consul.hashicorp.com/gateway-kind": "mesh-gateway" - "consul.hashicorp.com/mesh-gateway-consul-service-name": "{{ .Values.meshGateway.consulServiceName }}" - "consul.hashicorp.com/mesh-gateway-wan-address-source": "{{ .Values.meshGateway.wanAddress.source }}" + "consul.hashicorp.com/gateway-consul-service-name": "{{ .Values.meshGateway.consulServiceName }}" "consul.hashicorp.com/mesh-gateway-container-port": "{{ .Values.meshGateway.containerPort }}" - "consul.hashicorp.com/mesh-gateway-wan-address-static": "{{ .Values.meshGateway.wanAddress.static }}" + "consul.hashicorp.com/gateway-wan-address-source": "{{ .Values.meshGateway.wanAddress.source }}" + "consul.hashicorp.com/gateway-wan-address-static": "{{ .Values.meshGateway.wanAddress.static }}" {{- if eq .Values.meshGateway.wanAddress.source "Service" }} {{- if eq .Values.meshGateway.service.type "NodePort" }} - "consul.hashicorp.com/mesh-gateway-wan-port": "{{ .Values.meshGateway.service.nodePort }}" + "consul.hashicorp.com/gateway-wan-port": "{{ .Values.meshGateway.service.nodePort }}" {{- else }} - "consul.hashicorp.com/mesh-gateway-wan-port": "{{ .Values.meshGateway.service.port }}" + "consul.hashicorp.com/gateway-wan-port": "{{ .Values.meshGateway.service.port }}" {{- end }} {{- else }} - "consul.hashicorp.com/mesh-gateway-wan-port": "{{ .Values.meshGateway.wanAddress.port }}" + "consul.hashicorp.com/gateway-wan-port": "{{ .Values.meshGateway.wanAddress.port }}" {{- end }} {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }} "vault.hashicorp.com/agent-init-first": "true" @@ -172,9 +172,6 @@ spec: {{- end }} {{- end }} volumeMounts: - - mountPath: /consul/service - name: consul-service - readOnly: true {{- if .Values.global.tls.enabled }} - name: consul-ca-cert mountPath: /consul/tls/ca @@ -206,7 +203,7 @@ spec: {{- end }} -proxy-service-id=$POD_NAME \ -service-node-name="k8s-service-mesh" \ - {{- if and .Values.global.tls.enabled }} + {{- if .Values.global.tls.enabled }} {{- if (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) }} -ca-certs=/consul/tls/ca/tls.crt \ {{- end }} diff --git a/charts/consul/test/unit/ingress-gateways-deployment.bats b/charts/consul/test/unit/ingress-gateways-deployment.bats index 7c96569e3d..e89e6abd4f 100644 --- a/charts/consul/test/unit/ingress-gateways-deployment.bats +++ b/charts/consul/test/unit/ingress-gateways-deployment.bats @@ -39,27 +39,6 @@ load _helpers [ "${actual}" = "release-name-consul-ingress-gateway" ] } -@test "ingressGateways/Deployment: Adds consul service volumeMount to gateway container" { - cd `chart_dir` - local object=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | yq '.spec.template.spec.containers[0].volumeMounts[1]' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.name' | tee /dev/stderr) - [ "${actual}" = "consul-service" ] - - local actual=$(echo $object | - yq -r '.mountPath' | tee /dev/stderr) - [ "${actual}" = "/consul/service" ] - - local actual=$(echo $object | - yq -r '.readOnly' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - #-------------------------------------------------------------------- # prerequisites @@ -104,27 +83,15 @@ load _helpers [[ "$output" =~ "terminating gateways cannot have duplicate names of any ingress gateways" ]] } #-------------------------------------------------------------------- -# envoyImage - -@test "ingressGateways/Deployment: envoy image has default global value" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[0].image' | tee /dev/stderr) - [[ "${actual}" =~ "envoyproxy/envoy:v" ]] - -} +# dataplaneImage -@test "ingressGateways/Deployment: envoy image can be set using the global value" { +@test "ingressGateways/Deployment: dataplane image can be set using the global value" { cd `chart_dir` local actual=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'global.imageEnvoy=new/image' \ + --set 'global.imageConsulDataplane=new/image' \ . | tee /dev/stderr | yq -s -r '.[0].spec.template.spec.containers[0].image' | tee /dev/stderr) [ "${actual}" = "new/image" ] @@ -133,41 +100,35 @@ load _helpers #-------------------------------------------------------------------- # global.tls.enabled -@test "ingressGateways/Deployment: sets TLS env variables when global.tls.enabled" { +@test "ingressGateways/Deployment: sets flags when global.tls.enabled is flags" { cd `chart_dir` - local env=$(helm template \ + local object=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ + --set 'global.tls.enabled=false' \ . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[0].env[]' | tee /dev/stderr) + yq -s -r '.[0].spec.template.spec.containers[0].command[2]' | tee /dev/stderr) - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_ADDR") | .value' | tee /dev/stderr) - [ "${actual}" = 'https://$(HOST_IP):8501' ] - - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_GRPC_ADDR") | .value' | tee /dev/stderr) - [ "${actual}" = 'https://$(HOST_IP):8502' ] - - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_CACERT") | .value' | tee /dev/stderr) - [ "${actual}" = "/consul/tls/ca/tls.crt" ] + local actual=$(echo $object | yq -r '. | contains("-tls-disabled")' | tee /dev/stderr) + [ "${actual}" = "true" ] } -@test "ingressGateways/Deployment: sets TLS env variables in consul sidecar when global.tls.enabled" { +@test "ingressGateways/Deployment: sets TLS flags when global.tls.enabled" { cd `chart_dir` - local env=$(helm template \ + local object=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[1].env[]' | tee /dev/stderr) + yq -s -r '.[0].spec.template.spec.containers[0].command[2]' | tee /dev/stderr) - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_ADDR") | .value' | tee /dev/stderr) - [ "${actual}" = 'https://$(HOST_IP):8501' ] + local actual=$(echo $object | yq -r '. | contains("-ca-certs=/consul/tls/ca/tls.crt")' | tee /dev/stderr) + [ "${actual}" = "true" ] - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_CACERT") | .value' | tee /dev/stderr) - [ "${actual}" = "/consul/tls/ca/tls.crt" ] + local actual=$(echo $object | yq -r '. | contains("-tls-server-name=server.dc1.consul")' | tee /dev/stderr) + [ "${actual}" = "true" ] } @test "ingressGateways/Deployment: can overwrite CA secret with the provided one" { @@ -206,103 +167,39 @@ load _helpers } #-------------------------------------------------------------------- -# global.tls.enableAutoEncrypt +# global.acls.manageSystemACLs -@test "ingressGateways/Deployment: consul-auto-encrypt-ca-cert volume is added when TLS with auto-encrypt is enabled" { +@test "ingressGateways/Deployment: Adds consul envvars on ingress-gateway-init init container when ACLs are enabled and tls is enabled" { cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ + local env=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ + --set 'global.acls.manageSystemACLs=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ . | tee /dev/stderr | - yq -s '.[0].spec.template.spec.volumes[] | select(.name == "consul-auto-encrypt-ca-cert") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} + yq -r '.spec.template.spec.initContainers[0].env[]' | tee /dev/stderr) -@test "ingressGateways/Deployment: consul-auto-encrypt-ca-cert volumeMount is added when TLS with auto-encrypt is enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq -s '.[0].spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-auto-encrypt-ca-cert") | length > 0' | tee /dev/stderr) - [ "${actual}" = "true" ] -} + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_PORT") | .value' | tee /dev/stderr) + [ "${actual}" = "8501" ] -@test "ingressGateways/Deployment: get-auto-encrypt-client-ca init container is created when TLS with auto-encrypt is enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq -s '.[0].spec.template.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca") | length > 0' | tee /dev/stderr) + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_USE_TLS") | .value' | tee /dev/stderr) [ "${actual}" = "true" ] -} -@test "ingressGateways/Deployment: consul-ca-cert volume is not added if externalServers.enabled=true and externalServers.useSystemRoots=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'externalServers.enabled=true' \ - --set 'externalServers.hosts[0]=foo.com' \ - --set 'externalServers.useSystemRoots=true' \ - . | tee /dev/stderr | - yq -s '.[0].spec.template.spec.volumes[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) - [ "${actual}" = "" ] -} - -#-------------------------------------------------------------------- -# global.acls.manageSystemACLs + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_TLS_SERVER_NAME") | .value' | tee /dev/stderr) + [ "${actual}" = "server.dc1.consul" ] -@test "ingressGateways/Deployment: consul-sidecar uses -token-file flag when global.acls.manageSystemACLs=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq -s '[.[0].spec.template.spec.containers[1].command[7]] | any(contains("-token-file=/consul/service/acl-token"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_CACERT_FILE") | .value' | tee /dev/stderr) + [ "${actual}" = "/consul/tls/ca/tls.crt" ] -@test "ingressGateways/Deployment: consul-sidecar uses -consul-api-timeout flag" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -s '[.[0].spec.template.spec.containers[1].command[6]] | any(contains("-consul-api-timeout=5s"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_AUTH_METHOD") | .value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method" ] -@test "ingressGateways/Deployment: Adds consul envvars CONSUL_HTTP_ADDR on ingress-gateway-init init container when ACLs are enabled and tls is enabled" { - cd `chart_dir` - local env=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.initContainers[1].env[]' | tee /dev/stderr) + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_DATACENTER") | .value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] - local actual - actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_ADDR") | .value' | tee /dev/stderr) - [ "${actual}" = "https://\$(HOST_IP):8501" ] + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_META") | .value' | tee /dev/stderr) + [ "${actual}" = 'component=ingress-gateway,pod=$(NAMESPACE)/$(POD_NAME)' ] } @test "ingressGateways/Deployment: Adds consul envvars CONSUL_HTTP_ADDR on ingress-gateway-init init container when ACLs are enabled and tls is not enabled" { @@ -314,74 +211,73 @@ load _helpers --set 'ingressGateways.enabled=true' \ --set 'global.acls.manageSystemACLs=true' \ . | tee /dev/stderr | - yq -r '.spec.template.spec.initContainers[1].env[]' | tee /dev/stderr) + yq -r '.spec.template.spec.initContainers[0].env[]' | tee /dev/stderr) - local actual - actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_ADDR") | .value' | tee /dev/stderr) - [ "${actual}" = "http://\$(HOST_IP):8500" ] -} + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_HTTP_PORT") | .value' | tee /dev/stderr) + [ "${actual}" = "8500" ] -@test "ingressGateways/Deployment: Does not add consul envvars CONSUL_CACERT on ingress-gateway-init init container when ACLs are enabled and tls is not enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1].env[] | select(.name == "CONSUL_CACERT")' | tee /dev/stderr) + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_AUTH_METHOD") | .value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method" ] - [ "${actual}" = "" ] -} + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_DATACENTER") | .value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] -@test "ingressGateways/Deployment: Adds consul envvars CONSUL_CACERT on ingress-gateway-init init container when ACLs are enabled and tls is enabled" { - cd `chart_dir` - local env=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.initContainers[1].env[]' | tee /dev/stderr) + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_LOGIN_META") | .value' | tee /dev/stderr) + [ "${actual}" = 'component=ingress-gateway,pod=$(NAMESPACE)/$(POD_NAME)' ] - local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_CACERT") | .value' | tee /dev/stderr) - [ "${actual}" = "/consul/tls/ca/tls.crt" ] + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_USE_TLS") | .value' | tee /dev/stderr) + [ "${actual}" = "" ] + + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_TLS_SERVER_NAME") | .value' | tee /dev/stderr) + [ "${actual}" = "" ] + + local actual=$(echo $env | jq -r '. | select(.name == "CONSUL_CACERT_FILE") | .value' | tee /dev/stderr) + [ "${actual}" = "" ] } -@test "ingressGateways/Deployment: CONSUL_HTTP_TOKEN_FILE is not set when acls are disabled" { +@test "ingressGateways/Deployment: ACL flags is not set when acls are disabled" { cd `chart_dir` - local actual=$(helm template \ + local object=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ --set 'connectInject.enabled=true' \ --set 'ingressGateways.enabled=true' \ --set 'global.acls.manageSystemACLs=false' \ . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].env[0].name] | any(contains("CONSUL_HTTP_TOKEN_FILE"))' | tee /dev/stderr) + yq -s -r '.[0].spec.template.spec.containers[0].command[2]' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '. | contains("-login-bearer-path")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -r '. | contains("-login-meta")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -r '. | contains("-login-method")' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$(echo $object | yq -r '. | contains("-credential-type=login")' | tee /dev/stderr) [ "${actual}" = "false" ] } @test "ingressGateways/Deployment: CONSUL_HTTP_TOKEN_FILE is set when acls are enabled" { cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ + local object=$(helm template \ + -s templates/ingress-gateways-deployment.yaml \ --set 'connectInject.enabled=true' \ + --set 'ingressGateways.enabled=true' \ --set 'global.acls.manageSystemACLs=true' \ . | tee /dev/stderr | - yq -s '[.[0].spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_HTTP_TOKEN_FILE"))' | tee /dev/stderr) + yq -s -r '.[0].spec.template.spec.containers[0].command[2]' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '. | contains("-login-bearer-path=/var/run/secrets/kubernetes.io/serviceaccount/token")' | tee /dev/stderr) [ "${actual}" = "true" ] -} -@test "ingressGateways/Deployment: consul-logout preStop hook is added when ACLs are enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].lifecycle.preStop.exec.command[3]] | any(contains("/consul-bin/consul logout"))' | tee /dev/stderr) + local actual=$(echo $object | yq -r '. | contains("-login-meta=component=ingress-gateway")' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | yq -r '. | contains("-login-method=release-name-consul-k8s-component-auth-method")' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | yq -r '. | contains("-credential-type=login")' | tee /dev/stderr) [ "${actual}" = "true" ] } @@ -424,19 +320,6 @@ load _helpers [ "${actual}" = "/metrics" ] } -@test "ingressGateways/Deployment: when global.metrics.enabled=true, sets proxy setting" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.metrics.enabled=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) - - [ "${actual}" = "true" ] -} - @test "ingressGateways/Deployment: when global.metrics.enableGatewayMetrics=false, does not set proxy setting" { cd `chart_dir` local object=$(helm template \ @@ -448,9 +331,6 @@ load _helpers . | tee /dev/stderr | yq '.spec.template' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) - [ "${actual}" = "false" ] - local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) [ "${actual}" = "null" ] @@ -471,9 +351,6 @@ load _helpers . | tee /dev/stderr | yq '.spec.template' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.spec.initContainers[1].command | join(" ") | contains("envoy_prometheus_bind_addr = \"${POD_IP}:20200\"")' | tee /dev/stderr) - [ "${actual}" = "false" ] - local actual=$(echo $object | yq -s -r '.[0].metadata.annotations."prometheus.io/path"' | tee /dev/stderr) [ "${actual}" = "null" ] @@ -697,135 +574,16 @@ load _helpers yq -s -r '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr) local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "25Mi" ] + [ "${actual}" = "50Mi" ] local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) [ "${actual}" = "50m" ] - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) - [ "${actual}" = "150Mi" ] - - local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "50m" ] -} - -@test "ingressGateways/Deployment: init container resources can be set through defaults" { - cd `chart_dir` - local object=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.requests.memory=memory' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.requests.cpu=cpu' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.limits.memory=memory2' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.limits.cpu=cpu2' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "memory" ] - - local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu" ] - - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) - [ "${actual}" = "memory2" ] - - local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu2" ] -} - -@test "ingressGateways/Deployment: init container resources can be set through specific gateway, overriding defaults" { - cd `chart_dir` - local object=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.requests.memory=memory' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.requests.cpu=cpu' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.limits.memory=memory2' \ - --set 'ingressGateways.defaults.initCopyConsulContainer.resources.limits.cpu=cpu2' \ - --set 'ingressGateways.gateways[0].name=gateway1' \ - --set 'ingressGateways.gateways[0].initCopyConsulContainer.resources.requests.memory=gwmemory' \ - --set 'ingressGateways.gateways[0].initCopyConsulContainer.resources.requests.cpu=gwcpu' \ - --set 'ingressGateways.gateways[0].initCopyConsulContainer.resources.limits.memory=gwmemory2' \ - --set 'ingressGateways.gateways[0].initCopyConsulContainer.resources.limits.cpu=gwcpu2' \ - . | tee /dev/stderr | - yq -s '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "gwmemory" ] - - local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) - [ "${actual}" = "gwcpu" ] - - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) - [ "${actual}" = "gwmemory2" ] - - local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "gwcpu2" ] -} - -#-------------------------------------------------------------------- -# consul sidecar resources - -@test "ingressGateways/Deployment: consul sidecar has default resources" { - cd `chart_dir` - local object=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[1].resources' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "25Mi" ] - - local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) - [ "${actual}" = "20m" ] - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) [ "${actual}" = "50Mi" ] local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "20m" ] -} - -@test "ingressGateways/Deployment: consul sidecar resources can be set" { - cd `chart_dir` - local object=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.consulSidecarContainer.resources.requests.memory=memory' \ - --set 'global.consulSidecarContainer.resources.requests.cpu=cpu' \ - --set 'global.consulSidecarContainer.resources.limits.memory=memory2' \ - --set 'global.consulSidecarContainer.resources.limits.cpu=cpu2' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[1].resources' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) - [ "${actual}" = "memory" ] - - local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu" ] - - local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) - [ "${actual}" = "memory2" ] - - local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) - [ "${actual}" = "cpu2" ] -} - -@test "ingressGateways/Deployment: fails if global.lifecycleSidecarContainer is set" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.lifecycleSidecarContainer.resources.requests.memory=100Mi' . - [ "$status" -eq 1 ] - [[ "$output" =~ "global.lifecycleSidecarContainer has been renamed to global.consulSidecarContainer. Please set values using global.consulSidecarContainer." ]] + [ "${actual}" = "50m" ] } #-------------------------------------------------------------------- @@ -1040,7 +798,7 @@ load _helpers --set 'connectInject.enabled=true' \ . | tee /dev/stderr | yq -s -r '.[0].spec.template.metadata.annotations | length' | tee /dev/stderr) - [ "${actual}" = "1" ] + [ "${actual}" = "5" ] } @test "ingressGateways/Deployment: extra annotations can be set through defaults" { @@ -1055,7 +813,7 @@ key2: value2' \ yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) local actual=$(echo $object | yq '. | length' | tee /dev/stderr) - [ "${actual}" = "3" ] + [ "${actual}" = "7" ] local actual=$(echo $object | yq -r '.key1' | tee /dev/stderr) [ "${actual}" = "value1" ] @@ -1077,7 +835,7 @@ key2: value2' \ yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) local actual=$(echo $object | yq '. | length' | tee /dev/stderr) - [ "${actual}" = "3" ] + [ "${actual}" = "7" ] local actual=$(echo $object | yq -r '.key1' | tee /dev/stderr) [ "${actual}" = "value1" ] @@ -1100,7 +858,7 @@ key2: value2' \ yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) local actual=$(echo $object | yq '. | length' | tee /dev/stderr) - [ "${actual}" = "4" ] + [ "${actual}" = "8" ] local actual=$(echo $object | yq -r '.defaultkey' | tee /dev/stderr) [ "${actual}" = "defaultvalue" ] @@ -1112,222 +870,6 @@ key2: value2' \ [ "${actual}" = "value2" ] } -#-------------------------------------------------------------------- -# WAN_ADDR - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for ClusterIP service set in defaults (the default)" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"$(cat /tmp/address.txt)\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for ClusterIP service set in specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=Static' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=ClusterIP' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"$(cat /tmp/address.txt)\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for LoadBalancer service set in defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=LoadBalancer' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"$(cat /tmp/address.txt)\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for LoadBalancer service set in specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=LoadBalancer' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"$(cat /tmp/address.txt)\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for NodePort service set in defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - --set 'ingressGateways.defaults.service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"${HOST_IP}\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR set correctly for NodePort service set in specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=NodePort' \ - --set 'ingressGateways.gateways[0].service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_ADDR=\"${HOST_IP}\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_ADDR definition fails if using unknown service type in defaults" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=Static' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "currently set ingressGateway value service.type is not supported" ]] -} - -@test "ingressGateways/Deployment: WAN_ADDR definition fails if using unknown service type in specific gateway overriding defaults" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=Static' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "currently set ingressGateway value service.type is not supported" ]] -} - -#-------------------------------------------------------------------- -# WAN_PORT - -@test "ingressGateways/Deployment: WAN_PORT set correctly for non-NodePort service in defaults (the default)" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_PORT=80")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_PORT can be set for non-NodePort service in defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.ports[0].port=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_PORT=1234")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_PORT set correctly for non-NodePort service in specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.ports[0].port=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_PORT=1234")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_PORT set correctly for NodePort service with nodePort set in defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - --set 'ingressGateways.defaults.service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_PORT=1234")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_PORT set correctly for NodePort service with nodePort set in specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.ports[0].nodePort=8888' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=NodePort' \ - --set 'ingressGateways.gateways[0].service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("WAN_PORT=1234")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: WAN_PORT definition fails if .service.type=NodePort and ports[0].nodePort is empty in defaults" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "if ingressGateways .service.type=NodePort and using ingressGateways.defaults.service.ports, the first port entry must include a nodePort" ]] -} - -@test "ingressGateways/Deployment: WAN_PORT definition fails if .service.type=NodePort and ports[0].nodePort is empty in specific gateway and not provided in defaults" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.ports[0].port=1234' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "if ingressGateways .service.type=NodePort and defining ingressGateways.gateways.service.ports, the first port entry must include a nodePort" ]] -} - -@test "ingressGateways/Deployment: WAN_PORT definition fails if .service.type=NodePort and ports[0].nodePort is empty in defaults and specific gateway" { - cd `chart_dir` - run helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - --set 'ingressGateways.defaults.service.ports=null' \ - . - - [ "$status" -eq 1 ] - [[ "$output" =~ "if ingressGateways .service.type=NodePort, the first port entry in either the defaults or specific gateway must include a nodePort" ]] -} - #-------------------------------------------------------------------- # ingress-gateway-init init container @@ -1340,185 +882,17 @@ key2: value2' \ . | tee /dev/stderr | yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2]' | tee /dev/stderr) - exp='consul-k8s-control-plane service-address \ - -log-level=info \ - -log-json=false \ - -k8s-namespace=default \ - -name=release-name-consul-ingress-gateway \ - -output-file=/tmp/address.txt -WAN_ADDR="$(cat /tmp/address.txt)" -WAN_PORT=8080 - -cat > /consul/service/service.hcl << EOF -service { - kind = "ingress-gateway" - name = "ingress-gateway" - id = "${POD_NAME}" - port = ${WAN_PORT} - address = "${WAN_ADDR}" - tagged_addresses { - lan { - address = "${POD_IP}" - port = 21000 - } - wan { - address = "${WAN_ADDR}" - port = ${WAN_PORT} - } - } - proxy { - config { - envoy_gateway_no_default_bind = true - envoy_gateway_bind_addresses { - all-interfaces { - address = "0.0.0.0" - } - } - } - } - checks = [ - { - name = "Ingress Gateway Listening" - interval = "10s" - tcp = "${POD_IP}:21000" - deregister_critical_service_after = "6h" - } - ] -} -EOF - -/consul-bin/consul services register \ - /consul/service/service.hcl' - - [ "${actual}" = "${exp}" ] -} - -@test "ingressGateways/Deployment: ingress-gateway-init init container with acls.manageSystemACLs=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2]' | tee /dev/stderr) - - exp='consul-k8s-control-plane acl-init \ - -component-name=ingress-gateway/release-name-consul-ingress-gateway \ - -acl-auth-method=release-name-consul-k8s-component-auth-method \ - -token-sink-file=/consul/service/acl-token \ - -consul-api-timeout=5s \ - -log-level=info \ - -log-json=false - -consul-k8s-control-plane service-address \ - -log-level=info \ - -log-json=false \ - -k8s-namespace=default \ - -name=release-name-consul-ingress-gateway \ - -output-file=/tmp/address.txt -WAN_ADDR="$(cat /tmp/address.txt)" -WAN_PORT=8080 - -cat > /consul/service/service.hcl << EOF -service { - kind = "ingress-gateway" - name = "ingress-gateway" - id = "${POD_NAME}" - port = ${WAN_PORT} - address = "${WAN_ADDR}" - tagged_addresses { - lan { - address = "${POD_IP}" - port = 21000 - } - wan { - address = "${WAN_ADDR}" - port = ${WAN_PORT} - } - } - proxy { - config { - envoy_gateway_no_default_bind = true - envoy_gateway_bind_addresses { - all-interfaces { - address = "0.0.0.0" - } - } - } - } - checks = [ - { - name = "Ingress Gateway Listening" - interval = "10s" - tcp = "${POD_IP}:21000" - deregister_critical_service_after = "6h" - } - ] -} -EOF - -/consul-bin/consul services register \ - -token-file=/consul/service/acl-token \ - /consul/service/service.hcl' + exp='consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${NAMESPACE} \ +-gateway-kind="ingress-gateway" \ +-consul-node-name="k8s-service-mesh" \ +-proxy-id-file=/consul/service/proxy-id \ +-service-name=release-name-consul-ingress-gateway \ +-log-level=info \ +-log-json=false' [ "${actual}" = "${exp}" ] } -@test "ingressGateways/Deployment: ingress-gateway-init init container includes service-address command for LoadBalancer set through defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=LoadBalancer' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("consul-k8s-control-plane service-address")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: ingress-gateway-init init container includes service-address command for LoadBalancer set through specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=LoadBalancer' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("consul-k8s-control-plane service-address")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "ingressGateways/Deployment: ingress-gateway-init init container does not include service-address command for NodePort set through defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.defaults.service.type=NodePort' \ - --set 'ingressGateways.defaults.service.ports[0].port=80' \ - --set 'ingressGateways.defaults.service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("consul-k8s-control-plane service-address")' | tee /dev/stderr) - [ "${actual}" = "false" ] -} - -@test "ingressGateways/Deployment: ingress-gateway-init init container does not include service-address command for NodePort set through specific gateway overriding defaults" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=ingress-gateway' \ - --set 'ingressGateways.gateways[0].service.type=NodePort' \ - --set 'ingressGateways.gateways[0].service.ports[0].port=80' \ - --set 'ingressGateways.gateways[0].service.ports[0].nodePort=1234' \ - . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.initContainers | map(select(.name == "ingress-gateway-init"))[0] | .command[2] | contains("consul-k8s-control-plane service-address")' | tee /dev/stderr) - [ "${actual}" = "false" ] -} - #-------------------------------------------------------------------- # namespaces @@ -1529,13 +903,10 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[0]' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.command | any(contains("-namespace"))' | tee /dev/stderr) - [ "${actual}" = "false" ] + yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.lifecycle.preStop.exec.command | any(contains("-namespace"))' | tee /dev/stderr) - [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '."consul.hashicorp.com/gateway-namespace"' | tee /dev/stderr) + [ "${actual}" = "null" ] } @test "ingressGateways/Deployment: namespace command flag is specified through defaults" { @@ -1547,16 +918,13 @@ EOF --set 'global.enableConsulNamespaces=true' \ --set 'ingressGateways.defaults.consulNamespace=namespace' \ . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[0]' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.command | any(contains("-namespace=namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] + yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.lifecycle.preStop.exec.command | any(contains("-namespace=namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] + local actual=$(echo $object | yq -r '."consul.hashicorp.com/gateway-namespace"' | tee /dev/stderr) + [ "${actual}" = "namespace" ] } -@test "ingressGateways/Deployment: namespace command flag is specified through specific gateway overriding defaults" { +@test "ingressGateways/Deployment: namespace annotation is specified through specific gateway overriding defaults" { cd `chart_dir` local object=$(helm template \ -s templates/ingress-gateways-deployment.yaml \ @@ -1567,13 +935,10 @@ EOF --set 'ingressGateways.gateways[0].name=ingress-gateway' \ --set 'ingressGateways.gateways[0].consulNamespace=new-namespace' \ . | tee /dev/stderr | - yq -s -r '.[0].spec.template.spec.containers[0]' | tee /dev/stderr) - - local actual=$(echo $object | yq -r '.command | any(contains("-namespace=new-namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] + yq -s -r '.[0].spec.template.metadata.annotations' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.lifecycle.preStop.exec.command | any(contains("-namespace=new-namespace"))' | tee /dev/stderr) - [ "${actual}" = "true" ] + local actual=$(echo $object | yq -r '."consul.hashicorp.com/gateway-namespace"' | tee /dev/stderr) + [ "${actual}" = "new-namespace" ] } #-------------------------------------------------------------------- @@ -1590,9 +955,6 @@ EOF local actual=$(echo $object | yq -r '.command | any(contains("-partition"))' | tee /dev/stderr) [ "${actual}" = "false" ] - - local actual=$(echo $object | yq -r '.lifecycle.preStop.exec.command | any(contains("-partition"))' | tee /dev/stderr) - [ "${actual}" = "false" ] } @test "ingressGateways/Deployment: partition command flag is specified through partition name" { @@ -1607,10 +969,7 @@ EOF . | tee /dev/stderr | yq -s -r '.[0].spec.template.spec.containers[0]' | tee /dev/stderr) - local actual=$(echo $object | yq -r '.command | any(contains("-partition=default"))' | tee /dev/stderr) - [ "${actual}" = "true" ] - - local actual=$(echo $object | yq -r '.lifecycle.preStop.exec.command | any(contains("-partition=default"))' | tee /dev/stderr) + local actual=$(echo $object | yq -r '.command | any(contains("-service-partition=default"))' | tee /dev/stderr) [ "${actual}" = "true" ] } @@ -1657,38 +1016,6 @@ EOF [ "${actual}" = "false" ] } -#-------------------------------------------------------------------- -# get-auto-encrypt-client-ca - -@test "ingressGateways/Deployment: get-auto-encrypt-client-ca uses server's stateful set address by default and passes ca cert" { - cd `chart_dir` - local command=$(helm template \ - -s templates/ingress-gateways-deployment.yaml \ - --set 'ingressGateways.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'ingressGateways.gateways[0].name=gateway1' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - . | tee /dev/stderr | - yq '.spec.template.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca").command | join(" ")' | tee /dev/stderr) - - # check server address - actual=$(echo $command | jq ' . | contains("-server-addr=release-name-consul-server")') - [ "${actual}" = "true" ] - - # check server port - actual=$(echo $command | jq ' . | contains("-server-port=8501")') - [ "${actual}" = "true" ] - - # check server's CA cert - actual=$(echo $command | jq ' . | contains("-ca-file=/consul/tls/ca/tls.crt")') - [ "${actual}" = "true" ] - - # check consul-api-timeout - actual=$(echo $command | jq ' . | contains("-consul-api-timeout=5s")') - [ "${actual}" = "true" ] -} - #-------------------------------------------------------------------- # Vault @@ -1699,7 +1026,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=test' \ @@ -1732,7 +1058,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ @@ -1740,8 +1065,10 @@ EOF --set 'global.secretsBackend.vault.consulCARole=test' \ . | tee /dev/stderr | yq -r '.spec.template' | tee /dev/stderr) + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') [ "${actual}" = "false" ] } @@ -1754,7 +1081,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ @@ -1776,7 +1102,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ @@ -1798,7 +1123,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ @@ -1829,7 +1153,7 @@ EOF --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.consulCARole=carole' \ . | tee /dev/stderr | - yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role")' | tee /dev/stderr) + yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role") | del(."consul.hashicorp.com/gateway-wan-address-source") | del(."consul.hashicorp.com/gateway-wan-port") | del(."vconsul.hashicorp.com/gateway-wan-address-source") | del(."consul.hashicorp.com/gateway-consul-service-name") | del(."consul.hashicorp.com/gateway-kind")' | tee /dev/stderr) [ "${actual}" = "{}" ] } @@ -1840,7 +1164,6 @@ EOF --set 'ingressGateways.enabled=true' \ --set 'connectInject.enabled=true' \ --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=test' \ --set 'global.secretsBackend.vault.consulServerRole=foo' \ diff --git a/charts/consul/test/unit/mesh-gateway-deployment.bats b/charts/consul/test/unit/mesh-gateway-deployment.bats index 6312abcec9..fc9ecd3029 100755 --- a/charts/consul/test/unit/mesh-gateway-deployment.bats +++ b/charts/consul/test/unit/mesh-gateway-deployment.bats @@ -904,10 +904,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8888" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeIP" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "9999" ] } @@ -924,10 +924,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeIP" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } @@ -944,10 +944,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "NodeName" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } @@ -979,13 +979,13 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Static" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-static"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-static"]' | tee /dev/stderr) [ "${actual}" = "example.com" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } @@ -1004,10 +1004,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } @@ -1026,10 +1026,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "9999" ] } @@ -1062,10 +1062,10 @@ key2: value2' \ local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-container-port"]' | tee /dev/stderr) [ "${actual}" = "8443" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-address-source"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]' | tee /dev/stderr) [ "${actual}" = "Service" ] - local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/mesh-gateway-wan-port"]' | tee /dev/stderr) + local actual=$(echo $annotations | yq -r '.["consul.hashicorp.com/gateway-wan-port"]' | tee /dev/stderr) [ "${actual}" = "443" ] } @@ -1309,7 +1309,7 @@ key2: value2' \ --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.consulCARole=carole' \ . | tee /dev/stderr | - yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role") | del(."consul.hashicorp.com/gateway-kind") | del(."consul.hashicorp.com/mesh-gateway-wan-address-source") | del(."consul.hashicorp.com/mesh-gateway-container-port") | del(."consul.hashicorp.com/mesh-gateway-wan-address-static") | del(."consul.hashicorp.com/mesh-gateway-wan-port") | del(."consul.hashicorp.com/mesh-gateway-consul-service-name")' | tee /dev/stderr) + yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/role") | del(."consul.hashicorp.com/gateway-kind") | del(."consul.hashicorp.com/gateway-wan-address-source") | del(."consul.hashicorp.com/mesh-gateway-container-port") | del(."consul.hashicorp.com/gateway-wan-address-static") | del(."consul.hashicorp.com/gateway-wan-port") | del(."consul.hashicorp.com/gateway-consul-service-name")' | tee /dev/stderr) [ "${actual}" = "{}" ] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 555bb124ad..7671c5cd28 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -2665,18 +2665,6 @@ ingressGateways: memory: "100Mi" cpu: "100m" - # The resource settings for the `copy-consul-bin` init container. - # @recurse: false - # @type: map - initCopyConsulContainer: - resources: - requests: - memory: "25Mi" - cpu: "50m" - limits: - memory: "150Mi" - cpu: "50m" - # By default, we set an anti-affinity so that two of the same gateway pods # won't be on the same node. NOTE: Gateways require that Consul client agents are # also running on the nodes alongside each gateway pod. diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index 0ec49f7ae9..35a7e51577 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -25,27 +25,27 @@ const ( // value that is either "mesh", "ingress" or "terminating". annotationGatewayKind = "consul.hashicorp.com/gateway-kind" - // annotationMeshGatewayConsulServiceName is the key of the annotation whose value + // annotationGatewayConsulServiceName is the key of the annotation whose value // is the service name with which the mesh gateway is registered. - annotationMeshGatewayConsulServiceName = "consul.hashicorp.com/mesh-gateway-consul-service-name" + annotationGatewayConsulServiceName = "consul.hashicorp.com/gateway-consul-service-name" // annotationMeshGatewayContainerPort is the key of the annotation whose value is // used as the port and also registered as the LAN port when the mesh-gateway // service is registered. annotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" - // annotationMeshGatewaySource is the key of the annotation that determines which + // annotationGatewayWANSource is the key of the annotation that determines which // source to use to determine the wan address and wan port for the mesh-gateway // service registration. - annotationMeshGatewaySource = "consul.hashicorp.com/mesh-gateway-wan-address-source" + annotationGatewayWANSource = "consul.hashicorp.com/gateway-wan-address-source" - // annotationMeshGatewayWANAddress is the key of the annotation that when the source + // annotationGatewayWANAddress is the key of the annotation that when the source // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. - annotationMeshGatewayWANAddress = "consul.hashicorp.com/mesh-gateway-wan-address-static" + annotationGatewayWANAddress = "consul.hashicorp.com/gateway-wan-address-static" - // annotationMeshGatewayWANPort is the key of the annotation whose value is the + // annotationGatewayWANPort is the key of the annotation whose value is the // WAN port for the mesh-gateway service registration. - annotationMeshGatewayWANPort = "consul.hashicorp.com/mesh-gateway-wan-port" + annotationGatewayWANPort = "consul.hashicorp.com/gateway-wan-port" // annotationGatewayNamespace is the key of the annotation that indicates the // Consul namespace where a Terminating or Ingress Gateway pod is deployed. diff --git a/control-plane/connect-inject/endpoints_controller.go b/control-plane/connect-inject/endpoints_controller.go index 019bd5d0af..79f1cda068 100644 --- a/control-plane/connect-inject/endpoints_controller.go +++ b/control-plane/connect-inject/endpoints_controller.go @@ -43,7 +43,6 @@ const ( kubernetesSuccessReasonMsg = "Kubernetes health checks passing" envoyPrometheusBindAddr = "envoy_prometheus_bind_addr" sidecarContainer = "consul-dataplane" - wildcardNamespace = "*" defaultNS = "default" // clusterIPTaggedAddressName is the key for the tagged address to store the service's cluster IP and service port @@ -315,6 +314,13 @@ func (r *EndpointsController) registerGateway(apiClient *api.Client, pod corev1. return err } + if r.EnableConsulNamespaces { + if _, err := namespaces.EnsureExists(apiClient, serviceRegistration.Service.Namespace, r.CrossNSACLPolicy); err != nil { + r.Log.Error(err, "failed to ensure Consul namespace exists", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace, "consul ns", serviceRegistration.Service.Namespace) + return err + } + } + // Register the service instance with Consul. r.Log.Info("registering gateway with Consul", "name", serviceRegistration.Service.Service, "id", serviceRegistration.ID) @@ -636,7 +642,6 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service switch pod.Annotations[annotationGatewayKind] { case MeshGateway: service.Kind = api.ServiceKindMeshGateway - service.Service = MeshGateway if r.EnableConsulNamespaces { service.Namespace = defaultNS consulNS = defaultNS @@ -652,11 +657,11 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service meta[MetaKeyConsulWANFederation] = "1" } - meshGatewayServiceName, ok := pod.Annotations[annotationMeshGatewayConsulServiceName] + gatewayServiceName, ok := pod.Annotations[annotationGatewayConsulServiceName] if !ok { - return nil, fmt.Errorf("failed to read annontation %s from pod %s/%s", annotationMeshGatewayConsulServiceName, pod.Namespace, pod.Name) + return nil, fmt.Errorf("failed to read annontation %s from pod %s/%s", annotationGatewayConsulServiceName, pod.Namespace, pod.Name) } - service.Service = meshGatewayServiceName + service.Service = gatewayServiceName wanAddr, wanPort, err := r.getWanData(pod, serviceEndpoints) if err != nil { @@ -676,20 +681,62 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service service.Kind = api.ServiceKindTerminatingGateway service.Service = serviceEndpoints.Name service.Port = 8443 + if ns, ok := pod.Annotations[annotationGatewayNamespace]; ok && r.EnableConsulNamespaces { + service.Namespace = ns + consulNS = ns + } + case IngressGateway: + service.Kind = api.ServiceKindIngressGateway + gatewayServiceName, ok := pod.Annotations[annotationGatewayConsulServiceName] + if !ok { + return nil, fmt.Errorf("failed to read annontation %s from pod %s/%s", annotationGatewayConsulServiceName, pod.Namespace, pod.Name) + } + service.Service = gatewayServiceName if ns, ok := pod.Annotations[annotationGatewayNamespace]; ok && r.EnableConsulNamespaces { service.Namespace = ns consulNS = ns } + + wanAddr, wanPort, err := r.getWanData(pod, serviceEndpoints) + if err != nil { + return nil, err + } + service.Port = 21000 + service.TaggedAddresses = map[string]api.ServiceAddress{ + "lan": { + Address: pod.Status.PodIP, + Port: 21000, + }, + "wan": { + Address: wanAddr, + Port: wanPort, + }, + } + service.Proxy = &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "envoy_gateway_no_default_bind": true, + "envoy_gateway_bind_addresses": map[string]interface{}{ + "all-interfaces": map[string]interface{}{ + "address": "0.0.0.0", + }, + }, + }, + } + default: return nil, fmt.Errorf("%s must be one of %s, %s, or %s", annotationGatewayKind, MeshGateway, TerminatingGateway, IngressGateway) } if r.MetricsConfig.DefaultEnableMetrics && r.MetricsConfig.EnableGatewayMetrics { - service.Proxy = &api.AgentServiceConnectProxyConfig{ - Config: map[string]interface{}{ - "envoy_prometheus_bind_addr": fmt.Sprintf("%s:20200", pod.Status.PodIP), - }, + if pod.Annotations[annotationGatewayKind] == IngressGateway { + service.Proxy.Config["envoy_prometheus_bind_addr"] = fmt.Sprintf("%s:20200", pod.Status.PodIP) + } else { + service.Proxy = &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "envoy_prometheus_bind_addr": fmt.Sprintf("%s:20200", pod.Status.PodIP), + }, + } } } @@ -714,10 +761,9 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoints) (string, int, error) { var wanAddr string - var wanPort int - source, ok := pod.Annotations[annotationMeshGatewaySource] + source, ok := pod.Annotations[annotationGatewayWANSource] if !ok { - return "", 0, fmt.Errorf("failed to read annotation %s", annotationMeshGatewaySource) + return "", 0, fmt.Errorf("failed to read annotation %s", annotationGatewayWANSource) } switch source { case "NodeName": @@ -725,7 +771,7 @@ func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoi case "NodeIP": wanAddr = pod.Status.HostIP case "Static": - wanAddr = pod.Annotations[annotationMeshGatewayWANAddress] + wanAddr = pod.Annotations[annotationGatewayWANAddress] case "Service": svc, err := r.getService(endpoints) if err != nil { @@ -752,9 +798,9 @@ func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoi } } - wanPort, err := strconv.Atoi(pod.Annotations[annotationMeshGatewayWANPort]) + wanPort, err := strconv.Atoi(pod.Annotations[annotationGatewayWANPort]) if err != nil { - return "", 0, fmt.Errorf("failed to parse WAN port from value %s", pod.Annotations[annotationMeshGatewayWANPort]) + return "", 0, fmt.Errorf("failed to parse WAN port from value %s", pod.Annotations[annotationGatewayWANPort]) } return wanAddr, wanPort, nil } @@ -989,7 +1035,7 @@ func (r *EndpointsController) serviceInstancesForK8SServiceNameAndNamespace(apiC filter := fmt.Sprintf(`Meta[%q] == %q and Meta[%q] == %q and Meta[%q] == %q`, MetaKeyKubeServiceName, k8sServiceName, MetaKeyKubeNS, k8sServiceNamespace, MetaKeyManagedBy, managedByValue) if r.EnableConsulNamespaces { - serviceList, _, err = apiClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter, Namespace: wildcardNamespace}) + serviceList, _, err = apiClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter, Namespace: namespaces.WildcardNamespace}) } else { serviceList, _, err = apiClient.Catalog().NodeServiceList(ConsulNodeName, &api.QueryOptions{Filter: filter}) } diff --git a/control-plane/connect-inject/endpoints_controller_ent_test.go b/control-plane/connect-inject/endpoints_controller_ent_test.go index 7a28448577..a3535c6049 100644 --- a/control-plane/connect-inject/endpoints_controller_ent_test.go +++ b/control-plane/connect-inject/endpoints_controller_ent_test.go @@ -11,6 +11,8 @@ import ( logrtest "github.com/go-logr/logr/testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" "github.com/stretchr/testify/require" @@ -20,9 +22,6 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/hashicorp/consul-k8s/control-plane/helper/test" - "github.com/hashicorp/consul-k8s/control-plane/namespaces" ) // TestReconcileCreateEndpoint tests the logic to create service instances in Consul from the addresses in the Endpoints @@ -315,15 +314,39 @@ func TestReconcileCreateGatewayWithNamespaces(t *testing.T) { }{ k8sObjects: func() []runtime.Object { meshGateway := createGatewayWithNamespace("mesh-gateway", "default", "3.3.3.3", map[string]string{ - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: MeshGateway, - annotationMeshGatewayConsulServiceName: "mesh-gateway"}) + annotationGatewayWANSource: "Static", + annotationGatewayWANAddress: "2.3.4.5", + annotationGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: MeshGateway, + annotationGatewayConsulServiceName: "mesh-gateway"}) terminatingGateway := createGatewayWithNamespace("terminating-gateway", "default", "4.4.4.4", map[string]string{ annotationGatewayKind: TerminatingGateway, annotationGatewayNamespace: testCase.ConsulNS}) + ingressGateway := createGatewayWithNamespace("ingress-gateway", "default", "5.5.5.5", map[string]string{ + annotationGatewayWANSource: "Service", + annotationGatewayWANPort: "8443", + annotationGatewayNamespace: testCase.ConsulNS, + annotationGatewayKind: IngressGateway, + annotationGatewayConsulServiceName: "ingress-gateway"}) + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + { + IP: "5.6.7.8", + }, + }, + }, + }, + } endpoints := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "gateway", @@ -348,11 +371,19 @@ func TestReconcileCreateGatewayWithNamespaces(t *testing.T) { Namespace: "default", }, }, + { + IP: "5.5.5.5", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "ingress-gateway", + Namespace: "default", + }, + }, }, }, }, } - return []runtime.Object{meshGateway, terminatingGateway, endpoints} + return []runtime.Object{meshGateway, terminatingGateway, ingressGateway, svc, endpoints} }, expectedConsulSvcInstances: []*api.CatalogService{ { @@ -383,6 +414,25 @@ func TestReconcileCreateGatewayWithNamespaces(t *testing.T) { ServicePort: 8443, Namespace: testCase.ConsulNS, }, + { + ServiceID: "ingress-gateway", + ServiceName: "ingress-gateway", + ServiceAddress: "5.5.5.5", + ServiceMeta: map[string]string{MetaKeyPodName: "ingress-gateway", MetaKeyKubeServiceName: "gateway", MetaKeyKubeNS: "default", MetaKeyManagedBy: managedByValue}, + ServiceTags: []string{}, + ServicePort: 21000, + ServiceTaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "5.5.5.5", + Port: 21000, + }, + "wan": { + Address: "5.6.7.8", + Port: 8443, + }, + }, + Namespace: testCase.ConsulNS, + }, }, expectedHealthChecks: []*api.HealthCheck{ { @@ -405,6 +455,16 @@ func TestReconcileCreateGatewayWithNamespaces(t *testing.T) { Type: ConsulKubernetesCheckType, Namespace: testCase.ConsulNS, }, + { + CheckID: "default/ingress-gateway", + ServiceName: "ingress-gateway", + ServiceID: "ingress-gateway", + Name: ConsulKubernetesCheckName, + Status: api.HealthPassing, + Output: kubernetesSuccessReasonMsg, + Type: ConsulKubernetesCheckType, + Namespace: testCase.ConsulNS, + }, }, } t.Run(name, func(t *testing.T) { @@ -1875,6 +1935,66 @@ func TestReconcileDeleteGatewayWithNamespaces(t *testing.T) { }, enableACLs: true, }, + { + name: "ingress-gateway", + initialConsulSvcs: []*api.AgentService{ + { + ID: "ingress-gateway", + Kind: api.ServiceKindIngressGateway, + Service: "ingress-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "gateway", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "ingress-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + Namespace: ts.ConsulNS, + }, + }, + enableACLs: false, + }, + { + name: "ingress-gateway with ACLs enabled", + initialConsulSvcs: []*api.AgentService{ + { + ID: "ingress-gateway", + Kind: api.ServiceKindIngressGateway, + Service: "ingress-gateway", + Port: 80, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "ingress-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 80, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + Namespace: ts.ConsulNS, + }, + }, + enableACLs: true, + }, } for _, tt := range cases { t.Run(fmt.Sprintf("%s:%s", name, tt.name), func(t *testing.T) { diff --git a/control-plane/connect-inject/endpoints_controller_test.go b/control-plane/connect-inject/endpoints_controller_test.go index df28cbcc08..66d2f62464 100644 --- a/control-plane/connect-inject/endpoints_controller_test.go +++ b/control-plane/connect-inject/endpoints_controller_test.go @@ -1088,12 +1088,12 @@ func TestReconcileCreateEndpoint(t *testing.T) { consulSvcName: "mesh-gateway", k8sObjects: func() []runtime.Object { gateway := createGatewayPod("mesh-gateway", "1.2.3.4", map[string]string{ - annotationMeshGatewayConsulServiceName: "mesh-gateway", - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: MeshGateway}) + annotationGatewayConsulServiceName: "mesh-gateway", + annotationGatewayWANSource: "Static", + annotationGatewayWANAddress: "2.3.4.5", + annotationGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: MeshGateway}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "mesh-gateway", @@ -1155,12 +1155,12 @@ func TestReconcileCreateEndpoint(t *testing.T) { consulSvcName: "mesh-gateway", k8sObjects: func() []runtime.Object { gateway := createGatewayPod("mesh-gateway", "1.2.3.4", map[string]string{ - annotationMeshGatewayConsulServiceName: "mesh-gateway", - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "2.3.4.5", - annotationMeshGatewayWANPort: "443", - annotationMeshGatewayContainerPort: "8443", - annotationGatewayKind: MeshGateway}) + annotationGatewayConsulServiceName: "mesh-gateway", + annotationGatewayWANSource: "Static", + annotationGatewayWANAddress: "2.3.4.5", + annotationGatewayWANPort: "443", + annotationMeshGatewayContainerPort: "8443", + annotationGatewayKind: MeshGateway}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "mesh-gateway", @@ -1226,27 +1226,7 @@ func TestReconcileCreateEndpoint(t *testing.T) { svcName: "terminating-gateway", consulSvcName: "terminating-gateway", k8sObjects: func() []runtime.Object { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - Namespace: "default", - Labels: map[string]string{ - keyManagedBy: managedByValue, - }, - Annotations: map[string]string{ - annotationGatewayKind: TerminatingGateway, - }, - }, - Status: corev1.PodStatus{ - PodIP: "1.2.3.4", - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - }, - }, - } + gateway := createGatewayPod("terminating-gateway", "1.2.3.4", map[string]string{annotationGatewayKind: TerminatingGateway}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "terminating-gateway", @@ -1267,7 +1247,7 @@ func TestReconcileCreateEndpoint(t *testing.T) { }, }, } - return []runtime.Object{pod, endpoint} + return []runtime.Object{gateway, endpoint} }, expectedConsulSvcInstances: []*api.CatalogService{ { @@ -1285,7 +1265,6 @@ func TestReconcileCreateEndpoint(t *testing.T) { ServiceProxy: &api.AgentServiceConnectProxyConfig{}, }, }, - expectedProxySvcInstances: []*api.CatalogService{}, expectedHealthChecks: []*api.HealthCheck{ { CheckID: "default/terminating-gateway", @@ -1304,27 +1283,7 @@ func TestReconcileCreateEndpoint(t *testing.T) { svcName: "terminating-gateway", consulSvcName: "terminating-gateway", k8sObjects: func() []runtime.Object { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - Namespace: "default", - Labels: map[string]string{ - keyManagedBy: managedByValue, - }, - Annotations: map[string]string{ - annotationGatewayKind: TerminatingGateway, - }, - }, - Status: corev1.PodStatus{ - PodIP: "1.2.3.4", - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - }, - }, - } + gateway := createGatewayPod("terminating-gateway", "1.2.3.4", map[string]string{annotationGatewayKind: TerminatingGateway}) endpoint := &corev1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Name: "terminating-gateway", @@ -1345,7 +1304,7 @@ func TestReconcileCreateEndpoint(t *testing.T) { }, }, } - return []runtime.Object{pod, endpoint} + return []runtime.Object{gateway, endpoint} }, expectedConsulSvcInstances: []*api.CatalogService{ { @@ -1367,7 +1326,6 @@ func TestReconcileCreateEndpoint(t *testing.T) { }, }, }, - expectedProxySvcInstances: []*api.CatalogService{}, expectedHealthChecks: []*api.HealthCheck{ { CheckID: "default/terminating-gateway", @@ -1380,6 +1338,204 @@ func TestReconcileCreateEndpoint(t *testing.T) { }, }, }, + { + name: "Ingress Gateway", + svcName: "ingress-gateway", + consulSvcName: "ingress-gateway", + k8sObjects: func() []runtime.Object { + gateway := createGatewayPod("ingress-gateway", "1.2.3.4", map[string]string{ + annotationGatewayConsulServiceName: "ingress-gateway", + annotationGatewayKind: IngressGateway, + annotationGatewayWANSource: "Service", + annotationGatewayWANPort: "8443", + }) + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-gateway", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "ingress-gateway", + Namespace: "default", + }, + }, + }, + }, + }, + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-gateway", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + { + IP: "5.6.7.8", + }, + }, + }, + }, + } + return []runtime.Object{gateway, endpoint, svc} + }, + expectedConsulSvcInstances: []*api.CatalogService{ + { + ServiceID: "ingress-gateway", + ServiceName: "ingress-gateway", + ServiceAddress: "1.2.3.4", + ServicePort: 21000, + ServiceMeta: map[string]string{ + MetaKeyPodName: "ingress-gateway", + MetaKeyKubeServiceName: "ingress-gateway", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + }, + ServiceTags: []string{}, + ServiceTaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 21000, + }, + "wan": { + Address: "5.6.7.8", + Port: 8443, + }, + }, + ServiceProxy: &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "envoy_gateway_no_default_bind": true, + "envoy_gateway_bind_addresses": map[string]interface{}{ + "all-interfaces": map[string]interface{}{ + "address": "0.0.0.0", + }, + }, + }, + }, + }, + }, + expectedHealthChecks: []*api.HealthCheck{ + { + CheckID: "default/ingress-gateway", + ServiceName: "ingress-gateway", + ServiceID: "ingress-gateway", + Name: ConsulKubernetesCheckName, + Status: api.HealthPassing, + Output: kubernetesSuccessReasonMsg, + Type: ConsulKubernetesCheckType, + }, + }, + }, + { + name: "Ingress Gateway with Metrics enabled", + metricsEnabled: true, + svcName: "ingress-gateway", + consulSvcName: "ingress-gateway", + k8sObjects: func() []runtime.Object { + gateway := createGatewayPod("ingress-gateway", "1.2.3.4", map[string]string{ + annotationGatewayConsulServiceName: "ingress-gateway", + annotationGatewayKind: IngressGateway, + annotationGatewayWANSource: "Service", + annotationGatewayWANPort: "8443", + }) + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-gateway", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "ingress-gateway", + Namespace: "default", + }, + }, + }, + }, + }, + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-gateway", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + { + IP: "5.6.7.8", + }, + }, + }, + }, + } + return []runtime.Object{gateway, endpoint, svc} + }, + expectedConsulSvcInstances: []*api.CatalogService{ + { + ServiceID: "ingress-gateway", + ServiceName: "ingress-gateway", + ServiceAddress: "1.2.3.4", + ServicePort: 21000, + ServiceMeta: map[string]string{ + MetaKeyPodName: "ingress-gateway", + MetaKeyKubeServiceName: "ingress-gateway", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + }, + ServiceTags: []string{}, + ServiceTaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 21000, + }, + "wan": { + Address: "5.6.7.8", + Port: 8443, + }, + }, + ServiceProxy: &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "envoy_gateway_no_default_bind": true, + "envoy_gateway_bind_addresses": map[string]interface{}{ + "all-interfaces": map[string]interface{}{ + "address": "0.0.0.0", + }, + }, + "envoy_prometheus_bind_addr": "1.2.3.4:20200", + }, + }, + }, + }, + expectedHealthChecks: []*api.HealthCheck{ + { + CheckID: "default/ingress-gateway", + ServiceName: "ingress-gateway", + ServiceID: "ingress-gateway", + Name: ConsulKubernetesCheckName, + Status: api.HealthPassing, + Output: kubernetesSuccessReasonMsg, + Type: ConsulKubernetesCheckType, + }, + }, + }, { name: "Endpoints with multiple addresses", svcName: "service-created", @@ -1869,12 +2025,6 @@ func TestReconcileCreateEndpoint(t *testing.T) { EnableGatewayMetrics: true, } } - if tt.metricsEnabled { - ep.MetricsConfig = MetricsConfig{ - DefaultEnableMetrics: true, - EnableGatewayMetrics: true, - } - } namespacedName := types.NamespacedName{ Namespace: "default", Name: tt.svcName, @@ -3421,6 +3571,67 @@ func TestReconcileDeleteEndpoint(t *testing.T) { }, enableACLs: true, }, + { + name: "Ingress Gateway", + consulSvcName: "service-deleted", + expectServicesToBeDeleted: true, + initialConsulSvcs: []*api.AgentService{ + { + ID: "ingress-gateway", + Kind: api.ServiceKindIngressGateway, + Service: "ingress-gateway", + Port: 21000, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "ingress-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 21000, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + }, + }, + }, + { + name: "When ACLs are enabled, the ingress-gateway token should be deleted", + consulSvcName: "service-deleted", + expectServicesToBeDeleted: true, + initialConsulSvcs: []*api.AgentService{ + { + ID: "ingress-gateway", + Kind: api.ServiceKindIngressGateway, + Service: "ingress-gateway", + Port: 21000, + Address: "1.2.3.4", + Meta: map[string]string{ + MetaKeyKubeServiceName: "service-deleted", + MetaKeyKubeNS: "default", + MetaKeyManagedBy: managedByValue, + MetaKeyPodName: "ingress-gateway", + }, + TaggedAddresses: map[string]api.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 21000, + }, + "wan": { + Address: "5.6.7.8", + Port: 8080, + }, + }, + }, + }, + enableACLs: true, + }, { name: "Terminating Gateway", consulSvcName: "service-deleted", @@ -5341,9 +5552,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "NodeName", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "NodeName", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5389,9 +5600,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "NodeIP", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "NodeIP", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5437,9 +5648,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Static", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Static", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5485,9 +5696,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5533,9 +5744,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5581,9 +5792,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5629,9 +5840,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5677,8 +5888,8 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5718,16 +5929,16 @@ func Test_GetWANData(t *testing.T) { }, wanAddr: "test-loadbalancer-hostname", wanPort: 1234, - expErr: "failed to read annotation consul.hashicorp.com/mesh-gateway-wan-address-source", + expErr: "failed to read annotation consul.hashicorp.com/gateway-wan-address-source", }, "no Service with Source=Service": { gatewayPod: corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5753,9 +5964,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "not-a-valid-port", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "not-a-valid-port", }, }, Spec: corev1.PodSpec{ @@ -5802,9 +6013,9 @@ func Test_GetWANData(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "gateway", Annotations: map[string]string{ - annotationMeshGatewaySource: "Service", - annotationMeshGatewayWANAddress: "test-wan-address", - annotationMeshGatewayWANPort: "1234", + annotationGatewayWANSource: "Service", + annotationGatewayWANAddress: "test-wan-address", + annotationGatewayWANPort: "1234", }, }, Spec: corev1.PodSpec{ @@ -5862,6 +6073,91 @@ func Test_GetWANData(t *testing.T) { } } +func TestCreateTerminatingGatewayRegistrations(t *testing.T) { + const ( + podName = "terminating-gateway" + serviceName = "terminating-gateway" + serviceNamespace = "consul" + ) + + cases := map[string]struct { + annotations map[string]string + service *api.AgentService + metricsConfig MetricsConfig + }{ + "Terminating Gateway Default": { + annotations: map[string]string{ + annotationGatewayKind: TerminatingGateway, + }, + }, + "Terminating Gateway with Consul Namespace": { + annotations: map[string]string{ + annotationGatewayKind: TerminatingGateway, + annotationGatewayNamespace: "consul-namespace", + }, + }, + "Terminating Gateway with Metrics Enabled": { + annotations: map[string]string{ + annotationGatewayKind: TerminatingGateway, + }, + metricsConfig: MetricsConfig{ + DefaultEnableMetrics: true, + EnableGatewayMetrics: true, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "consul", + Name: podName, + Annotations: c.annotations, + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + }, + } + serviceEndpoint := corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: serviceNamespace, + Name: serviceName, + }, + } + + r := EndpointsController{ + MetricsConfig: c.metricsConfig, + } + + registration, err := r.createGatewayRegistrations(pod, serviceEndpoint, api.HealthPassing) + require.NoError(t, err) + + require.Equal(t, api.ServiceKindTerminatingGateway, registration.Service.Kind) + require.Equal(t, "terminating-gateway", registration.Service.Service) + require.Equal(t, 8443, registration.Service.Port) + + require.Equal(t, podName, registration.Service.ID) + require.Equal(t, "1.2.3.4", registration.Service.Address) + require.Equal(t, podName, registration.Service.Meta[MetaKeyPodName]) + require.Equal(t, serviceName, registration.Service.Meta[MetaKeyKubeServiceName]) + require.Equal(t, serviceNamespace, registration.Service.Meta[MetaKeyKubeNS]) + + require.Equal(t, fmt.Sprintf("%s/%s", "consul", podName), registration.Check.CheckID) + require.Equal(t, api.HealthPassing, registration.Check.Status) + require.Equal(t, podName, registration.Check.ServiceID) + + if value, ok := c.annotations[annotationConsulNamespace]; ok { + require.Equal(t, value, registration.Service.Namespace) + } + + if c.metricsConfig.DefaultEnableMetrics && c.metricsConfig.EnableGatewayMetrics { + require.Equal(t, "1.2.3.4:20200", registration.Service.Proxy.Config["envoy_prometheus_bind_addr"]) + } + }) + } +} + func createServicePod(name, ip string, inject bool, managedByEndpointsController bool) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/control-plane/subcommand/connect-init/command.go b/control-plane/subcommand/connect-init/command.go index 65d5ad3c12..7954669169 100644 --- a/control-plane/subcommand/connect-init/command.go +++ b/control-plane/subcommand/connect-init/command.go @@ -14,6 +14,7 @@ import ( "github.com/cenkalti/backoff" connectinject "github.com/hashicorp/consul-k8s/control-plane/connect-inject" "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" "github.com/hashicorp/consul-server-connection-manager/discovery" @@ -115,7 +116,7 @@ func (c *Command) Run(args []string) int { // Start Consul server Connection manager. serverConnMgrCfg, err := c.consul.ConsulServerConnMgrConfig() - // Disable server watch because we we only need to get server IPs once. + // Disable server watch because we only need to get server IPs once. serverConnMgrCfg.ServerWatchDisabled = true if err != nil { c.UI.Error(fmt.Sprintf("unable to create config for consul-server-connection-manager: %s", err)) @@ -261,10 +262,15 @@ func (c *Command) getGatewayRegistration(client *api.Client) backoff.Operation { registrationRetryCount := 0 return func() error { registrationRetryCount++ + var gatewayList *api.CatalogNodeServiceList + var err error filter := fmt.Sprintf("Meta[%q] == %q and Meta[%q] == %q ", connectinject.MetaKeyPodName, c.flagPodName, connectinject.MetaKeyKubeNS, c.flagPodNamespace) - - gatewayList, _, err := client.Catalog().NodeServiceList(c.flagConsulNodeName, &api.QueryOptions{Filter: filter}) + if c.consul.Namespace != "" { + gatewayList, _, err = client.Catalog().NodeServiceList(c.flagConsulNodeName, &api.QueryOptions{Filter: filter, Namespace: namespaces.WildcardNamespace}) + } else { + gatewayList, _, err = client.Catalog().NodeServiceList(c.flagConsulNodeName, &api.QueryOptions{Filter: filter}) + } if err != nil { c.logger.Error("Unable to get gateway", "error", err) return err diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index 54ee6937ec..7ce6f6654a 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -388,6 +388,14 @@ func (c *Command) Run(args []string) int { go watcher.Run() defer watcher.Stop() + // This is a blocking command that is run in order to ensure we only start the + // connect-inject controllers only after we have access to the Consul server. + _, err = watcher.State() + if err != nil { + c.UI.Error(fmt.Sprintf("unable to start Consul server watcher: %s", err)) + return 1 + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, LeaderElection: true,