Skip to content

Commit

Permalink
agentless: allow providing custom http port for connect-inject-init c…
Browse files Browse the repository at this point in the history 
…ontainer (#1453)
  • Loading branch information
@ishustava @thisisnotashwin
ishustava authored and thisisnotashwin committed Sep 19, 2022
1 parent 197dca0 commit 2da7277
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 40 deletions.
24 changes: 19 additions & 5 deletions control-plane/connect-inject/container_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,15 @@ type initContainerCommandData struct {
// ConsulAPITimeout is the duration that the consul API client will
// wait for a response from the API before cancelling the request.
ConsulAPITimeout time.Duration

// TLSEnabled indicates whether we should use TLS for communicating to Consul.
TLSEnabled bool

// ConsulHTTPPort is the HTTP or HTTPs port we should use to talk to Consul.
ConsulHTTPPort string

// ConsulGRPCPort is the gRPC port we should use to talk to Consul.
ConsulGRPCPort string
}

// initCopyContainer returns the init container spec for the copy container which places
Expand Down Expand Up @@ -172,6 +181,9 @@ func (w *MeshWebhook) containerInit(namespace corev1.Namespace, pod corev1.Pod,
ConsulNamespace: w.consulNamespace(namespace.Name),
NamespaceMirroringEnabled: w.EnableK8SNSMirroring,
ConsulCACert: w.ConsulCACert,
TLSEnabled: w.TLSEnabled,
ConsulHTTPPort: w.ConsulHTTPPort,
ConsulGRPCPort: w.ConsulGRPCPort,
ConsulAddress: w.ConsulAddress,
ConsulNodeName: ConsulNodeName,
EnableTransparentProxy: tproxyEnabled,
Expand Down Expand Up @@ -394,16 +406,18 @@ func splitCommaSeparatedItemsFromAnnotation(annotation string, pod corev1.Pod) [
// initContainerCommandTpl is the template for the command executed by
// the init container.
const initContainerCommandTpl = `
{{- if .ConsulCACert}}
export CONSUL_HTTP_ADDR="https://{{ .ConsulAddress }}:8501"
export CONSUL_GRPC_ADDR="https://{{ .ConsulAddress }}:8502"
{{- if .TLSEnabled }}
export CONSUL_HTTP_ADDR="https://{{ .ConsulAddress }}:{{ .ConsulHTTPPort }}"
export CONSUL_GRPC_ADDR="https://{{ .ConsulAddress }}:{{ .ConsulGRPCPort }}"
{{- if .ConsulCACert }}
export CONSUL_CACERT=/consul/connect-inject/consul-ca.pem
cat <<EOF >/consul/connect-inject/consul-ca.pem
{{ .ConsulCACert }}
EOF
{{- end }}
{{- else}}
export CONSUL_HTTP_ADDR="{{ .ConsulAddress }}:8500"
export CONSUL_GRPC_ADDR="{{ .ConsulAddress }}:8502"
export CONSUL_HTTP_ADDR="{{ .ConsulAddress }}:{{ .ConsulHTTPPort }}"
export CONSUL_GRPC_ADDR="{{ .ConsulAddress }}:{{ .ConsulGRPCPort }}"
{{- end}}
consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \
-consul-api-timeout={{ .ConsulAPITimeout }} \
Expand Down
105 changes: 74 additions & 31 deletions control-plane/connect-inject/container_init_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@ func TestHandlerContainerInit(t *testing.T) {
return pod
},
MeshWebhook{
ConsulAddress: "10.0.0.0",
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -96,6 +98,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
AuthMethod: "an-auth-method",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -615,6 +619,8 @@ func TestHandlerContainerInit_namespacesAndPartitionsEnabled(t *testing.T) {
ConsulPartition: "",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -643,6 +649,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
ConsulPartition: "default",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -673,6 +681,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
ConsulPartition: "",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -701,6 +711,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
ConsulPartition: "non-default-part",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -732,6 +744,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
ConsulPartition: "default",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -770,6 +784,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
ConsulPartition: "non-default",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -807,6 +823,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
EnableTransparentProxy: true,
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -842,6 +860,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
EnableTransparentProxy: true,
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -883,6 +903,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
EnableTransparentProxy: true,
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
`/bin/sh -ec
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
Expand Down Expand Up @@ -987,6 +1009,8 @@ func TestHandlerContainerInit_Multiport(t *testing.T) {
MeshWebhook{
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
2,
[]multiPortInfo{
Expand Down Expand Up @@ -1043,6 +1067,8 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
AuthMethod: "auth-method",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
ConsulHTTPPort: "8500",
ConsulGRPCPort: "8502",
},
2,
[]multiPortInfo{
Expand Down Expand Up @@ -1161,43 +1187,60 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
-bootstrap > /consul/connect-inject/envoy-bootstrap.yaml`)
}

// If Consul CA cert is set,
// If TLSEnabled is set,
// Consul addresses should use HTTPS
// and CA cert should be set as env variable.
func TestHandlerContainerInit_WithTLS(t *testing.T) {
w := MeshWebhook{
ConsulCACert: "consul-ca-cert",
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
annotationService: "foo",
},
},
// and CA cert should be set as env variable if provided.
// Additionally, test that the init container is correctly configured
// when http or gRPC ports are different from defaults.
func TestHandlerContainerInit_WithTLSAndCustomPorts(t *testing.T) {
for _, caProvided := range []bool{true, false} {
name := fmt.Sprintf("ca provided: %t", caProvided)
t.Run(name, func(t *testing.T) {
w := MeshWebhook{
ConsulAPITimeout: 5 * time.Second,
ConsulAddress: "10.0.0.0",
TLSEnabled: true,
ConsulHTTPPort: "443",
ConsulGRPCPort: "8503",
}
if caProvided {
w.ConsulCACert = "consul-ca-cert"
}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
annotationService: "foo",
},
},

Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "web",
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "web",
},
},
},
},
},
}
container, err := w.containerInit(testNS, *pod, multiPortInfo{})
require.NoError(t, err)
actual := strings.Join(container.Command, " ")
require.Contains(t, actual, `
export CONSUL_HTTP_ADDR="https://10.0.0.0:8501"
export CONSUL_GRPC_ADDR="https://10.0.0.0:8502"
}
container, err := w.containerInit(testNS, *pod, multiPortInfo{})
require.NoError(t, err)
actual := strings.Join(container.Command, " ")
if caProvided {
require.Contains(t, actual, `
export CONSUL_HTTP_ADDR="https://10.0.0.0:443"
export CONSUL_GRPC_ADDR="https://10.0.0.0:8503"
export CONSUL_CACERT=/consul/connect-inject/consul-ca.pem
cat <<EOF >/consul/connect-inject/consul-ca.pem
consul-ca-cert
EOF`)
require.NotContains(t, actual, `
export CONSUL_HTTP_ADDR="10.0.0.0:8500"
export CONSUL_GRPC_ADDR="10.0.0.0:8502"`)
} else {
require.Contains(t, actual, `
export CONSUL_HTTP_ADDR="https://10.0.0.0:443"
export CONSUL_GRPC_ADDR="https://10.0.0.0:8503"
`)
}

})
}
}

func TestHandlerContainerInit_Resources(t *testing.T) {
Expand Down
9 changes: 9 additions & 0 deletions control-plane/connect-inject/mesh_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,15 @@ type MeshWebhook struct {
// If not set, will use HTTP.
ConsulCACert string

// TLSEnabled indicates whether we should use TLS for communicating to Consul.
TLSEnabled bool

// ConsulHTTPPort is the HTTP or HTTPs port we should use to talk to Consul.
ConsulHTTPPort string

// ConsulGRPCPort is the gRPC port we should use to talk to Consul.
ConsulGRPCPort string

// ConsulAddress is the address of the Consul server. This should be only the
// host (i.e. not including port or protocol).
ConsulAddress string
Expand Down
11 changes: 7 additions & 4 deletions control-plane/subcommand/inject-connect/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,14 +382,14 @@ func (c *Command) Run(args []string) int {
cfg.Address = serverAddr
}

consulURLRaw := cfg.Address
// cfg.Address may or may not be prefixed with scheme.
if !strings.Contains(cfg.Address, "://") {
consulURLRaw = fmt.Sprintf("%s://%s", cfg.Scheme, cfg.Address)
cfg.Address = fmt.Sprintf("%s://%s", cfg.Scheme, cfg.Address)
}
consulURL, err := url.Parse(consulURLRaw)

consulURL, err := url.Parse(cfg.Address)
if err != nil {
c.UI.Error(fmt.Sprintf("error parsing consul address %q: %s", consulURLRaw, err))
c.UI.Error(fmt.Sprintf("error parsing consul address %q: %s", cfg.Address, err))
return 1
}

Expand Down Expand Up @@ -537,6 +537,9 @@ func (c *Command) Run(args []string) int {
RequireAnnotation: !c.flagDefaultInject,
AuthMethod: c.flagACLAuthMethod,
ConsulCACert: string(consulCACert),
TLSEnabled: consulURL.Scheme == "https",
ConsulHTTPPort: consulURL.Port(),
ConsulGRPCPort: "8502", // todo(ishustava): should be passed via flag
ConsulAddress: consulURL.Hostname(),
DefaultProxyCPURequest: sidecarProxyCPURequest,
DefaultProxyCPULimit: sidecarProxyCPULimit,
Expand Down

0 comments on commit 2da7277

Please sign in to comment.