From 5b4aba9cd2ae6c822e63a4039e1b2c57b0cbd682 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 13 Dec 2019 14:46:40 +0100 Subject: [PATCH 1/5] Mount annotations in the Pod --- pkg/controller/common/volume/downward_api.go | 46 +++++++++++++++++++ .../elasticsearch/nodespec/readiness_probe.go | 37 ++++++++++++--- .../elasticsearch/nodespec/volumes.go | 4 ++ pkg/controller/elasticsearch/volume/names.go | 4 ++ 4 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 pkg/controller/common/volume/downward_api.go diff --git a/pkg/controller/common/volume/downward_api.go b/pkg/controller/common/volume/downward_api.go new file mode 100644 index 0000000000..f976f266cf --- /dev/null +++ b/pkg/controller/common/volume/downward_api.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package volume + +import ( + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" + corev1 "k8s.io/api/core/v1" +) + +var downwardApiVolume = corev1.Volume{ + Name: volume.DownwardApiVolumeName, + VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{ + Items: []corev1.DownwardAPIVolumeFile{ + { + Path: volume.LabelsFile, + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels", + }, + }, + }, + }, + }, +} + +var downwardApiVolumeMount = corev1.VolumeMount{ + Name: volume.DownwardApiVolumeName, + MountPath: volume.DownwardApiMountPath, + ReadOnly: true, +} + +type DownwardApi struct{} + +func (DownwardApi) Name() string { + return volume.DownwardApiVolumeName +} + +func (DownwardApi) Volume() corev1.Volume { + return downwardApiVolume +} + +func (DownwardApi) VolumeMount() corev1.VolumeMount { + return downwardApiVolumeMount +} diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index b9c72e87c4..68a83ec04b 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -28,7 +28,25 @@ func NewReadinessProbe() *corev1.Probe { const ReadinessProbeScriptConfigKey = "readiness-probe-script.sh" const ReadinessProbeScript = `#!/usr/bin/env bash -# Consider a node to be healthy if it responds to a simple GET on "/_cat/nodes?local" + +# fail should be called as a last resort to help the user to understand why the probe failed +function fail { + timestamp=$(date --iso-8601=seconds) + echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/1 + exit 1 +} + +labels="` + volume.DownwardApiMountPath + "/" + volume.LabelsFile + `" + +if [[ ! -f "${labels}" ]]; then + fail "\"reason\": \"${labels} does not exist\"" +fi + +# get Elasticsearch version from the downward API +version=$(grep "elasticsearch.k8s.elastic.co/version" ${labels} | cut -d '=' -f 2) +# remove quotes +version=$(echo "${version}" | tr -d '"') + CURL_TIMEOUT=3 # Check if PROBE_PASSWORD_PATH is set, otherwise fall back to its former name in 1.0.0.beta-1: PROBE_PASSWORD_FILE @@ -46,14 +64,19 @@ else BASIC_AUTH='' fi -# request Elasticsearch -ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/_cat/nodes?local" +# request Elasticsearch on / +ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/" status=$(curl -o /dev/null -w "%{http_code}" --max-time $CURL_TIMEOUT -XGET -s -k ${BASIC_AUTH} $ENDPOINT) +curl_rc=$? + +if [[ ${curl_rc} -ne 0 ]]; then + fail "\"curl_rc\": \"${curl_rc}\"" +fi -# ready if status code 200 -if [[ $status == "200" ]]; then - exit 0 +# ready if status code 200, 503 is tolerable if ES version is 6.x +if [[ ${status} == "200" ]] || [[ ${status} == "503" && ${version:0:2} == "6." ]]; then + exit 0 else - exit 1 + fail " \"status\": \"${status}\", \"version\":\"${version}\" " fi ` diff --git a/pkg/controller/elasticsearch/nodespec/volumes.go b/pkg/controller/elasticsearch/nodespec/volumes.go index 24c26f8b3f..299fb4ddcc 100644 --- a/pkg/controller/elasticsearch/nodespec/volumes.go +++ b/pkg/controller/elasticsearch/nodespec/volumes.go @@ -17,6 +17,8 @@ import ( esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" ) +var downwardApiVolume = volume.DownwardApi{} + func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keystore.Resources) ([]corev1.Volume, []corev1.VolumeMount) { configVolume := settings.ConfigSecretVolume(esv1.StatefulSet(esName, nodeSpec.Name)) @@ -70,6 +72,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.Volume(), scriptsVolume.Volume(), configVolume.Volume(), + downwardApiVolume.Volume(), )...) if keystoreResources != nil { volumes = append(volumes, keystoreResources.Volume) @@ -86,6 +89,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.VolumeMount(), scriptsVolume.VolumeMount(), configVolume.VolumeMount(), + downwardApiVolume.VolumeMount(), ) return volumes, volumeMounts diff --git a/pkg/controller/elasticsearch/volume/names.go b/pkg/controller/elasticsearch/volume/names.go index b242689564..cc3e97832f 100644 --- a/pkg/controller/elasticsearch/volume/names.go +++ b/pkg/controller/elasticsearch/volume/names.go @@ -35,4 +35,8 @@ const ( ScriptsVolumeName = "elastic-internal-scripts" ScriptsVolumeMountPath = "/mnt/elastic-internal/scripts" + + DownwardApiVolumeName = "downward-api" + DownwardApiMountPath = "/mnt/elastic-internal/downward-api" + LabelsFile = "labels" ) From 2daebf50dbd0d22c4fe647461e2039ae6fa3eccf Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 16 Dec 2019 13:26:49 +0100 Subject: [PATCH 2/5] Api -> API --- pkg/controller/common/volume/downward_api.go | 24 +++++++++---------- .../elasticsearch/nodespec/readiness_probe.go | 2 +- .../elasticsearch/nodespec/volumes.go | 9 ++++--- pkg/controller/elasticsearch/volume/names.go | 4 ++-- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/pkg/controller/common/volume/downward_api.go b/pkg/controller/common/volume/downward_api.go index f976f266cf..b11f53bf5f 100644 --- a/pkg/controller/common/volume/downward_api.go +++ b/pkg/controller/common/volume/downward_api.go @@ -9,8 +9,8 @@ import ( corev1 "k8s.io/api/core/v1" ) -var downwardApiVolume = corev1.Volume{ - Name: volume.DownwardApiVolumeName, +var downwardAPIVolume = corev1.Volume{ + Name: volume.DownwardAPIVolumeName, VolumeSource: corev1.VolumeSource{ DownwardAPI: &corev1.DownwardAPIVolumeSource{ Items: []corev1.DownwardAPIVolumeFile{ @@ -25,22 +25,22 @@ var downwardApiVolume = corev1.Volume{ }, } -var downwardApiVolumeMount = corev1.VolumeMount{ - Name: volume.DownwardApiVolumeName, - MountPath: volume.DownwardApiMountPath, +var downwardAPIVolumeMount = corev1.VolumeMount{ + Name: volume.DownwardAPIVolumeName, + MountPath: volume.DownwardAPIMountPath, ReadOnly: true, } -type DownwardApi struct{} +type DownwardAPI struct{} -func (DownwardApi) Name() string { - return volume.DownwardApiVolumeName +func (DownwardAPI) Name() string { + return volume.DownwardAPIVolumeName } -func (DownwardApi) Volume() corev1.Volume { - return downwardApiVolume +func (DownwardAPI) Volume() corev1.Volume { + return downwardAPIVolume } -func (DownwardApi) VolumeMount() corev1.VolumeMount { - return downwardApiVolumeMount +func (DownwardAPI) VolumeMount() corev1.VolumeMount { + return downwardAPIVolumeMount } diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index 2abb388d04..56cc45d61c 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -36,7 +36,7 @@ function fail { exit 1 } -labels="` + volume.DownwardApiMountPath + "/" + volume.LabelsFile + `" +labels="` + volume.DownwardAPIMountPath + "/" + volume.LabelsFile + `" if [[ ! -f "${labels}" ]]; then fail "\"reason\": \"${labels} does not exist\"" diff --git a/pkg/controller/elasticsearch/nodespec/volumes.go b/pkg/controller/elasticsearch/nodespec/volumes.go index 299fb4ddcc..535344269b 100644 --- a/pkg/controller/elasticsearch/nodespec/volumes.go +++ b/pkg/controller/elasticsearch/nodespec/volumes.go @@ -5,8 +5,6 @@ package nodespec import ( - corev1 "k8s.io/api/core/v1" - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/common/keystore" @@ -15,9 +13,10 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user" esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" + corev1 "k8s.io/api/core/v1" ) -var downwardApiVolume = volume.DownwardApi{} +var downwardAPIVolume = volume.DownwardAPI{} func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keystore.Resources) ([]corev1.Volume, []corev1.VolumeMount) { @@ -72,7 +71,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.Volume(), scriptsVolume.Volume(), configVolume.Volume(), - downwardApiVolume.Volume(), + downwardAPIVolume.Volume(), )...) if keystoreResources != nil { volumes = append(volumes, keystoreResources.Volume) @@ -89,7 +88,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.VolumeMount(), scriptsVolume.VolumeMount(), configVolume.VolumeMount(), - downwardApiVolume.VolumeMount(), + downwardAPIVolume.VolumeMount(), ) return volumes, volumeMounts diff --git a/pkg/controller/elasticsearch/volume/names.go b/pkg/controller/elasticsearch/volume/names.go index cc3e97832f..1e193f41f7 100644 --- a/pkg/controller/elasticsearch/volume/names.go +++ b/pkg/controller/elasticsearch/volume/names.go @@ -36,7 +36,7 @@ const ( ScriptsVolumeName = "elastic-internal-scripts" ScriptsVolumeMountPath = "/mnt/elastic-internal/scripts" - DownwardApiVolumeName = "downward-api" - DownwardApiMountPath = "/mnt/elastic-internal/downward-api" + DownwardAPIVolumeName = "downward-api" + DownwardAPIMountPath = "/mnt/elastic-internal/downward-api" LabelsFile = "labels" ) From a11d11e3577583f31e61813288b599e5de4140fd Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 16 Dec 2019 13:37:27 +0100 Subject: [PATCH 3/5] make it explicit that DownwardAPI implements VolumeLike --- pkg/controller/common/volume/downward_api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/controller/common/volume/downward_api.go b/pkg/controller/common/volume/downward_api.go index b11f53bf5f..a0f3dde373 100644 --- a/pkg/controller/common/volume/downward_api.go +++ b/pkg/controller/common/volume/downward_api.go @@ -33,6 +33,8 @@ var downwardAPIVolumeMount = corev1.VolumeMount{ type DownwardAPI struct{} +var _ VolumeLike = DownwardAPI{} + func (DownwardAPI) Name() string { return volume.DownwardAPIVolumeName } From 654a03d5c6ee720def5872b1ffcb5843dc93870f Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 16 Dec 2019 15:57:47 +0100 Subject: [PATCH 4/5] Redirect errors to stderr --- pkg/controller/elasticsearch/nodespec/readiness_probe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index 56cc45d61c..10f56868b2 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -32,7 +32,7 @@ const ReadinessProbeScript = `#!/usr/bin/env bash # fail should be called as a last resort to help the user to understand why the probe failed function fail { timestamp=$(date --iso-8601=seconds) - echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/1 + echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/2 exit 1 } From 229eda11c16e0725a62110adfb1474041bb248ae Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 16 Dec 2019 17:13:15 +0100 Subject: [PATCH 5/5] Reuse label.VersionLabelName --- pkg/controller/elasticsearch/nodespec/readiness_probe.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index 10f56868b2..ca1b80ac87 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -7,6 +7,7 @@ package nodespec import ( "path" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" corev1 "k8s.io/api/core/v1" ) @@ -43,7 +44,7 @@ if [[ ! -f "${labels}" ]]; then fi # get Elasticsearch version from the downward API -version=$(grep "elasticsearch.k8s.elastic.co/version" ${labels} | cut -d '=' -f 2) +version=$(grep "` + label.VersionLabelName + `" ${labels} | cut -d '=' -f 2) # remove quotes version=$(echo "${version}" | tr -d '"')