From 3a246eb90d8ac8c035377522e65c71517083993c Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:04:19 +0200 Subject: [PATCH 01/92] first commit --- delivery.yaml | 6 ++++++ e2e/kind-config-multikind.yaml | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 e2e/kind-config-multikind.yaml diff --git a/delivery.yaml b/delivery.yaml index 1866486f8..2b85d78bf 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -41,6 +41,12 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator go test ./... + - desc: 'Run e2e tests' + cmd: | + export PATH=$PATH:$HOME/go/bin + go get -u sigs.k8s.io/kind + cd $OPERATOR_TOP_DIR/postgres-operator + kind create cluster --name kind-m --config ./kind-config-multikind.yaml --loglevel debug - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/e2e/kind-config-multikind.yaml b/e2e/kind-config-multikind.yaml new file mode 100644 index 000000000..d95d68b2e --- /dev/null +++ b/e2e/kind-config-multikind.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +nodes: +- role: control-plane +- role: worker +- role: worker +- role: worker From bbaf4cfde7c80eff0e179cef5ecf7d601881cd53 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:05:24 +0200 Subject: [PATCH 02/92] fix path --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index 2b85d78bf..286d89813 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,7 +46,7 @@ pipeline: export PATH=$PATH:$HOME/go/bin go get -u sigs.k8s.io/kind cd $OPERATOR_TOP_DIR/postgres-operator - kind create cluster --name kind-m --config ./kind-config-multikind.yaml --loglevel debug + kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin From 5fc371d95a97e4b48e77bced087920f7a0edd129 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:13:39 +0200 Subject: [PATCH 03/92] make go get verbose --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 286d89813..9dcd67ef2 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -43,8 +43,8 @@ pipeline: go test ./... - desc: 'Run e2e tests' cmd: | - export PATH=$PATH:$HOME/go/bin - go get -u sigs.k8s.io/kind + export PATH=$PATH:$HOME/go/bin + go get -v -u sigs.k8s.io/kind cd $OPERATOR_TOP_DIR/postgres-operator kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug - desc: 'Push docker image' From 699b276ad6f4dc7faaf2a9386d719839203d398b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:27:01 +0200 Subject: [PATCH 04/92] adjust makefile && test client connection to kind --- Makefile | 5 +++++ delivery.yaml | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5b27281c2..20e103a43 100644 --- a/Makefile +++ b/Makefile @@ -91,3 +91,8 @@ deps: test: hack/verify-codegen.sh @go test ./... + +e2e: + kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug + export KUBECONFIG="$(kind get kubeconfig-path --name="kind-m")" + kubectl cluster-info diff --git a/delivery.yaml b/delivery.yaml index 9dcd67ef2..31d3006bf 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -44,9 +44,13 @@ pipeline: - desc: 'Run e2e tests' cmd: | export PATH=$PATH:$HOME/go/bin - go get -v -u sigs.k8s.io/kind cd $OPERATOR_TOP_DIR/postgres-operator - kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug + go get -u -v sigs.k8s.io/kind + echo "INFO install kubectl to test 'kind' from client side" + (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && + chmod +x ./kubectl && + mv ./kubectl /usr/local/bin/kubectl) || exit 1 + make e2e - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin From 61bf8357534d5c051548e53019b5928c7ebcdd1e Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:40:28 +0200 Subject: [PATCH 05/92] add mock e2e script --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 20e103a43..ce59bfe81 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean local test linux macos docker push scm-source.json +.PHONY: clean local test linux macos docker push scm-source.json e2e BINARY ?= postgres-operator BUILD_FLAGS ?= -v @@ -93,6 +93,4 @@ test: @go test ./... e2e: - kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug - export KUBECONFIG="$(kind get kubeconfig-path --name="kind-m")" - kubectl cluster-info + e2e/mock_e2e.sh From 264833cd6a3f1bbfa670a2ffbbbb5f19ec606b31 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:44:29 +0200 Subject: [PATCH 06/92] push the script --- delivery.yaml | 2 +- e2e/mock_e2e.sh | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 e2e/mock_e2e.sh diff --git a/delivery.yaml b/delivery.yaml index 31d3006bf..1ae6aeb38 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -45,7 +45,7 @@ pipeline: cmd: | export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator - go get -u -v sigs.k8s.io/kind + go get -u sigs.k8s.io/kind echo "INFO install kubectl to test 'kind' from client side" (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x ./kubectl && diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh new file mode 100755 index 000000000..48ac5efde --- /dev/null +++ b/e2e/mock_e2e.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# enable unofficial bash strict mode +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + +kind create cluster --name kind-m2 --config ./e2e/kind-config-multikind.yaml --loglevel debug +export KUBECONFIG="$(kind get kubeconfig-path --name="kind-m")" +kubectl cluster-info From e320fbcf12419e5814b74bf040c35a7541d67242 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 24 Apr 2019 17:47:28 +0200 Subject: [PATCH 07/92] fix cluster name --- e2e/mock_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh index 48ac5efde..51afba223 100755 --- a/e2e/mock_e2e.sh +++ b/e2e/mock_e2e.sh @@ -6,6 +6,6 @@ set -o nounset set -o pipefail IFS=$'\n\t' -kind create cluster --name kind-m2 --config ./e2e/kind-config-multikind.yaml --loglevel debug +kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug export KUBECONFIG="$(kind get kubeconfig-path --name="kind-m")" kubectl cluster-info From 7049c260be7dfc9d0f0c2b24fb452ec3da195e50 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 14:23:52 +0200 Subject: [PATCH 08/92] delete existing kind cluster --- e2e/mock_e2e.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh index 51afba223..f092b259b 100755 --- a/e2e/mock_e2e.sh +++ b/e2e/mock_e2e.sh @@ -6,6 +6,16 @@ set -o nounset set -o pipefail IFS=$'\n\t' -kind create cluster --name kind-m --config ./e2e/kind-config-multikind.yaml --loglevel debug -export KUBECONFIG="$(kind get kubeconfig-path --name="kind-m")" +readonly cluster_name="kind-test-postgres-operator" + +# avoid interference with previous test runs +if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] +then + rm "$KUBECONFIG" + unset KUBECONFIG + kind delete cluster --name ${cluster_name} +fi + +kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind.yaml +export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info From 86a17e35e455ec6aefc3871817d5cde7a9d1638b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 14:48:51 +0200 Subject: [PATCH 09/92] add tests skeleton --- e2e/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 e2e/tests.py diff --git a/e2e/tests.py b/e2e/tests.py new file mode 100644 index 000000000..992e435db --- /dev/null +++ b/e2e/tests.py @@ -0,0 +1,21 @@ +from kubernetes import client, config + +def main(): + + config.load_kube_config() + v1 = client.CoreV1Api() + + body = { + "metadata": { + "labels": { + "lifecycle-status": "ready", + } + } + } + + nodes = ["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"] + for node in nodes: + _ = v1.patch_node(node, body) + +if __name__ == '__main__': + main() \ No newline at end of file From 2bfcb0ed23ccba7bf2027561955c28c25fdea9e3 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 14:59:59 +0200 Subject: [PATCH 10/92] run tests from the script --- e2e/mock_e2e.sh | 2 ++ e2e/tests.py | 2 ++ 2 files changed, 4 insertions(+) mode change 100644 => 100755 e2e/tests.py diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh index f092b259b..a88ab7e06 100755 --- a/e2e/mock_e2e.sh +++ b/e2e/mock_e2e.sh @@ -19,3 +19,5 @@ fi kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind.yaml export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info + +./e2e/tests.py \ No newline at end of file diff --git a/e2e/tests.py b/e2e/tests.py old mode 100644 new mode 100755 index 992e435db..8b971dda0 --- a/e2e/tests.py +++ b/e2e/tests.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from kubernetes import client, config def main(): From f62d740c270d452b729d32e5bae12cf5a1397ca0 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 17:17:36 +0200 Subject: [PATCH 11/92] set up simple unit test --- e2e/tests.py | 57 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/e2e/tests.py b/e2e/tests.py index 8b971dda0..aec41dcac 100755 --- a/e2e/tests.py +++ b/e2e/tests.py @@ -1,23 +1,54 @@ #!/usr/bin/env python3 +import unittest from kubernetes import client, config +from pprint import pprint -def main(): +class SampleTestCase(unittest.TestCase): - config.load_kube_config() - v1 = client.CoreV1Api() + nodes = set(["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"]) - body = { - "metadata": { - "labels": { - "lifecycle-status": "ready", - } + def setUp(self): + self.config = config.load_kube_config() + self.v1 = client.CoreV1Api() + + def test_assign_labels_to_nodes(self): + """ + Ensure labeling nodes through the externally connected Python client works. + Sample test case to illustrate potential test structure + """ + body = { + "metadata": { + "labels": { + "lifecycle-status": "ready" + } + } } - } + for node in self.nodes: + _ = self.v1.patch_node(node, body) + + labelled_nodes = set([]) + for node in self.nodes: + v1_node_var = self.v1.read_node(node) + if v1_node_var.metadata.labels['lifecycle-status'] == 'ready': + labelled_nodes.add(v1_node_var.metadata.name) - nodes = ["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"] - for node in nodes: - _ = v1.patch_node(node, body) + assert self.nodes == labelled_nodes, "nodes incorrectly labelled" + + def tearDown(self): + """ + Each test must restore the original cluster state + to avoid introducing dependencies between tests + """ + body = { + "metadata": { + "labels": { + "lifecycle-status": None # deletes label + } + } + } + for node in self.nodes: + _ = self.v1.patch_node(node, body) if __name__ == '__main__': - main() \ No newline at end of file + unittest.main() \ No newline at end of file From 215d6ba93676da0b49134c9d1b9ba5d16e4f4e76 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 17:19:40 +0200 Subject: [PATCH 12/92] add requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..807e21be4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +kubernetes From dda9466109d2ed0560205a6fe0183a83153d7bee Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 17:29:59 +0200 Subject: [PATCH 13/92] install requirements during build --- delivery.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/delivery.yaml b/delivery.yaml index 1ae6aeb38..8c28cf7f3 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,6 +46,7 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator go get -u sigs.k8s.io/kind + pip3 install -r requirements.txt echo "INFO install kubectl to test 'kind' from client side" (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x ./kubectl && From 744328917ace1c43dd0bde288ac0170f9d3e47e0 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 25 Apr 2019 17:38:10 +0200 Subject: [PATCH 14/92] install python3 --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index 8c28cf7f3..66bc3ca6e 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -11,7 +11,7 @@ pipeline: apt-get update - desc: 'Install required build software' cmd: | - apt-get install -y make git apt-transport-https ca-certificates curl build-essential + apt-get install -y make git apt-transport-https ca-certificates curl build-essential python3 python3-pip - desc: 'Install go' cmd: | cd /tmp From d9e905bcdca020010eeea318e1ee750ed22e62a0 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 10:35:29 +0200 Subject: [PATCH 15/92] minor fixes --- e2e/mock_e2e.sh | 7 +++++-- e2e/tests.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh index a88ab7e06..9f29a1ee6 100755 --- a/e2e/mock_e2e.sh +++ b/e2e/mock_e2e.sh @@ -11,8 +11,11 @@ readonly cluster_name="kind-test-postgres-operator" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then - rm "$KUBECONFIG" - unset KUBECONFIG + # true if variable is set; bash >= v4.2 + if [[ -v KUBECONFIG ]];then + rm "$KUBECONFIG" + unset KUBECONFIG + fi kind delete cluster --name ${cluster_name} fi diff --git a/e2e/tests.py b/e2e/tests.py index aec41dcac..253872670 100755 --- a/e2e/tests.py +++ b/e2e/tests.py @@ -33,7 +33,7 @@ def test_assign_labels_to_nodes(self): if v1_node_var.metadata.labels['lifecycle-status'] == 'ready': labelled_nodes.add(v1_node_var.metadata.name) - assert self.nodes == labelled_nodes, "nodes incorrectly labelled" + self.assertEqual(self.nodes, labelled_nodes,"nodes incorrectly labelled") def tearDown(self): """ From a967672cd0b240536b0349f4e7dd9f40f1ef633c Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 10:47:17 +0200 Subject: [PATCH 16/92] move tests to a separate dir --- e2e/mock_e2e.sh | 2 +- e2e/{tests.py => tests/test_example.py} | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) rename e2e/{tests.py => tests/test_example.py} (98%) diff --git a/e2e/mock_e2e.sh b/e2e/mock_e2e.sh index 9f29a1ee6..a95558fe9 100755 --- a/e2e/mock_e2e.sh +++ b/e2e/mock_e2e.sh @@ -23,4 +23,4 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind. export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info -./e2e/tests.py \ No newline at end of file +python3 -m unittest discover -s e2e/tests/ \ No newline at end of file diff --git a/e2e/tests.py b/e2e/tests/test_example.py similarity index 98% rename from e2e/tests.py rename to e2e/tests/test_example.py index 253872670..859c87cd9 100755 --- a/e2e/tests.py +++ b/e2e/tests/test_example.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import unittest from kubernetes import client, config From d358e16cf7a0f8f87076631f22715e75b852e8e7 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 10:50:54 +0200 Subject: [PATCH 17/92] rename test runner --- Makefile | 2 +- e2e/{mock_e2e.sh => run.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename e2e/{mock_e2e.sh => run.sh} (100%) diff --git a/Makefile b/Makefile index ce59bfe81..9a88068dc 100644 --- a/Makefile +++ b/Makefile @@ -93,4 +93,4 @@ test: @go test ./... e2e: - e2e/mock_e2e.sh + e2e/run.sh diff --git a/e2e/mock_e2e.sh b/e2e/run.sh similarity index 100% rename from e2e/mock_e2e.sh rename to e2e/run.sh From 84715901e6e649dfa7df5e13c0c498062cbe913b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 10:54:37 +0200 Subject: [PATCH 18/92] add Python to .gitignore --- .gitignore | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.gitignore b/.gitignore index e09b6644b..af02ef71f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,59 @@ scm-source.json # diagrams *.aux *.log + +# Python +# Adapted from https://github.com/github/gitignore/blob/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot From 78d2b9ce97f30e6023ee973f6791cdac1c1718b8 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:12:51 +0200 Subject: [PATCH 19/92] run e2e tests in Travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0fd48a9ca..886515b20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,14 @@ go: before_install: - go get github.com/Masterminds/glide - go get github.com/mattn/goveralls + - go get sigs.k8s.io/kind + - apt-get install -y python3 python3-pip install: - make deps + - pip3 install -r requirements.txt script: - hack/verify-codegen.sh + - make e2e - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v From 1ae51f9fa22b2efd25ab279d07a9952864ba8dfe Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:18:18 +0200 Subject: [PATCH 20/92] change priv in Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 886515b20..7bf7f3b85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ dist: trusty -sudo: false +sudo: true branches: only: From eafd30284b0b519fd03548035e27c7ae455f6d01 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:22:55 +0200 Subject: [PATCH 21/92] add sudo to travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7bf7f3b85..f939c22d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - go get github.com/Masterminds/glide - go get github.com/mattn/goveralls - go get sigs.k8s.io/kind - - apt-get install -y python3 python3-pip + - sudo apt-get install -y python3 python3-pip install: - make deps From d801ac3f6b777804a3adb610aa8b92181f799c68 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:33:25 +0200 Subject: [PATCH 22/92] try to bump up pip3 version --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f939c22d9..8b57f8eeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial sudo: true branches: @@ -14,6 +14,7 @@ before_install: - go get github.com/Masterminds/glide - go get github.com/mattn/goveralls - go get sigs.k8s.io/kind + - sudo apt update - sudo apt-get install -y python3 python3-pip install: From e9b2f0b705a51814c59b54fa69f263d53a3bb3f4 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:35:47 +0200 Subject: [PATCH 23/92] one more try --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8b57f8eeb..b7ee12d70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ before_install: install: - make deps + - pip3 install -U pip3 - pip3 install -r requirements.txt script: From 179e459704323316d853dfed912e182b71672c80 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 11:47:21 +0200 Subject: [PATCH 24/92] restore original travis yaml --- .travis.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7ee12d70..0fd48a9ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ -dist: xenial -sudo: true +dist: trusty +sudo: false branches: only: @@ -13,16 +13,10 @@ go: before_install: - go get github.com/Masterminds/glide - go get github.com/mattn/goveralls - - go get sigs.k8s.io/kind - - sudo apt update - - sudo apt-get install -y python3 python3-pip install: - make deps - - pip3 install -U pip3 - - pip3 install -r requirements.txt script: - hack/verify-codegen.sh - - make e2e - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v From 0d0480d515c71847e55ab04955e97307e39e20ea Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 12:07:45 +0200 Subject: [PATCH 25/92] Add flake --- .flake8 | 3 +++ Makefile | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..812954650 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +exclude=.git,__pycache__ +max-line-length=120 diff --git a/Makefile b/Makefile index 9a88068dc..b0ac8a254 100644 --- a/Makefile +++ b/Makefile @@ -94,3 +94,5 @@ test: e2e: e2e/run.sh + # TODO run before tests once there are implemented completely + flake8 --exit-zero From 646de3d77e153c9f06c71f6ea54967182022ee16 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 13:02:47 +0200 Subject: [PATCH 26/92] minor fixes --- Makefile | 2 +- e2e/run.sh | 2 +- requirements.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b0ac8a254..b2f162b70 100644 --- a/Makefile +++ b/Makefile @@ -94,5 +94,5 @@ test: e2e: e2e/run.sh - # TODO run before tests once there are implemented completely + # TODO run before tests once they are implemented completely flake8 --exit-zero diff --git a/e2e/run.sh b/e2e/run.sh index a95558fe9..a118b5d0f 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -23,4 +23,4 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind. export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info -python3 -m unittest discover -s e2e/tests/ \ No newline at end of file +python3 -m unittest discover --start-directory e2e/tests/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 807e21be4..6fff5e0ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ kubernetes +flake8 From 0474be923f5afdb4aad9804c8eaa2f61b9393279 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 26 Apr 2019 17:11:48 +0200 Subject: [PATCH 27/92] deploy operator before running test --- e2e/tests/test_example.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/e2e/tests/test_example.py b/e2e/tests/test_example.py index 859c87cd9..70b96e27c 100755 --- a/e2e/tests/test_example.py +++ b/e2e/tests/test_example.py @@ -1,12 +1,35 @@ -import unittest -from kubernetes import client, config +import unittest, yaml +from kubernetes import client, config, utils from pprint import pprint +import subprocess class SampleTestCase(unittest.TestCase): nodes = set(["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"]) + @classmethod + def setUpClass(cls): + + # deploy operator + + _ = config.load_kube_config() + k8s_client = client.ApiClient() + + # HACK create_from_yaml fails with multiple object defined within a single file + # which is exactly the case with RBAC definition + subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) + + for filename in ["configmap.yaml", "postgres-operator.yaml"]: + path = "manifests/" + filename + utils.create_from_yaml(k8s_client, path) + + #TODO wait until operator pod starts up label ; name=postgres-operator + + @classmethod + def tearDownClass(cls): + pass + def setUp(self): self.config = config.load_kube_config() self.v1 = client.CoreV1Api() From 02c2d488bcef5d01005f5e02f4a6c7167f920f2c Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 29 Apr 2019 12:23:39 +0200 Subject: [PATCH 28/92] create/delete the operator deployment --- e2e/tests/test_example.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/e2e/tests/test_example.py b/e2e/tests/test_example.py index 70b96e27c..5d77b9f6d 100755 --- a/e2e/tests/test_example.py +++ b/e2e/tests/test_example.py @@ -1,5 +1,5 @@ -import unittest, yaml +import unittest, yaml, time from kubernetes import client, config, utils from pprint import pprint import subprocess @@ -8,6 +8,7 @@ class SampleTestCase(unittest.TestCase): nodes = set(["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"]) + config = None @classmethod def setUpClass(cls): @@ -16,19 +17,28 @@ def setUpClass(cls): _ = config.load_kube_config() k8s_client = client.ApiClient() - # HACK create_from_yaml fails with multiple object defined within a single file - # which is exactly the case with RBAC definition + # TODO split into multiple files subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) for filename in ["configmap.yaml", "postgres-operator.yaml"]: path = "manifests/" + filename utils.create_from_yaml(k8s_client, path) - - #TODO wait until operator pod starts up label ; name=postgres-operator + + v1 = client.CoreV1Api() + pod_phase = None + + while pod_phase != 'Running': + pods = v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items + if pods: + operator_pod = pods[0] + pod_phase = operator_pod.status.phase + print("Waiting for the operator pod to start. Current phase: " + pod_phase) + time.sleep(5) @classmethod def tearDownClass(cls): - pass + apps_v1 = client.AppsV1Api() + _ = apps_v1.delete_namespaced_deployment("postgres-operator", "default") def setUp(self): self.config = config.load_kube_config() @@ -54,7 +64,7 @@ def test_assign_labels_to_nodes(self): v1_node_var = self.v1.read_node(node) if v1_node_var.metadata.labels['lifecycle-status'] == 'ready': labelled_nodes.add(v1_node_var.metadata.name) - + self.assertEqual(self.nodes, labelled_nodes,"nodes incorrectly labelled") def tearDown(self): From b63a11ea315c885413e1c0bcef40e4d80f40297d Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 29 Apr 2019 15:47:55 +0200 Subject: [PATCH 29/92] deploy/delete DB for each test --- e2e/run.sh | 8 ++--- e2e/tests/test_example.py | 68 +++++++++++++++++++++++++++------------ 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index a118b5d0f..3462499fd 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -11,11 +11,6 @@ readonly cluster_name="kind-test-postgres-operator" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then - # true if variable is set; bash >= v4.2 - if [[ -v KUBECONFIG ]];then - rm "$KUBECONFIG" - unset KUBECONFIG - fi kind delete cluster --name ${cluster_name} fi @@ -23,4 +18,5 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind. export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info -python3 -m unittest discover --start-directory e2e/tests/ \ No newline at end of file +python3 -m unittest discover --start-directory e2e/tests/ && +kind delete cluster --name ${cluster_name} \ No newline at end of file diff --git a/e2e/tests/test_example.py b/e2e/tests/test_example.py index 5d77b9f6d..db3814297 100755 --- a/e2e/tests/test_example.py +++ b/e2e/tests/test_example.py @@ -8,16 +8,20 @@ class SampleTestCase(unittest.TestCase): nodes = set(["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"]) - config = None @classmethod def setUpClass(cls): - - # deploy operator + ''' + Deploy operator to a "kind" cluster created by /e2e/run.sh using examples from /manifests. + This operator deployment is to be shared among all tests of this suit. + ''' _ = config.load_kube_config() k8s_client = client.ApiClient() - # TODO split into multiple files + # HACK + # 1. creating RBAC entites with a separate client fails with + # "AttributeError: object has no attribute 'select_header_accept'" + # 2. utils.create_from_yaml cannot create multiple entites from a single file subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) for filename in ["configmap.yaml", "postgres-operator.yaml"]: @@ -26,24 +30,44 @@ def setUpClass(cls): v1 = client.CoreV1Api() pod_phase = None - while pod_phase != 'Running': pods = v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items if pods: operator_pod = pods[0] pod_phase = operator_pod.status.phase print("Waiting for the operator pod to start. Current phase: " + pod_phase) - time.sleep(5) + time.sleep(5) @classmethod def tearDownClass(cls): - apps_v1 = client.AppsV1Api() - _ = apps_v1.delete_namespaced_deployment("postgres-operator", "default") + ''' + /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities. + In the case of test failure the cluster will stay to enable manual examination; + next invocation of "make e2e" will re-create it. + ''' + pass def setUp(self): + ''' + Deploy a new Postgres DB for each test. + ''' self.config = config.load_kube_config() self.v1 = client.CoreV1Api() + k8s_client = client.ApiClient() + + # TODO substitue with utils.create_from_yaml and Python client for acid.zalan.do + subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) + + pod_phase = None + while pod_phase != 'Running': + pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items + if pods: + operator_pod = pods[0] + pod_phase = operator_pod.status.phase + print("Waiting for the Spilo master pod to start. Current phase: " + pod_phase) + time.sleep(5) + def test_assign_labels_to_nodes(self): """ Ensure labeling nodes through the externally connected Python client works. @@ -64,23 +88,27 @@ def test_assign_labels_to_nodes(self): v1_node_var = self.v1.read_node(node) if v1_node_var.metadata.labels['lifecycle-status'] == 'ready': labelled_nodes.add(v1_node_var.metadata.name) - + self.assertEqual(self.nodes, labelled_nodes,"nodes incorrectly labelled") def tearDown(self): """ - Each test must restore the original cluster state - to avoid introducing dependencies between tests + Delete the database to avoid introducing dependencies between tests """ - body = { - "metadata": { - "labels": { - "lifecycle-status": None # deletes label - } - } - } - for node in self.nodes: - _ = self.v1.patch_node(node, body) + # HACK workaround for #551 + time.sleep(60) + + _ = config.load_kube_config() + crd = client.CustomObjectsApi() + body = client.V1DeleteOptions() + _ = crd.delete_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + + # wait for the pods to be deleted + pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items + while pods: + pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items + print("Waiting for the database to be deleted.") + time.sleep(5) if __name__ == '__main__': unittest.main() \ No newline at end of file From 63bd8469488e1a499cfcba925ddde4cd342ef7e9 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 29 Apr 2019 17:11:46 +0200 Subject: [PATCH 30/92] prepare smoke test template --- e2e/tests/test_example.py | 94 ++++++++++++++------------------------- 1 file changed, 33 insertions(+), 61 deletions(-) diff --git a/e2e/tests/test_example.py b/e2e/tests/test_example.py index db3814297..35131d58a 100755 --- a/e2e/tests/test_example.py +++ b/e2e/tests/test_example.py @@ -4,23 +4,24 @@ from pprint import pprint import subprocess -class SampleTestCase(unittest.TestCase): - - nodes = set(["kind-test-postgres-operator-worker", "kind-test-postgres-operator-worker2", "kind-test-postgres-operator-worker3"]) +class SmokeTestCase(unittest.TestCase): @classmethod def setUpClass(cls): ''' Deploy operator to a "kind" cluster created by /e2e/run.sh using examples from /manifests. - This operator deployment is to be shared among all tests of this suit. + This operator deployment is to be shared among all tests. + + /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities. + In the case of test failure the cluster will stay to enable manual examination; + next invocation of "make e2e" will re-create it. ''' _ = config.load_kube_config() k8s_client = client.ApiClient() # HACK - # 1. creating RBAC entites with a separate client fails with - # "AttributeError: object has no attribute 'select_header_accept'" + # 1. creating RBAC entites with a separate client fails with "AttributeError: object has no attribute 'select_header_accept'" # 2. utils.create_from_yaml cannot create multiple entites from a single file subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) @@ -35,79 +36,50 @@ def setUpClass(cls): if pods: operator_pod = pods[0] pod_phase = operator_pod.status.phase - print("Waiting for the operator pod to start. Current phase: " + pod_phase) + print("Waiting for the operator pod to start. Current phase of pod lifecycle: " + str(pod_phase)) time.sleep(5) - @classmethod - def tearDownClass(cls): - ''' - /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities. - In the case of test failure the cluster will stay to enable manual examination; - next invocation of "make e2e" will re-create it. - ''' - pass - - def setUp(self): - ''' - Deploy a new Postgres DB for each test. - ''' - self.config = config.load_kube_config() - self.v1 = client.CoreV1Api() - k8s_client = client.ApiClient() - # TODO substitue with utils.create_from_yaml and Python client for acid.zalan.do + # HACK around the lack of Python client for the acid.zalan.do resource subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - pod_phase = None + pod_phase = 'None' while pod_phase != 'Running': - pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items + pods = v1.list_namespaced_pod('default', label_selector='spilo-role=master').items if pods: operator_pod = pods[0] pod_phase = operator_pod.status.phase - print("Waiting for the Spilo master pod to start. Current phase: " + pod_phase) - time.sleep(5) + print("Waiting for the Spilo master pod to start. Current phase: " + str(pod_phase)) + time.sleep(5) - def test_assign_labels_to_nodes(self): + def test_master_is_unique(self): """ - Ensure labeling nodes through the externally connected Python client works. - Sample test case to illustrate potential test structure + Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ - body = { - "metadata": { - "labels": { - "lifecycle-status": "ready" - } - } - } - for node in self.nodes: - _ = self.v1.patch_node(node, body) - - labelled_nodes = set([]) - for node in self.nodes: - v1_node_var = self.v1.read_node(node) - if v1_node_var.metadata.labels['lifecycle-status'] == 'ready': - labelled_nodes.add(v1_node_var.metadata.name) - - self.assertEqual(self.nodes, labelled_nodes,"nodes incorrectly labelled") + _ = config.load_kube_config() + v1 = client.CoreV1Api() + master_pods = v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items + self.assertEqual(len(master_pods), 1, "Expected 1 master pod,found " + str(len(master_pods))) - def tearDown(self): + def test_scaling(self): """ - Delete the database to avoid introducing dependencies between tests + Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ - # HACK workaround for #551 - time.sleep(60) _ = config.load_kube_config() - crd = client.CustomObjectsApi() - body = client.V1DeleteOptions() - _ = crd.delete_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) - - # wait for the pods to be deleted - pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items - while pods: - pods = self.v1.list_namespaced_pod('default', label_selector='spilo-role=master').items - print("Waiting for the database to be deleted.") + crd_api = client.CustomObjectsApi() + v1 = client.CoreV1Api() + + body = { + "spec": { + "numberOfInstances": 3 + } + } + _ = crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + + while len(v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 3: + print("Waiting for the cluster to scale up to 3 podes.") time.sleep(5) if __name__ == '__main__': From 021ed9b8b09fdb9d8c42bf23594618352e0aaeac Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 30 Apr 2019 13:14:40 +0200 Subject: [PATCH 31/92] explicitly name smoke tests --- ...ind-config-multikind.yaml => kind-config-smoke-tests.yaml} | 0 e2e/run.sh | 4 ++-- e2e/tests/{test_example.py => test_smoke.py} | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) rename e2e/{kind-config-multikind.yaml => kind-config-smoke-tests.yaml} (100%) rename e2e/tests/{test_example.py => test_smoke.py} (97%) diff --git a/e2e/kind-config-multikind.yaml b/e2e/kind-config-smoke-tests.yaml similarity index 100% rename from e2e/kind-config-multikind.yaml rename to e2e/kind-config-smoke-tests.yaml diff --git a/e2e/run.sh b/e2e/run.sh index 3462499fd..d6673610e 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,7 +6,7 @@ set -o nounset set -o pipefail IFS=$'\n\t' -readonly cluster_name="kind-test-postgres-operator" +readonly cluster_name="kind-smoke-test-postgres-operator" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] @@ -14,7 +14,7 @@ then kind delete cluster --name ${cluster_name} fi -kind create cluster --name ${cluster_name} --config ./e2e/kind-config-multikind.yaml +kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-tests.yaml export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info diff --git a/e2e/tests/test_example.py b/e2e/tests/test_smoke.py similarity index 97% rename from e2e/tests/test_example.py rename to e2e/tests/test_smoke.py index 35131d58a..baeae7b13 100755 --- a/e2e/tests/test_example.py +++ b/e2e/tests/test_smoke.py @@ -5,6 +5,9 @@ import subprocess class SmokeTestCase(unittest.TestCase): + ''' + Test the most basic e2e functionality of the operator. + ''' @classmethod def setUpClass(cls): From 2652d18f24e2fb8c1f026ee2578e3cd673a7b4c0 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 30 Apr 2019 13:49:42 +0200 Subject: [PATCH 32/92] add timeouts --- e2e/tests/test_smoke.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index baeae7b13..7ad51cf1b 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -2,13 +2,13 @@ import unittest, yaml, time from kubernetes import client, config, utils from pprint import pprint +import timeout_decorator import subprocess class SmokeTestCase(unittest.TestCase): ''' Test the most basic e2e functionality of the operator. ''' - @classmethod def setUpClass(cls): ''' @@ -56,6 +56,7 @@ def setUpClass(cls): print("Waiting for the Spilo master pod to start. Current phase: " + str(pod_phase)) time.sleep(5) + @timeout_decorator.timeout(60) def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". @@ -65,6 +66,7 @@ def test_master_is_unique(self): master_pods = v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items self.assertEqual(len(master_pods), 1, "Expected 1 master pod,found " + str(len(master_pods))) + @timeout_decorator.timeout(60) def test_scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. From af202bf55019a33411996f62b229739bf08a9ad9 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 30 Apr 2019 13:49:51 +0200 Subject: [PATCH 33/92] start the e2e docs --- docs/developer.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index a18ac3323..17745cca1 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -315,6 +315,23 @@ Then you can for example check the Patroni logs: kubectl logs acid-minimal-cluster-0 ``` +## End-to-end tests + +The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. +Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. +We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory. + +### Smoke tests + +The first provided collection of [smoke tests](../e2e/tests/test_smoke.py) covers the most basic operator capabilities. These tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The `kind-smoke-test-postgres-operator` cluster is deleted if tests complete successfully. + +### Contributing tests suites + +1. Consider using a separate `kind` cluster per a logically separate collection of tests. This approach enables simple clean up after test success by deleting the relevant cluster. +2. Make tests time out; for now we use the [timeout-decorator](https://github.com/pnpnpn/timeout-decorator) for that purpose. +3. Please briefly describe the use case for the new test collection here and in the comments. + + ## Introduce additional configuration parameters In the case you want to add functionality to the operator that shall be From 806ca7ba81f148a51768f7634ac20e2a7472cbf2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 30 Apr 2019 15:18:07 +0200 Subject: [PATCH 34/92] assemble k8s client; patch scaling test --- e2e/tests/test_smoke.py | 63 ++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 7ad51cf1b..23fcc22bc 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -5,10 +5,21 @@ import timeout_decorator import subprocess + +class K8sApi: + + def __init__(self): + self.config = config.load_kube_config() + self.k8s_client = client.ApiClient() + self.core_v1 = client.CoreV1Api() + self.crd_api = client.CustomObjectsApi() + + class SmokeTestCase(unittest.TestCase): ''' Test the most basic e2e functionality of the operator. ''' + @classmethod def setUpClass(cls): ''' @@ -19,73 +30,79 @@ def setUpClass(cls): In the case of test failure the cluster will stay to enable manual examination; next invocation of "make e2e" will re-create it. ''' - - _ = config.load_kube_config() - k8s_client = client.ApiClient() - + # HACK # 1. creating RBAC entites with a separate client fails with "AttributeError: object has no attribute 'select_header_accept'" # 2. utils.create_from_yaml cannot create multiple entites from a single file subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) + k8s_api = K8sApi() + for filename in ["configmap.yaml", "postgres-operator.yaml"]: path = "manifests/" + filename - utils.create_from_yaml(k8s_client, path) + utils.create_from_yaml(k8s_api.k8s_client, path) - v1 = client.CoreV1Api() pod_phase = None while pod_phase != 'Running': - pods = v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items if pods: operator_pod = pods[0] pod_phase = operator_pod.status.phase print("Waiting for the operator pod to start. Current phase of pod lifecycle: " + str(pod_phase)) time.sleep(5) - k8s_client = client.ApiClient() - # HACK around the lack of Python client for the acid.zalan.do resource subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) pod_phase = 'None' while pod_phase != 'Running': - pods = v1.list_namespaced_pod('default', label_selector='spilo-role=master').items + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master').items if pods: operator_pod = pods[0] pod_phase = operator_pod.status.phase print("Waiting for the Spilo master pod to start. Current phase: " + str(pod_phase)) time.sleep(5) - - @timeout_decorator.timeout(60) + + @timeout_decorator.timeout(240) def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ - _ = config.load_kube_config() - v1 = client.CoreV1Api() - master_pods = v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items + k8s = K8sApi() + master_pods = k8s.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items self.assertEqual(len(master_pods), 1, "Expected 1 master pod,found " + str(len(master_pods))) - @timeout_decorator.timeout(60) + @timeout_decorator.timeout(240) def test_scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ - - _ = config.load_kube_config() - crd_api = client.CustomObjectsApi() - v1 = client.CoreV1Api() + k8s = K8sApi() body = { "spec": { "numberOfInstances": 3 } } - _ = crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + _ = k8s.crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + + while len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 3: + print("Waiting for the cluster to scale up to 3 pods.") + time.sleep(5) + self.assertEqual(3, len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items)) + + body = { + "spec": { + "numberOfInstances": 2 + } + } + + _ = k8s.crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) - while len(v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 3: - print("Waiting for the cluster to scale up to 3 podes.") + while len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 2: + print("Waiting for the cluster to scale down to 2 pods.") time.sleep(5) + self.assertEqual(2, len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items)) if __name__ == '__main__': unittest.main() \ No newline at end of file From 30e7445c804869b9a74d4a68b189f087e0a860d3 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 11:46:14 +0200 Subject: [PATCH 35/92] submit custom operator images --- Makefile | 3 +-- e2e/kind-config-smoke-tests.yaml | 1 - e2e/run.sh | 3 +++ e2e/tests/test_smoke.py | 24 +++++++++++++++++++++++- requirements.txt | 2 ++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b2f162b70..3ec27256c 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,6 @@ test: hack/verify-codegen.sh @go test ./... -e2e: +e2e: docker e2e/run.sh - # TODO run before tests once they are implemented completely flake8 --exit-zero diff --git a/e2e/kind-config-smoke-tests.yaml b/e2e/kind-config-smoke-tests.yaml index d95d68b2e..a59746fd3 100644 --- a/e2e/kind-config-smoke-tests.yaml +++ b/e2e/kind-config-smoke-tests.yaml @@ -4,4 +4,3 @@ nodes: - role: control-plane - role: worker - role: worker -- role: worker diff --git a/e2e/run.sh b/e2e/run.sh index d6673610e..c2ccf4d2a 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -18,5 +18,8 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-test export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info +version=$(git describe --tags --always --dirty) +kind load docker-image "registry.opensource.zalan.do/acid/postgres-operator:${version}" --name ${cluster_name} + python3 -m unittest discover --start-directory e2e/tests/ && kind delete cluster --name ${cluster_name} \ No newline at end of file diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 23fcc22bc..15044e398 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -4,7 +4,7 @@ from pprint import pprint import timeout_decorator import subprocess - +import git class K8sApi: @@ -13,6 +13,8 @@ def __init__(self): self.k8s_client = client.ApiClient() self.core_v1 = client.CoreV1Api() self.crd_api = client.CustomObjectsApi() + self.apps_v1 = client.AppsV1Api() + class SmokeTestCase(unittest.TestCase): @@ -41,6 +43,26 @@ def setUpClass(cls): for filename in ["configmap.yaml", "postgres-operator.yaml"]: path = "manifests/" + filename utils.create_from_yaml(k8s_api.k8s_client, path) + + # submit most recent operator image built locally; see VERSION in Makefile + repo = git.Repo(".") + version = repo.git.describe("--tags", "--always", "--dirty") + + body = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name" : "postgres-operator", + "image": "registry.opensource.zalan.do/acid/postgres-operator:" + version + } + ] + } + } + } + } + k8s_api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) pod_phase = None while pod_phase != 'Running': diff --git a/requirements.txt b/requirements.txt index 6fff5e0ff..d47e7d619 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ kubernetes flake8 +git +timeout_decorator From 25fea48f8a93630268afe3ddcc499d72327dc874 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 12:04:01 +0200 Subject: [PATCH 36/92] fix flake8 warnings --- e2e/tests/test_smoke.py | 73 ++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 15044e398..edbb3ab8c 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -1,20 +1,21 @@ -import unittest, yaml, time -from kubernetes import client, config, utils -from pprint import pprint +import unittest +import time import timeout_decorator import subprocess import git +from kubernetes import client, config, utils + + class K8sApi: def __init__(self): - self.config = config.load_kube_config() - self.k8s_client = client.ApiClient() - self.core_v1 = client.CoreV1Api() - self.crd_api = client.CustomObjectsApi() - self.apps_v1 = client.AppsV1Api() - + self.config = config.load_kube_config() + self.k8s_client = client.ApiClient() + self.core_v1 = client.CoreV1Api() + self.crd = client.CustomObjectsApi() + self.apps_v1 = client.AppsV1Api() class SmokeTestCase(unittest.TestCase): @@ -32,9 +33,10 @@ def setUpClass(cls): In the case of test failure the cluster will stay to enable manual examination; next invocation of "make e2e" will re-create it. ''' - - # HACK - # 1. creating RBAC entites with a separate client fails with "AttributeError: object has no attribute 'select_header_accept'" + + # HACK + # 1. creating RBAC entites with a separate client fails + # with "AttributeError: object has no attribute 'select_header_accept'" # 2. utils.create_from_yaml cannot create multiple entites from a single file subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) @@ -43,20 +45,20 @@ def setUpClass(cls): for filename in ["configmap.yaml", "postgres-operator.yaml"]: path = "manifests/" + filename utils.create_from_yaml(k8s_api.k8s_client, path) - + # submit most recent operator image built locally; see VERSION in Makefile repo = git.Repo(".") version = repo.git.describe("--tags", "--always", "--dirty") - + body = { "spec": { "template": { "spec": { - "containers": [ - { - "name" : "postgres-operator", - "image": "registry.opensource.zalan.do/acid/postgres-operator:" + version - } + "containers": [ + { + "name": "postgres-operator", + "image": "registry.opensource.zalan.do/acid/postgres-operator:" + version + } ] } } @@ -68,8 +70,8 @@ def setUpClass(cls): while pod_phase != 'Running': pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items if pods: - operator_pod = pods[0] - pod_phase = operator_pod.status.phase + operator_pod = pods[0] + pod_phase = operator_pod.status.phase print("Waiting for the operator pod to start. Current phase of pod lifecycle: " + str(pod_phase)) time.sleep(5) @@ -80,19 +82,20 @@ def setUpClass(cls): while pod_phase != 'Running': pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master').items if pods: - operator_pod = pods[0] - pod_phase = operator_pod.status.phase + operator_pod = pods[0] + pod_phase = operator_pod.status.phase print("Waiting for the Spilo master pod to start. Current phase: " + str(pod_phase)) time.sleep(5) - + @timeout_decorator.timeout(240) def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ k8s = K8sApi() - master_pods = k8s.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items - self.assertEqual(len(master_pods), 1, "Expected 1 master pod,found " + str(len(master_pods))) + labels = 'spilo-role=master,version=acid-minimal-cluster' + master_pods = k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items + self.assertEqual(len(master_pods), 1, "Expected 1 master pod, found " + str(len(master_pods))) @timeout_decorator.timeout(240) def test_scaling(self): @@ -106,12 +109,14 @@ def test_scaling(self): "numberOfInstances": 3 } } - _ = k8s.crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", + "v1", "default", "postgresqls", "acid-minimal-cluster", body) - while len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 3: + labels = 'version=acid-minimal-cluster' + while len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) != 3: print("Waiting for the cluster to scale up to 3 pods.") time.sleep(5) - self.assertEqual(3, len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items)) + self.assertEqual(3, len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items)) body = { "spec": { @@ -119,12 +124,14 @@ def test_scaling(self): } } - _ = k8s.crd_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) + _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", + "v1", "default", "postgresqls", "acid-minimal-cluster", body) - while len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items) != 2: + while len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) != 2: print("Waiting for the cluster to scale down to 2 pods.") time.sleep(5) - self.assertEqual(2, len(k8s.core_v1.list_namespaced_pod('default', label_selector='version=acid-minimal-cluster').items)) + self.assertEqual(2, len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items)) + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 78582802d0c36b9d64c5e0f8bdfe3f462905c55c Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 13:15:51 +0200 Subject: [PATCH 37/92] refactor into helper methods --- e2e/tests/test_smoke.py | 116 ++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index edbb3ab8c..cae18a22b 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -1,4 +1,3 @@ - import unittest import time import timeout_decorator @@ -8,21 +7,14 @@ from kubernetes import client, config, utils -class K8sApi: - - def __init__(self): - self.config = config.load_kube_config() - self.k8s_client = client.ApiClient() - self.core_v1 = client.CoreV1Api() - self.crd = client.CustomObjectsApi() - self.apps_v1 = client.AppsV1Api() - - class SmokeTestCase(unittest.TestCase): ''' - Test the most basic e2e functionality of the operator. + Test the most basic functions of the operator. ''' + TEST_TIMEOUT_SEC = 240 + RETRY_TIMEOUT_SEC = 5 + @classmethod def setUpClass(cls): ''' @@ -34,22 +26,19 @@ def setUpClass(cls): next invocation of "make e2e" will re-create it. ''' + k8s_api = K8sApi() + # HACK # 1. creating RBAC entites with a separate client fails # with "AttributeError: object has no attribute 'select_header_accept'" # 2. utils.create_from_yaml cannot create multiple entites from a single file subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) - k8s_api = K8sApi() - for filename in ["configmap.yaml", "postgres-operator.yaml"]: - path = "manifests/" + filename - utils.create_from_yaml(k8s_api.k8s_client, path) - - # submit most recent operator image built locally; see VERSION in Makefile - repo = git.Repo(".") - version = repo.git.describe("--tags", "--always", "--dirty") + utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) + # submit the most recent operator image built locally; see VERSION in Makefile + version = git.Repo(".").git.describe("--tags", "--always", "--dirty") body = { "spec": { "template": { @@ -66,71 +55,80 @@ def setUpClass(cls): } k8s_api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - pod_phase = None - while pod_phase != 'Running': - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items - if pods: - operator_pod = pods[0] - pod_phase = operator_pod.status.phase - print("Waiting for the operator pod to start. Current phase of pod lifecycle: " + str(pod_phase)) - time.sleep(5) + Utils.wait_for_pod_start(k8s_api, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) # HACK around the lack of Python client for the acid.zalan.do resource subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - pod_phase = 'None' - while pod_phase != 'Running': - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master').items - if pods: - operator_pod = pods[0] - pod_phase = operator_pod.status.phase - print("Waiting for the Spilo master pod to start. Current phase: " + str(pod_phase)) - time.sleep(5) + Utils.wait_for_pod_start(k8s_api, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) - @timeout_decorator.timeout(240) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ k8s = K8sApi() labels = 'spilo-role=master,version=acid-minimal-cluster' - master_pods = k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items - self.assertEqual(len(master_pods), 1, "Expected 1 master pod, found " + str(len(master_pods))) - @timeout_decorator.timeout(240) + num_of_master_pods = Utils.count_pods_with_label(k8s, labels) + self.assertEqual(num_of_master_pods, 1, f"Expected 1 master pod, found {num_of_master_pods}") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ k8s = K8sApi() + labels = "version=acid-minimal-cluster" - body = { - "spec": { - "numberOfInstances": 3 - } - } - _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", - "v1", "default", "postgresqls", "acid-minimal-cluster", body) + Utils.wait_for_pg_to_scale(k8s, 3, self.RETRY_TIMEOUT_SEC) + self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) - labels = 'version=acid-minimal-cluster' - while len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) != 3: - print("Waiting for the cluster to scale up to 3 pods.") - time.sleep(5) - self.assertEqual(3, len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items)) + Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) + self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) + + +class K8sApi: + + def __init__(self): + self.config = config.load_kube_config() + self.k8s_client = client.ApiClient() + self.core_v1 = client.CoreV1Api() + self.crd = client.CustomObjectsApi() + self.apps_v1 = client.AppsV1Api() + + +class Utils: + + @staticmethod + def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): + pod_phase = 'No pod running' + while pod_phase != 'Running': + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase + print(f"Wait for the {pod_labels} pod to start. Current pod phase: " + pod_phase) + time.sleep(retry_timeout_sec) + + @staticmethod + def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): body = { "spec": { - "numberOfInstances": 2 + "numberOfInstances": number_of_instances } } + _ = k8s_api.crd.patch_namespaced_custom_object("acid.zalan.do", + "v1", "default", "postgresqls", "acid-minimal-cluster", body) - _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", - "v1", "default", "postgresqls", "acid-minimal-cluster", body) + labels = 'version=acid-minimal-cluster' + while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: + print(f"Waiting for the cluster to scale to {number_of_instances} pods.") + time.sleep(retry_timeout_sec) - while len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) != 2: - print("Waiting for the cluster to scale down to 2 pods.") - time.sleep(5) - self.assertEqual(2, len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items)) + @staticmethod + def count_pods_with_label(k8s_api, labels): + return len(k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items) if __name__ == '__main__': From 0c4d80c8e6e394ec428c7cb5c15226246850746a Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 13:22:58 +0200 Subject: [PATCH 38/92] fix requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d47e7d619..9e9957e7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ kubernetes flake8 -git +gitpython timeout_decorator From cb9c77edcb32a06ba56c9bd317b1174cd327606a Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 13:27:56 +0200 Subject: [PATCH 39/92] mention docker build during e2e --- docs/developer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer.md b/docs/developer.md index 17745cca1..5a81011b0 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -319,7 +319,7 @@ kubectl logs acid-minimal-cluster-0 The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. -We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory. +We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory. Each e2e execution tests a Postgres operator image built from the current git branch. ### Smoke tests From f9d145c1b16c1efb81ec987f7b914555e024274a Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 13:32:32 +0200 Subject: [PATCH 40/92] make setUp timeout --- e2e/tests/test_smoke.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index cae18a22b..7dd6f5477 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -12,10 +12,11 @@ class SmokeTestCase(unittest.TestCase): Test the most basic functions of the operator. ''' - TEST_TIMEOUT_SEC = 240 + TEST_TIMEOUT_SEC = 300 RETRY_TIMEOUT_SEC = 5 @classmethod + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def setUpClass(cls): ''' Deploy operator to a "kind" cluster created by /e2e/run.sh using examples from /manifests. From 1ced414985e9072315ad1def1ae8ad8c79fc0238 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 14:20:46 +0200 Subject: [PATCH 41/92] minor changes --- e2e/tests/test_smoke.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 7dd6f5477..98506a9ee 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -3,6 +3,7 @@ import timeout_decorator import subprocess import git +import warnings from kubernetes import client, config, utils @@ -12,7 +13,7 @@ class SmokeTestCase(unittest.TestCase): Test the most basic functions of the operator. ''' - TEST_TIMEOUT_SEC = 300 + TEST_TIMEOUT_SEC = 600 RETRY_TIMEOUT_SEC = 5 @classmethod @@ -39,6 +40,7 @@ def setUpClass(cls): utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) # submit the most recent operator image built locally; see VERSION in Makefile + # TODO properly fetch the recent image tag from the local Docker version = git.Repo(".").git.describe("--tags", "--always", "--dirty") body = { "spec": { @@ -56,6 +58,7 @@ def setUpClass(cls): } k8s_api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) + # TODO check if CRD is registered instead Utils.wait_for_pod_start(k8s_api, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) # HACK around the lack of Python client for the acid.zalan.do resource @@ -85,6 +88,7 @@ def test_scaling(self): Utils.wait_for_pg_to_scale(k8s, 3, self.RETRY_TIMEOUT_SEC) self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) + # NB `kind` pods may stuck in the Terminating state for a few minutes Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) @@ -92,6 +96,10 @@ def test_scaling(self): class K8sApi: def __init__(self): + + # https://github.com/kubernetes-client/python/issues/309 + warnings.simplefilter("ignore", ResourceWarning) + self.config = config.load_kube_config() self.k8s_client = client.ApiClient() self.core_v1 = client.CoreV1Api() @@ -108,7 +116,7 @@ def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - print(f"Wait for the {pod_labels} pod to start. Current pod phase: " + pod_phase) + print(f"Wait for the pod '{pod_labels}' to start. Current pod phase: " + pod_phase) time.sleep(retry_timeout_sec) @staticmethod From 8352abe42808dc7f9c59115ef4c3133b3c4ece92 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 15:06:59 +0200 Subject: [PATCH 42/92] fetch image name from docker --- e2e/tests/test_smoke.py | 15 ++++++++------- requirements.txt | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 98506a9ee..8c2843390 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -2,8 +2,8 @@ import time import timeout_decorator import subprocess -import git import warnings +import docker from kubernetes import client, config, utils @@ -39,9 +39,10 @@ def setUpClass(cls): for filename in ["configmap.yaml", "postgres-operator.yaml"]: utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) - # submit the most recent operator image built locally; see VERSION in Makefile - # TODO properly fetch the recent image tag from the local Docker - version = git.Repo(".").git.describe("--tags", "--always", "--dirty") + # submit the most recent operator image built locally + # HACK assumes "images lists" returns the most recent image at index 0 + docker_client = docker.from_env() + image = docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] body = { "spec": { "template": { @@ -49,7 +50,7 @@ def setUpClass(cls): "containers": [ { "name": "postgres-operator", - "image": "registry.opensource.zalan.do/acid/postgres-operator:" + version + "image": image } ] } @@ -88,7 +89,7 @@ def test_scaling(self): Utils.wait_for_pg_to_scale(k8s, 3, self.RETRY_TIMEOUT_SEC) self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) - # NB `kind` pods may stuck in the Terminating state for a few minutes + # TODO `kind` pods may stuck in the Terminating state for a few minutes; reproduce/file a bug report Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) @@ -116,7 +117,7 @@ def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - print(f"Wait for the pod '{pod_labels}' to start. Current pod phase: " + pod_phase) + print(f"Wait for the pod '{pod_labels}' to start. Current pod phase: {pod_phase}") time.sleep(retry_timeout_sec) @staticmethod diff --git a/requirements.txt b/requirements.txt index 9e9957e7a..40fb9efa1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ kubernetes flake8 -gitpython timeout_decorator +docker From 777ee682f3e24fc5073252590de9b33213f1fbae Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 15:20:49 +0200 Subject: [PATCH 43/92] upload newest operator image to kind --- e2e/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index c2ccf4d2a..63e482954 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -18,8 +18,8 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-test export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kubectl cluster-info -version=$(git describe --tags --always --dirty) -kind load docker-image "registry.opensource.zalan.do/acid/postgres-operator:${version}" --name ${cluster_name} +image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) +kind load docker-image ${image} --name ${cluster_name} python3 -m unittest discover --start-directory e2e/tests/ && kind delete cluster --name ${cluster_name} \ No newline at end of file From 2aba5291d5d6e203c912bdd1782999ea90ded7b2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 15:33:53 +0200 Subject: [PATCH 44/92] remove python 3.6 formatting strings --- e2e/tests/test_smoke.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 8c2843390..a70cdec43 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -76,7 +76,7 @@ def test_master_is_unique(self): labels = 'spilo-role=master,version=acid-minimal-cluster' num_of_master_pods = Utils.count_pods_with_label(k8s, labels) - self.assertEqual(num_of_master_pods, 1, f"Expected 1 master pod, found {num_of_master_pods}") + self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): @@ -117,7 +117,7 @@ def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - print(f"Wait for the pod '{pod_labels}' to start. Current pod phase: {pod_phase}") + print("Wait for the pod '{}' to start. Current pod phase: {}".format(pod_labels, pod_phase)) time.sleep(retry_timeout_sec) @staticmethod @@ -133,7 +133,7 @@ def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): labels = 'version=acid-minimal-cluster' while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: - print(f"Waiting for the cluster to scale to {number_of_instances} pods.") + print("Waiting for the cluster to scale to {} pods.".format(number_of_instances)) time.sleep(retry_timeout_sec) @staticmethod From f2b73e0ae6fa78401a55dc3f19937319f3f06af9 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 2 May 2019 17:50:44 +0200 Subject: [PATCH 45/92] add a make target to isntall required tools for e2e --- Makefile | 7 ++++++- delivery.yaml | 4 +--- docs/developer.md | 2 +- requirements.txt => e2e/requirements.txt | 0 e2e/tests/test_smoke.py | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) rename requirements.txt => e2e/requirements.txt (100%) diff --git a/Makefile b/Makefile index 3ec27256c..378aa0082 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean local test linux macos docker push scm-source.json e2e +.PHONY: clean local test linux macos docker push scm-source.json e2e e2e-tools BINARY ?= postgres-operator BUILD_FLAGS ?= -v @@ -92,6 +92,11 @@ test: hack/verify-codegen.sh @go test ./... +e2e-tools: + @pip3 install -r e2e/requirements.txt + @go get -u sigs.k8s.io/kind + # assumes kubectl is already isntalled + e2e: docker e2e/run.sh flake8 --exit-zero diff --git a/delivery.yaml b/delivery.yaml index 66bc3ca6e..cd221229d 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -45,13 +45,11 @@ pipeline: cmd: | export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator - go get -u sigs.k8s.io/kind - pip3 install -r requirements.txt echo "INFO install kubectl to test 'kind' from client side" (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin/kubectl) || exit 1 - make e2e + make e2e-tools e2e - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/docs/developer.md b/docs/developer.md index 5a81011b0..efdd1df17 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -319,7 +319,7 @@ kubectl logs acid-minimal-cluster-0 The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. -We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory. Each e2e execution tests a Postgres operator image built from the current git branch. +We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory; run `make e2e-tools` to install `kind` and Python dependencies before the first run. Each e2e execution tests a Postgres operator image built from the current git branch. ### Smoke tests diff --git a/requirements.txt b/e2e/requirements.txt similarity index 100% rename from requirements.txt rename to e2e/requirements.txt diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index a70cdec43..e4fef0ad2 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -40,7 +40,7 @@ def setUpClass(cls): utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) # submit the most recent operator image built locally - # HACK assumes "images lists" returns the most recent image at index 0 + # HACK assumes "images list" returns the most recent image at index 0 docker_client = docker.from_env() image = docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] body = { From bfb6b5341d83583c8271dff8d7ac860cc9ea7048 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 3 May 2019 10:39:19 +0200 Subject: [PATCH 46/92] Add progress reporting and make tests verbose --- e2e/run.sh | 2 +- e2e/tests/test_smoke.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index 63e482954..6d612a0cd 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -21,5 +21,5 @@ kubectl cluster-info image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) kind load docker-image ${image} --name ${cluster_name} -python3 -m unittest discover --start-directory e2e/tests/ && +python3 -m unittest discover --start-directory e2e/tests/ -v && kind delete cluster --name ${cluster_name} \ No newline at end of file diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index e4fef0ad2..bb3b5f5b0 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -4,6 +4,7 @@ import subprocess import warnings import docker +from halo import Halo from kubernetes import client, config, utils @@ -113,11 +114,11 @@ class Utils: @staticmethod def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' - while pod_phase != 'Running': - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items - if pods: - pod_phase = pods[0].status.phase - print("Wait for the pod '{}' to start. Current pod phase: {}".format(pod_labels, pod_phase)) + with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'): + while pod_phase != 'Running': + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase time.sleep(retry_timeout_sec) @staticmethod @@ -132,9 +133,9 @@ def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' - while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: - print("Waiting for the cluster to scale to {} pods.".format(number_of_instances)) - time.sleep(retry_timeout_sec) + with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'): + while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: + time.sleep(retry_timeout_sec) @staticmethod def count_pods_with_label(k8s_api, labels): From af13605abf63051228512e0c3f05577fb73d2d75 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 3 May 2019 11:24:00 +0200 Subject: [PATCH 47/92] add 'halo' to requirements --- e2e/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 40fb9efa1..3ce3ac640 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -2,3 +2,4 @@ kubernetes flake8 timeout_decorator docker +halo From 52e53f152fc50a4cf67501544285524e40cc9c02 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 8 May 2019 10:33:42 +0200 Subject: [PATCH 48/92] add Dockerfile for e2e tests --- e2e/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 e2e/Dockerfile diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 000000000..aec611c9f --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.5-slim +MAINTAINER Team ACID @ Zalando + +WORKDIR /e2e + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY tests . \ No newline at end of file From 084167ff35db3b6dbf896663aa5ca45a7f4be1f7 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 9 May 2019 18:09:00 +0200 Subject: [PATCH 49/92] run e2e in a separate Docker image, 1st commit --- e2e/Dockerfile | 17 +++++++++---- ...d-config-kind-smoke-test-postgres-operator | 19 +++++++++++++++ e2e/requirements.txt | 1 - e2e/run.sh | 12 ++++++++-- e2e/tests/test_smoke.py | 24 +++++++++---------- 5 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 e2e/kind-config-kind-smoke-test-postgres-operator diff --git a/e2e/Dockerfile b/e2e/Dockerfile index aec611c9f..6c3011d02 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,9 +1,18 @@ -FROM python:3.5-slim +FROM ubuntu:18.04 MAINTAINER Team ACID @ Zalando WORKDIR /e2e -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +COPY e2e/requirements.txt ./ +COPY e2e/manifests ./manifests +COPY e2e/tests ./tests +COPY e2e/kind-config-kind-smoke-test-postgres-operator /root/.kube/config -COPY tests . \ No newline at end of file +RUN apt-get update +RUN apt-get install -y python3 python3-pip curl \ + && pip3 install --no-cache-dir -r requirements.txt +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +CMD ["python3", "-m", "unittest", "discover", "--start-directory", "tests", "-v"] \ No newline at end of file diff --git a/e2e/kind-config-kind-smoke-test-postgres-operator b/e2e/kind-config-kind-smoke-test-postgres-operator new file mode 100644 index 000000000..e91770a72 --- /dev/null +++ b/e2e/kind-config-kind-smoke-test-postgres-operator @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXdPVEUxTlRVeU1Gb1hEVEk1TURVd05qRTFOVFV5TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWhZCjhTSVNNejlYb3VDMG9HQkRkNmVFdk5QQS9VZVJldHBucFRiUkNNU2tObUNkSFgzVWQ3SC9tTVdJcnVwdVhUeDQKWUV5dmJqWUt3YnJ5bUpTRU5xQ0h2RHE1SlV4TWE0dEZWRFBTNUZIaThiRUVHYW1UVUM3alo0Q2hma2hPN0ZSRQpKVW1xeDBLVDJ5SVVySDZTT203ajc3SDlOL0J5SGhhdnRJNXJsSlp3emdsbGw2eCtsV0RGbEExMndmYVVUaTdlCmpnd1Mzcjl4N0h3R1JQejVHNmhmaGlmSzZMVVhsNXAzYjB0RTZ1QVllNFBlWlNtQmJuS1UwdzAzQlJoYmpmTmIKSEdleUJIYm43RWhWZmJ4Y1dIV2RnaEMzV1dWRDdSa05NTXQvREtPbUdHR3Bab1RCeEpTSXR3UEFyMS9tUGVMRQp5VkxubjJmYzhDZ0dXT01SS0MwQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEb2RRS1Nzd2NJa0NRMFVGUWNJRi9yNmtMQ2YKSjFHL3ZkQ1FkTW1taU9rcWRsMW1tQjAyL2d1UDZtL3AxbUltRmRmWjZJNGlPNGpwUCsvaGllMDB5aUMwMS84MgpLaUE2a292ckpMVzZnQ05rR2Nyb3hhdHRITklEYlF1TkVoZ3RJYWgxK2cxclo4eStTblU4b2pSRVFXYTZJUGJQClJKQjQyZjZ4dFFxZ1BJanRBT2EyeGpuMFdDNGpRTXNiSUx0M2Z0S2RLOTFmNG5BTFBmZGRSdyt6N3NxcWZNeFEKQU9UL3NycVo0eVJqT3d1Q2w4clBsTFVEUzBMTkRuM2FST0VDNVF6dm1nUElKY2k1ZWIrV1JGWEthblpjU2FXMwp6VFlVck9oTStSNEEyUGRBWWFSbm5qdnRGZ3JuY1cwRHVSb2tWUnFHL0Rvd2xHbzdZRUxQNEg4eDVHST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + server: https://172.17.0.3:6443 + name: kind-smoke-test-postgres-operator +contexts: +- context: + cluster: kind-smoke-test-postgres-operator + user: kubernetes-admin + name: kubernetes-admin@kind-smoke-test-postgres-operator +current-context: kubernetes-admin@kind-smoke-test-postgres-operator +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJWTVVRkRiKzc1c1l3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T1RBMU1Ea3hOVFUxTWpCYUZ3MHlNREExTURneE5UVTFNakJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQW5OVkEvMFcrQmxqOENQcjYKZ1EvZUU0aWhMSnh3dlNuSEQ1VE1DYllRSFZFMTN5NmRiclRzTUhyU2duMlZBMmtaSGFEbmpPRXFieG0zMmt5aQowZmJhMzNZVm5BdGwrVmtyVzc2VW1USUFhb1RxaFNkRlpFczlrT3N5WUgweGluSXRMMExrWE9IeWwwVlVSUXVHCm94TVFxU2duaGpSMWJJVjdKbzYvYnhSbU42ZWJwb1FSRHN0QWk5Z2QramlHSU93Q05SWHVhM1B4Nm1DK04rVFoKTnZzRjNieHdIWURwTGo3aU1mNGUvRlM4MjQzVEU4eUo0Ymp1ejNyOHBsR2t4RkU0ZjZJbEtVQVpDK1dKN01KZAp2MDdOMDZVVFg3WlZxekE3R1l6QkROb2UrSmQ0S3JwcDFHbVdIWFgxaU9yRnhHNE9FSlFvbXE0c205VFhha25PCmlSVUQyUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDMHdMSDRjTnBNeHlTQlRiVmlXYzd0Yk8rT2dyRlYzMmltNQpCa3pyMDBrd05nYmw5VFFUNFpkZDlHRnFTRVFpUDNXT2lWNjErMHpNdEpUcEthL0hITnBhNXg5MmhLc2FKZHdpCnErRm5TRisyVytIK0pnTy9tM0l6Z1VQK25kUVNNUXdwWmQ3bWQ0TnE0Wnh6VThrOTYwWWRMYlhHRXdTQTF5Ty8KN3RVQUJWWHVWdFFFaTQvQ1NSTXFjNndLT3NFNUU3QUNiYkU2MXhFTThsUzg5ZVhyaSs0aDZGenBpbkZWa1ZiRwpXaTAyYUV6cVJuTGFqV1NOZThuYnh2ZG1sRGhUZjFxYjkvWGJ0cWxIQWRMc1ZpdU5GT2Jxa3Yzb3FwSW5kQVNBClVRUDMxWXZ4VERPaEpaY3k2dkVyK2tqT2RpZjgrUytJa05BRnhzeVFPK0Vmc05zWUdvdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBbk5WQS8wVytCbGo4Q1ByNmdRL2VFNGloTEp4d3ZTbkhENVRNQ2JZUUhWRTEzeTZkCmJyVHNNSHJTZ24yVkEya1pIYURuak9FcWJ4bTMya3lpMGZiYTMzWVZuQXRsK1Zrclc3NlVtVElBYW9UcWhTZEYKWkVzOWtPc3lZSDB4aW5JdEwwTGtYT0h5bDBWVVJRdUdveE1RcVNnbmhqUjFiSVY3Sm82L2J4Um1ONmVicG9RUgpEc3RBaTlnZCtqaUdJT3dDTlJYdWEzUHg2bUMrTitUWk52c0YzYnh3SFlEcExqN2lNZjRlL0ZTODI0M1RFOHlKCjRianV6M3I4cGxHa3hGRTRmNklsS1VBWkMrV0o3TUpkdjA3TjA2VVRYN1pWcXpBN0dZekJETm9lK0pkNEtycHAKMUdtV0hYWDFpT3JGeEc0T0VKUW9tcTRzbTlUWGFrbk9pUlVEMlFJREFRQUJBb0lCQVFDTTM3R0dteXJab015agpkRzNYeUZ6K3h0ZWZydFpGMUdVT1JlWVJRd3l1aU9nUEZWd3N1UzcvVFJRU1NxT3pjSkF5NFBtY3ZoVFR2eEk2CmNHUkFuYkIwMFNrUUJkMFBZVjFsQjRlTEpETGplNGo5R2cxbXpYNzcwWWhxeTRuWWhqNjRHU252bExYSDAycWkKcW52QnQ3cGJkOG9vN3E0YlVMc1NJMThwYy9WdFB5U1lmd3h3OVpFTEdRb3ZNNklRTGNBb1RxdHdKYXZDZmlKQQpKYjZFaXdROUljOTB1MDJxaGE3RGw4K3lrWmRaSnJmNmYwditoZDUrdGFRK3RkNnZMd3RvUGZCa2licXhBNkx1CkxId2IvWnBSOGlKbXJFTjkwdFNMTFpMWVAzQkhkYUNNcXdJYllXUkRsS2p2cE1DUUp2NDgrdDhuMERNcUVzSFUKQi9XK2EwdEJBb0dCQU02b3VHTmhUMWpkTTlSenBITzg3enpiaHh2T0VWQ3BDN29zdGl1dmJnUnVGcFl2WloxTQpzTHY5dVBrTFlmanAxais3VzJvWncvQ25Zcjh0ZGVYTHVBKzNpcXk5bzZlblhjVEExM0wwTFNHY0tuZXAwenpxCmtqUmNGYmZEOXNpM2Jsc0FmT0hnSzY5MFByZmdINjF1bW5rYTVjWEhSc3dTcThBbVBHVU5EQWxsQW9HQkFNSkgKR1Z3WWhlN2hlbE5FQ1FNTWxENW90c01zWGJtWEFzbUFCZ2dxQXhRVkw0dmVUTk9XZHV2b2t1eDFVVWZhMVhqdgpUNEduYTZmeCtvMDUyRlNkU1FLTkEwNlhwVWhxZUNIcHU1ZmV3MHV0WVJhblhrS2MzRnZ5NEJlbEd3aHZUUVJRCnIxVmdlWWc0V3A2bTFuaHJwRDEydGh2WDE2Rm9uN0I4Skp2clVxTmxBb0dCQU1vcVk3ZFV5cnEwS3EvN01UWEgKN29JcWY5SERsVXpERXFYZWQ1Zmsxa3VmSnBsbFpKS3RJM2ZFamQrVU14TytMY25MRDNLTUloS2FyUTg0K2MwRApyZHd5UVljYlBhNFZITFlOc0xiVUNCS0pJMEpNOEVqM2NHK29aZGFQN2l3TXhmaGdVY3JsOGRhQ2NaaVB1RzJCCmRieGpnOFFuWGlybFdQOXdhRVN5cnNQQkFvR0FmRkxwYktFWTNHU1lWajZja2NIei8vZ2N0TXRvY3dLck91MWQKYnM0THlFZENkUHhlSjYwTER5NTNEekNJUWpaTkU2WDVPQncrYld3UmpWeXVEbi9Vbi9oRFhJRDR1VjNBNE5ybApQR3ZHaUdBOFdEWGt3VFlHWWlVTHVMWGtsY0k4QS8zcUpmV2w4RUUzNUgwWmxGZzE4MHRMZ0lmZ3FwNzhTZ0UzCm9EdTRWMjBDZ1lCNER0MDNoNG4zWGxPMW8rZlBKMGJETnZnb1JGZnZYYWk2YjBVa1VpcXNKclRTZVFrRVEzenUKNnFPRnBLNUgzRDlWc3lweWF0N3JJNmk4bFQzbHlsaUU1dGdzMEV5ektvSjJCYVFEOE5MRVJueHVyRHFkV2paaQp6NXNJSjU0c0N6NDJjTytQSkVTUjRCTlBZU1ZNUEhDZURWd1diL1QwNXdOU1NxemIrRWJQN1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 3ce3ac640..40fb9efa1 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -2,4 +2,3 @@ kubernetes flake8 timeout_decorator docker -halo diff --git a/e2e/run.sh b/e2e/run.sh index 6d612a0cd..0471afd8a 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -21,5 +21,13 @@ kubectl cluster-info image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) kind load docker-image ${image} --name ${cluster_name} -python3 -m unittest discover --start-directory e2e/tests/ -v && -kind delete cluster --name ${cluster_name} \ No newline at end of file +cp -r ./manifests ./e2e/manifests +cp $KUBECONFIG ./e2e +d=$(docker inspect -f "{{ .NetworkSettings.IPAddress }}:6443" ${cluster_name}-control-plane) +sed -i "s/server.*$/server: https:\/\/$d/g" e2e/kind-config-${cluster_name} + +docker build --tag=postgres-operator-e2e-tests -f e2e/Dockerfile . && docker run postgres-operator-e2e-tests +#python3 -m unittest discover --start-directory e2e/tests/ -v && + +kind delete cluster --name ${cluster_name} +rm -rf ./e2e/manifests ./e2e/kind-smoke-test-postgres-operator.yaml diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index bb3b5f5b0..63b6b7376 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -4,7 +4,7 @@ import subprocess import warnings import docker -from halo import Halo +#from halo import Halo from kubernetes import client, config, utils @@ -42,8 +42,8 @@ def setUpClass(cls): # submit the most recent operator image built locally # HACK assumes "images list" returns the most recent image at index 0 - docker_client = docker.from_env() - image = docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] + # docker_client = docker.from_env() + image = "registry.opensource.zalan.do/acid/postgres-operator:v1.1.0-67-g52e53f1-dirty" # docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] body = { "spec": { "template": { @@ -114,12 +114,12 @@ class Utils: @staticmethod def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' - with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'): - while pod_phase != 'Running': - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items - if pods: - pod_phase = pods[0].status.phase - time.sleep(retry_timeout_sec) + #with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'): + while pod_phase != 'Running': + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase + time.sleep(retry_timeout_sec) @staticmethod def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): @@ -133,9 +133,9 @@ def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' - with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'): - while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: - time.sleep(retry_timeout_sec) + #with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'): + while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: + time.sleep(retry_timeout_sec) @staticmethod def count_pods_with_label(k8s_api, labels): From 407b2cab41e67b1e1d0cdfe269299804d7a86b6b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 10 May 2019 17:00:16 +0200 Subject: [PATCH 50/92] move tests to a Docker container --- .travis.yml | 3 ++- Makefile | 10 ++++---- delivery.yaml | 5 +--- docs/developer.md | 17 +++---------- e2e/Dockerfile | 18 ++++++------- ...-cluster-postgres-operator-e2e-tests.yaml} | 0 ...d-config-kind-smoke-test-postgres-operator | 19 -------------- e2e/requirements.txt | 6 ++--- e2e/run.sh | 25 +++++++++++-------- e2e/tests/test_smoke.py | 12 +++------ 10 files changed, 39 insertions(+), 76 deletions(-) rename e2e/{kind-config-smoke-tests.yaml => kind-cluster-postgres-operator-e2e-tests.yaml} (100%) delete mode 100644 e2e/kind-config-kind-smoke-test-postgres-operator diff --git a/.travis.yml b/.travis.yml index 0fd48a9ca..7e77ffb41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,9 @@ before_install: - go get github.com/mattn/goveralls install: - - make deps + - make deps e2e-tools e2e-build script: - hack/verify-codegen.sh - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v + - make e2e-run diff --git a/Makefile b/Makefile index 378aa0082..fe7a5af00 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean local test linux macos docker push scm-source.json e2e e2e-tools +.PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build BINARY ?= postgres-operator BUILD_FLAGS ?= -v @@ -92,11 +92,11 @@ test: hack/verify-codegen.sh @go test ./... +e2e-build: + docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile . + e2e-tools: - @pip3 install -r e2e/requirements.txt @go get -u sigs.k8s.io/kind - # assumes kubectl is already isntalled -e2e: docker +e2e-run: docker e2e/run.sh - flake8 --exit-zero diff --git a/delivery.yaml b/delivery.yaml index cd221229d..ee0306d86 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,10 +46,7 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator echo "INFO install kubectl to test 'kind' from client side" - (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && - chmod +x ./kubectl && - mv ./kubectl /usr/local/bin/kubectl) || exit 1 - make e2e-tools e2e + make e2e-tools e2e-build e2e-run - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/docs/developer.md b/docs/developer.md index efdd1df17..561b437e6 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -317,20 +317,11 @@ kubectl logs acid-minimal-cluster-0 ## End-to-end tests -The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. -Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. -We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory; run `make e2e-tools` to install `kind` and Python dependencies before the first run. Each e2e execution tests a Postgres operator image built from the current git branch. - -### Smoke tests - -The first provided collection of [smoke tests](../e2e/tests/test_smoke.py) covers the most basic operator capabilities. These tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The `kind-smoke-test-postgres-operator` cluster is deleted if tests complete successfully. - -### Contributing tests suites - -1. Consider using a separate `kind` cluster per a logically separate collection of tests. This approach enables simple clean up after test success by deleting the relevant cluster. -2. Make tests time out; for now we use the [timeout-decorator](https://github.com/pnpnpn/timeout-decorator) for that purpose. -3. Please briefly describe the use case for the new test collection here and in the comments. +The operator provides reference e2e (end-to-end) tests to ensure various infra parts work smoothly together. +Each e2e execution tests a Postgres operator image built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/) (local k8s) cluster and Docker container with tests. The k8s API client from within the container connects to the `kind` cluster using the standard Docker `bridge` network. +The tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The kind cluster is deleted if tests complete successfully. +End-to-end tests are executed automatically during builds; to invoke them locally use `make e2e-run` from the project's top directory. Run `make e2e-tools e2e-build` to install `kind` and build the tests' image locally before the first run. ## Introduce additional configuration parameters diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 6c3011d02..e8b0c3303 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,18 +1,16 @@ FROM ubuntu:18.04 -MAINTAINER Team ACID @ Zalando +LABEL maintainer="Team ACID @ Zalando " WORKDIR /e2e -COPY e2e/requirements.txt ./ -COPY e2e/manifests ./manifests -COPY e2e/tests ./tests -COPY e2e/kind-config-kind-smoke-test-postgres-operator /root/.kube/config +COPY manifests ./manifests +COPY e2e/requirements.txt e2e/tests ./ -RUN apt-get update -RUN apt-get install -y python3 python3-pip curl \ - && pip3 install --no-cache-dir -r requirements.txt -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ +RUN apt-get update \ + && apt-get install -y python3 python3-pip curl \ + && pip3 install --no-cache-dir -r requirements.txt \ + && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl -CMD ["python3", "-m", "unittest", "discover", "--start-directory", "tests", "-v"] \ No newline at end of file +CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] \ No newline at end of file diff --git a/e2e/kind-config-smoke-tests.yaml b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml similarity index 100% rename from e2e/kind-config-smoke-tests.yaml rename to e2e/kind-cluster-postgres-operator-e2e-tests.yaml diff --git a/e2e/kind-config-kind-smoke-test-postgres-operator b/e2e/kind-config-kind-smoke-test-postgres-operator deleted file mode 100644 index e91770a72..000000000 --- a/e2e/kind-config-kind-smoke-test-postgres-operator +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXdPVEUxTlRVeU1Gb1hEVEk1TURVd05qRTFOVFV5TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWhZCjhTSVNNejlYb3VDMG9HQkRkNmVFdk5QQS9VZVJldHBucFRiUkNNU2tObUNkSFgzVWQ3SC9tTVdJcnVwdVhUeDQKWUV5dmJqWUt3YnJ5bUpTRU5xQ0h2RHE1SlV4TWE0dEZWRFBTNUZIaThiRUVHYW1UVUM3alo0Q2hma2hPN0ZSRQpKVW1xeDBLVDJ5SVVySDZTT203ajc3SDlOL0J5SGhhdnRJNXJsSlp3emdsbGw2eCtsV0RGbEExMndmYVVUaTdlCmpnd1Mzcjl4N0h3R1JQejVHNmhmaGlmSzZMVVhsNXAzYjB0RTZ1QVllNFBlWlNtQmJuS1UwdzAzQlJoYmpmTmIKSEdleUJIYm43RWhWZmJ4Y1dIV2RnaEMzV1dWRDdSa05NTXQvREtPbUdHR3Bab1RCeEpTSXR3UEFyMS9tUGVMRQp5VkxubjJmYzhDZ0dXT01SS0MwQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEb2RRS1Nzd2NJa0NRMFVGUWNJRi9yNmtMQ2YKSjFHL3ZkQ1FkTW1taU9rcWRsMW1tQjAyL2d1UDZtL3AxbUltRmRmWjZJNGlPNGpwUCsvaGllMDB5aUMwMS84MgpLaUE2a292ckpMVzZnQ05rR2Nyb3hhdHRITklEYlF1TkVoZ3RJYWgxK2cxclo4eStTblU4b2pSRVFXYTZJUGJQClJKQjQyZjZ4dFFxZ1BJanRBT2EyeGpuMFdDNGpRTXNiSUx0M2Z0S2RLOTFmNG5BTFBmZGRSdyt6N3NxcWZNeFEKQU9UL3NycVo0eVJqT3d1Q2w4clBsTFVEUzBMTkRuM2FST0VDNVF6dm1nUElKY2k1ZWIrV1JGWEthblpjU2FXMwp6VFlVck9oTStSNEEyUGRBWWFSbm5qdnRGZ3JuY1cwRHVSb2tWUnFHL0Rvd2xHbzdZRUxQNEg4eDVHST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - server: https://172.17.0.3:6443 - name: kind-smoke-test-postgres-operator -contexts: -- context: - cluster: kind-smoke-test-postgres-operator - user: kubernetes-admin - name: kubernetes-admin@kind-smoke-test-postgres-operator -current-context: kubernetes-admin@kind-smoke-test-postgres-operator -kind: Config -preferences: {} -users: -- name: kubernetes-admin - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJWTVVRkRiKzc1c1l3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T1RBMU1Ea3hOVFUxTWpCYUZ3MHlNREExTURneE5UVTFNakJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQW5OVkEvMFcrQmxqOENQcjYKZ1EvZUU0aWhMSnh3dlNuSEQ1VE1DYllRSFZFMTN5NmRiclRzTUhyU2duMlZBMmtaSGFEbmpPRXFieG0zMmt5aQowZmJhMzNZVm5BdGwrVmtyVzc2VW1USUFhb1RxaFNkRlpFczlrT3N5WUgweGluSXRMMExrWE9IeWwwVlVSUXVHCm94TVFxU2duaGpSMWJJVjdKbzYvYnhSbU42ZWJwb1FSRHN0QWk5Z2QramlHSU93Q05SWHVhM1B4Nm1DK04rVFoKTnZzRjNieHdIWURwTGo3aU1mNGUvRlM4MjQzVEU4eUo0Ymp1ejNyOHBsR2t4RkU0ZjZJbEtVQVpDK1dKN01KZAp2MDdOMDZVVFg3WlZxekE3R1l6QkROb2UrSmQ0S3JwcDFHbVdIWFgxaU9yRnhHNE9FSlFvbXE0c205VFhha25PCmlSVUQyUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDMHdMSDRjTnBNeHlTQlRiVmlXYzd0Yk8rT2dyRlYzMmltNQpCa3pyMDBrd05nYmw5VFFUNFpkZDlHRnFTRVFpUDNXT2lWNjErMHpNdEpUcEthL0hITnBhNXg5MmhLc2FKZHdpCnErRm5TRisyVytIK0pnTy9tM0l6Z1VQK25kUVNNUXdwWmQ3bWQ0TnE0Wnh6VThrOTYwWWRMYlhHRXdTQTF5Ty8KN3RVQUJWWHVWdFFFaTQvQ1NSTXFjNndLT3NFNUU3QUNiYkU2MXhFTThsUzg5ZVhyaSs0aDZGenBpbkZWa1ZiRwpXaTAyYUV6cVJuTGFqV1NOZThuYnh2ZG1sRGhUZjFxYjkvWGJ0cWxIQWRMc1ZpdU5GT2Jxa3Yzb3FwSW5kQVNBClVRUDMxWXZ4VERPaEpaY3k2dkVyK2tqT2RpZjgrUytJa05BRnhzeVFPK0Vmc05zWUdvdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBbk5WQS8wVytCbGo4Q1ByNmdRL2VFNGloTEp4d3ZTbkhENVRNQ2JZUUhWRTEzeTZkCmJyVHNNSHJTZ24yVkEya1pIYURuak9FcWJ4bTMya3lpMGZiYTMzWVZuQXRsK1Zrclc3NlVtVElBYW9UcWhTZEYKWkVzOWtPc3lZSDB4aW5JdEwwTGtYT0h5bDBWVVJRdUdveE1RcVNnbmhqUjFiSVY3Sm82L2J4Um1ONmVicG9RUgpEc3RBaTlnZCtqaUdJT3dDTlJYdWEzUHg2bUMrTitUWk52c0YzYnh3SFlEcExqN2lNZjRlL0ZTODI0M1RFOHlKCjRianV6M3I4cGxHa3hGRTRmNklsS1VBWkMrV0o3TUpkdjA3TjA2VVRYN1pWcXpBN0dZekJETm9lK0pkNEtycHAKMUdtV0hYWDFpT3JGeEc0T0VKUW9tcTRzbTlUWGFrbk9pUlVEMlFJREFRQUJBb0lCQVFDTTM3R0dteXJab015agpkRzNYeUZ6K3h0ZWZydFpGMUdVT1JlWVJRd3l1aU9nUEZWd3N1UzcvVFJRU1NxT3pjSkF5NFBtY3ZoVFR2eEk2CmNHUkFuYkIwMFNrUUJkMFBZVjFsQjRlTEpETGplNGo5R2cxbXpYNzcwWWhxeTRuWWhqNjRHU252bExYSDAycWkKcW52QnQ3cGJkOG9vN3E0YlVMc1NJMThwYy9WdFB5U1lmd3h3OVpFTEdRb3ZNNklRTGNBb1RxdHdKYXZDZmlKQQpKYjZFaXdROUljOTB1MDJxaGE3RGw4K3lrWmRaSnJmNmYwditoZDUrdGFRK3RkNnZMd3RvUGZCa2licXhBNkx1CkxId2IvWnBSOGlKbXJFTjkwdFNMTFpMWVAzQkhkYUNNcXdJYllXUkRsS2p2cE1DUUp2NDgrdDhuMERNcUVzSFUKQi9XK2EwdEJBb0dCQU02b3VHTmhUMWpkTTlSenBITzg3enpiaHh2T0VWQ3BDN29zdGl1dmJnUnVGcFl2WloxTQpzTHY5dVBrTFlmanAxais3VzJvWncvQ25Zcjh0ZGVYTHVBKzNpcXk5bzZlblhjVEExM0wwTFNHY0tuZXAwenpxCmtqUmNGYmZEOXNpM2Jsc0FmT0hnSzY5MFByZmdINjF1bW5rYTVjWEhSc3dTcThBbVBHVU5EQWxsQW9HQkFNSkgKR1Z3WWhlN2hlbE5FQ1FNTWxENW90c01zWGJtWEFzbUFCZ2dxQXhRVkw0dmVUTk9XZHV2b2t1eDFVVWZhMVhqdgpUNEduYTZmeCtvMDUyRlNkU1FLTkEwNlhwVWhxZUNIcHU1ZmV3MHV0WVJhblhrS2MzRnZ5NEJlbEd3aHZUUVJRCnIxVmdlWWc0V3A2bTFuaHJwRDEydGh2WDE2Rm9uN0I4Skp2clVxTmxBb0dCQU1vcVk3ZFV5cnEwS3EvN01UWEgKN29JcWY5SERsVXpERXFYZWQ1Zmsxa3VmSnBsbFpKS3RJM2ZFamQrVU14TytMY25MRDNLTUloS2FyUTg0K2MwRApyZHd5UVljYlBhNFZITFlOc0xiVUNCS0pJMEpNOEVqM2NHK29aZGFQN2l3TXhmaGdVY3JsOGRhQ2NaaVB1RzJCCmRieGpnOFFuWGlybFdQOXdhRVN5cnNQQkFvR0FmRkxwYktFWTNHU1lWajZja2NIei8vZ2N0TXRvY3dLck91MWQKYnM0THlFZENkUHhlSjYwTER5NTNEekNJUWpaTkU2WDVPQncrYld3UmpWeXVEbi9Vbi9oRFhJRDR1VjNBNE5ybApQR3ZHaUdBOFdEWGt3VFlHWWlVTHVMWGtsY0k4QS8zcUpmV2w4RUUzNUgwWmxGZzE4MHRMZ0lmZ3FwNzhTZ0UzCm9EdTRWMjBDZ1lCNER0MDNoNG4zWGxPMW8rZlBKMGJETnZnb1JGZnZYYWk2YjBVa1VpcXNKclRTZVFrRVEzenUKNnFPRnBLNUgzRDlWc3lweWF0N3JJNmk4bFQzbHlsaUU1dGdzMEV5ektvSjJCYVFEOE5MRVJueHVyRHFkV2paaQp6NXNJSjU0c0N6NDJjTytQSkVTUjRCTlBZU1ZNUEhDZURWd1diL1QwNXdOU1NxemIrRWJQN1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 40fb9efa1..c46b334ad 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,4 +1,2 @@ -kubernetes -flake8 -timeout_decorator -docker +kubernetes==9.0.0 +timeout_decorator==0.4.1 diff --git a/e2e/run.sh b/e2e/run.sh index 0471afd8a..e78a0b5bb 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,7 +6,11 @@ set -o nounset set -o pipefail IFS=$'\n\t' -readonly cluster_name="kind-smoke-test-postgres-operator" +readonly cluster_name="postgres-operator-e2e-tests" +readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) +readonly e2e_test_image=${cluster_name} +readonly kind_api_server_port=6443 # well-known in the 'kind' codebase +readonly kubeconfig_path="./e2e/kind-config-${cluster_name}" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] @@ -14,20 +18,19 @@ then kind delete cluster --name ${cluster_name} fi -kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-tests.yaml +kind create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" -kubectl cluster-info -image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) -kind load docker-image ${image} --name ${cluster_name} +kind load docker-image ${operator_image} --name ${cluster_name} +kubectl cluster-info -cp -r ./manifests ./e2e/manifests +# use the actual kubeconfig to connect to the 'kind' API server +# but update the IP address of the API server to the one from the Docker 'bridge' network cp $KUBECONFIG ./e2e -d=$(docker inspect -f "{{ .NetworkSettings.IPAddress }}:6443" ${cluster_name}-control-plane) -sed -i "s/server.*$/server: https:\/\/$d/g" e2e/kind-config-${cluster_name} +kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) +sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} -docker build --tag=postgres-operator-e2e-tests -f e2e/Dockerfile . && docker run postgres-operator-e2e-tests -#python3 -m unittest discover --start-directory e2e/tests/ -v && +docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} kind delete cluster --name ${cluster_name} -rm -rf ./e2e/manifests ./e2e/kind-smoke-test-postgres-operator.yaml +rm -rf ${kubeconfig_path} diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 63b6b7376..72998a00a 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -3,8 +3,7 @@ import timeout_decorator import subprocess import warnings -import docker -#from halo import Halo +import os from kubernetes import client, config, utils @@ -40,10 +39,7 @@ def setUpClass(cls): for filename in ["configmap.yaml", "postgres-operator.yaml"]: utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) - # submit the most recent operator image built locally - # HACK assumes "images list" returns the most recent image at index 0 - # docker_client = docker.from_env() - image = "registry.opensource.zalan.do/acid/postgres-operator:v1.1.0-67-g52e53f1-dirty" # docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] + # submit the most recent operator image built on the Docker host body = { "spec": { "template": { @@ -51,7 +47,7 @@ def setUpClass(cls): "containers": [ { "name": "postgres-operator", - "image": image + "image": os.environ['OPERATOR_IMAGE'] } ] } @@ -114,7 +110,6 @@ class Utils: @staticmethod def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' - #with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'): while pod_phase != 'Running': pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: @@ -133,7 +128,6 @@ def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' - #with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'): while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: time.sleep(retry_timeout_sec) From e1befadedda2df1b45a6b8bcb5affa6cc91b1470 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 11:32:03 +0200 Subject: [PATCH 51/92] ping kind version for CDP build --- delivery.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index ee0306d86..ef8e6b35e 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,7 +46,11 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator echo "INFO install kubectl to test 'kind' from client side" - make e2e-tools e2e-build e2e-run + make e2e-build e2e-run + wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 + chmod +x kind-linux-amd64 + mv kind-linux-amd64 /tmp/kind + export PATH=$PATH:/tmp - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin From eeaa1a132cf9bb47cefdd951511f8eadaf28b74e Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 11:39:34 +0200 Subject: [PATCH 52/92] install kind before run --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index ef8e6b35e..53edbf7dc 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,11 +46,11 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator echo "INFO install kubectl to test 'kind' from client side" - make e2e-build e2e-run wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 chmod +x kind-linux-amd64 mv kind-linux-amd64 /tmp/kind export PATH=$PATH:/tmp + make e2e-build e2e-run - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin From e98510709092bfd69dbf5a3fd6a6b82399695dab Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 11:49:08 +0200 Subject: [PATCH 53/92] remove kubectl mentions --- delivery.yaml | 1 - e2e/run.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 53edbf7dc..6dd6faec8 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -45,7 +45,6 @@ pipeline: cmd: | export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator - echo "INFO install kubectl to test 'kind' from client side" wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 chmod +x kind-linux-amd64 mv kind-linux-amd64 /tmp/kind diff --git a/e2e/run.sh b/e2e/run.sh index e78a0b5bb..2c0c3d295 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -22,7 +22,6 @@ kind create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres- export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" kind load docker-image ${operator_image} --name ${cluster_name} -kubectl cluster-info # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network From 66a6d81bc0a4e6d7e365814a9f015ef69cc146ca Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 12:23:39 +0200 Subject: [PATCH 54/92] install pinned kind version to well-known location --- Makefile | 4 +++- delivery.yaml | 7 +------ e2e/run.sh | 2 ++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fe7a5af00..9f487d3b8 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,9 @@ e2e-build: docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile . e2e-tools: - @go get -u sigs.k8s.io/kind + wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 + chmod +x kind-linux-amd64 + mv kind-linux-amd64 /tmp/kind e2e-run: docker e2e/run.sh diff --git a/delivery.yaml b/delivery.yaml index 6dd6faec8..a60a656b1 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -43,13 +43,8 @@ pipeline: go test ./... - desc: 'Run e2e tests' cmd: | - export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator - wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 - chmod +x kind-linux-amd64 - mv kind-linux-amd64 /tmp/kind - export PATH=$PATH:/tmp - make e2e-build e2e-run + make e2e-tools e2e-build e2e-run - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/e2e/run.sh b/e2e/run.sh index 2c0c3d295..dc6a18b49 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,6 +6,8 @@ set -o nounset set -o pipefail IFS=$'\n\t' +export PATH=$PATH:/tmp/kind + readonly cluster_name="postgres-operator-e2e-tests" readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) readonly e2e_test_image=${cluster_name} From 82bf0d90edf8b88a8c5644846b9e939fb5c29453 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 12:34:49 +0200 Subject: [PATCH 55/92] debug builds --- e2e/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/run.sh b/e2e/run.sh index dc6a18b49..7d3902cb1 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -7,6 +7,8 @@ set -o pipefail IFS=$'\n\t' export PATH=$PATH:/tmp/kind +echo $PATH +ls -al /tmp/kind readonly cluster_name="postgres-operator-e2e-tests" readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) From b8e5567e9101e0f2077617c8ca40dc4cfade55a3 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 12:57:25 +0200 Subject: [PATCH 56/92] install Kind to GOENV/bin --- Makefile | 4 ++-- e2e/run.sh | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 9f487d3b8..0ff427182 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ ifdef CDP_PULL_REQUEST_NUMBER CDP_TAG := -${CDP_BUILD_VERSION} endif - +KIND_PATH := $(GOPATH)/bin PATH := $(GOPATH)/bin:$(PATH) SHELL := env PATH=$(PATH) $(SHELL) @@ -98,7 +98,7 @@ e2e-build: e2e-tools: wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 chmod +x kind-linux-amd64 - mv kind-linux-amd64 /tmp/kind + mv kind-linux-amd64 $(KIND_PATH) e2e-run: docker e2e/run.sh diff --git a/e2e/run.sh b/e2e/run.sh index 7d3902cb1..48c7697bf 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,16 +6,13 @@ set -o nounset set -o pipefail IFS=$'\n\t' -export PATH=$PATH:/tmp/kind -echo $PATH -ls -al /tmp/kind - readonly cluster_name="postgres-operator-e2e-tests" readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) readonly e2e_test_image=${cluster_name} readonly kind_api_server_port=6443 # well-known in the 'kind' codebase readonly kubeconfig_path="./e2e/kind-config-${cluster_name}" +alias kind="${GOPATH}/bin/kind-linux-amd64" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then @@ -36,4 +33,5 @@ sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} kind delete cluster --name ${cluster_name} +unalias kind rm -rf ${kubeconfig_path} From 0ec5a2c8b629117bfc4a83cab7022db6766e7653 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 13:08:42 +0200 Subject: [PATCH 57/92] use full binary name --- e2e/run.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index 48c7697bf..932cc2b7d 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -12,17 +12,16 @@ readonly e2e_test_image=${cluster_name} readonly kind_api_server_port=6443 # well-known in the 'kind' codebase readonly kubeconfig_path="./e2e/kind-config-${cluster_name}" -alias kind="${GOPATH}/bin/kind-linux-amd64" # avoid interference with previous test runs -if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] +if [[ $(kind-linux-amd64 get clusters | grep "^${cluster_name}*") != "" ]] then - kind delete cluster --name ${cluster_name} + kind-linux-amd64 delete cluster --name ${cluster_name} fi -kind create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml -export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" +kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml +export KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" -kind load docker-image ${operator_image} --name ${cluster_name} +kind-linux-amd64 load docker-image ${operator_image} --name ${cluster_name} # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network @@ -32,6 +31,5 @@ sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} -kind delete cluster --name ${cluster_name} -unalias kind +kind-linux-amd64 delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} From c7106dcc32cae3d11660fbfdf6c1f10184255f47 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 13 May 2019 13:24:27 +0200 Subject: [PATCH 58/92] substitute realname with readlink --- e2e/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/run.sh b/e2e/run.sh index 932cc2b7d..4e0d50a7b 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -29,7 +29,7 @@ cp $KUBECONFIG ./e2e kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} -docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} +docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} kind-linux-amd64 delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} From c3f5200d4b5dc0de84413f82c30f8075c4c8519d Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 14 May 2019 13:20:17 +0200 Subject: [PATCH 59/92] minor improvements in Dockerfile --- e2e/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/Dockerfile b/e2e/Dockerfile index e8b0c3303..3d569b2fb 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -7,10 +7,12 @@ COPY manifests ./manifests COPY e2e/requirements.txt e2e/tests ./ RUN apt-get update \ - && apt-get install -y python3 python3-pip curl \ + && apt-get install --no-install-recommends -y python3 python3-setuptools python3-pip curl \ && pip3 install --no-cache-dir -r requirements.txt \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ - && mv ./kubectl /usr/local/bin/kubectl + && mv ./kubectl /usr/local/bin/kubectl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] \ No newline at end of file From 9b05215b28cda3d7304f4039dcf7fccfca7799e1 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 14 May 2019 14:10:06 +0200 Subject: [PATCH 60/92] split run.sh into functions for readability --- Makefile | 2 ++ e2e/run.sh | 55 ++++++++++++++++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 0ff427182..643158061 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,8 @@ e2e-build: docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile . e2e-tools: + # install pinned version of 'kind' + # leave the name as is to avoid overwriting official binary named `kind` wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 chmod +x kind-linux-amd64 mv kind-linux-amd64 $(KIND_PATH) diff --git a/e2e/run.sh b/e2e/run.sh index 4e0d50a7b..a711a467f 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -9,27 +9,46 @@ IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) readonly e2e_test_image=${cluster_name} -readonly kind_api_server_port=6443 # well-known in the 'kind' codebase -readonly kubeconfig_path="./e2e/kind-config-${cluster_name}" +readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" -# avoid interference with previous test runs -if [[ $(kind-linux-amd64 get clusters | grep "^${cluster_name}*") != "" ]] -then - kind-linux-amd64 delete cluster --name ${cluster_name} -fi -kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml -export KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" +function start_kind(){ + + # avoid interference with previous test runs + if [[ $(kind-linux-amd64 get clusters | grep "^${cluster_name}*") != "" ]] + then + kind-linux-amd64 delete cluster --name ${cluster_name} + fi -kind-linux-amd64 load docker-image ${operator_image} --name ${cluster_name} + kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml + kind-linux-amd64 load docker-image ${operator_image} --name ${cluster_name} + export KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" +} -# use the actual kubeconfig to connect to the 'kind' API server -# but update the IP address of the API server to the one from the Docker 'bridge' network -cp $KUBECONFIG ./e2e -kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) -sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} +function set_kind_api_server_ip(){ + # use the actual kubeconfig to connect to the 'kind' API server + # but update the IP address of the API server to the one from the Docker 'bridge' network + cp ${KUBECONFIG} /tmp + readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase + readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) + sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} +} -docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} +function run_tests(){ + docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} +} -kind-linux-amd64 delete cluster --name ${cluster_name} -rm -rf ${kubeconfig_path} +function clean_up(){ + kind-linux-amd64 delete cluster --name ${cluster_name} + rm -rf ${kubeconfig_path} +} + +function main(){ + start_kind + set_kind_api_server_ip + run_tests + clean_up + exit 0 +} + +main "$@" From a00f0476ea0a05fa7e9f6899a6b1b58f037ff5b4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 20 May 2019 14:14:32 +0200 Subject: [PATCH 61/92] add test for taint-based eviction --- e2e/tests/test_smoke.py | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 72998a00a..e8451b1f2 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -90,6 +90,46 @@ def test_scaling(self): Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): + """ + Add taint "postgres=:NoExecute" to node with master. + """ + k8s = K8sApi() + labels = 'version=acid-minimal-cluster' + master_pod_node = '' + new_master_pod_node = '' + + body = { + "spec": { + "taints": [ + { + "effect": "NoExecute", + "key": "postgres" + } + ] + } + } + podsList = k8s.core_v1.list_namespaced_pod("default", label_selector=labels) + for pod in podsList.items: + if ('spilo-role', 'master') in pod.metadata.labels.items(): + master_pod_node = pod.spec.node_name + elif ('spilo-role', 'replica') in pod.metadata.labels.items(): + new_master_pod_node = pod.spec.node_name + + k8s.core_v1.patch_node(master_pod_node, body) + Utils.wait_for_master_failover(k8s, new_master_pod_node, self.RETRY_TIMEOUT_SEC) + + self.assertTrue(master_pod_node != new_master_pod_node, "Master on {} did not fail over to {}".format(master_pod_node, new_master_pod_node)) + + # undo the tainting + body = { + "spec": { + "taints": [] + } + } + k8s.core_v1.patch_node(new_master_pod_node, body) + class K8sApi: @@ -135,6 +175,17 @@ def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): def count_pods_with_label(k8s_api, labels): return len(k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items) + @staticmethod + def wait_for_master_failover(k8s_api, expected_master_pod_node, retry_timeout_sec): + pod_phase = 'Failing over' + new_master_pod_node = '' + while (pod_phase != 'Running') & (new_master_pod_node != expected_master_pod_node): + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items + if pods: + new_master_pod_node = pods[0].spec.node_name + pod_phase = pods[0].status.phase + time.sleep(retry_timeout_sec) + if __name__ == '__main__': unittest.main() From 461beec310bb77059cebd1c46240ec326ffbd5a5 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 20 May 2019 14:52:15 +0200 Subject: [PATCH 62/92] additional check plus comments --- e2e/tests/test_smoke.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index e8451b1f2..ab7a0289f 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -100,6 +100,7 @@ def test_taint_based_eviction(self): master_pod_node = '' new_master_pod_node = '' + # taint node with postgres=:NoExecute to force failover body = { "spec": { "taints": [ @@ -110,6 +111,8 @@ def test_taint_based_eviction(self): ] } } + + # get nodes of master and replica podsList = k8s.core_v1.list_namespaced_pod("default", label_selector=labels) for pod in podsList.items: if ('spilo-role', 'master') in pod.metadata.labels.items(): @@ -117,6 +120,15 @@ def test_taint_based_eviction(self): elif ('spilo-role', 'replica') in pod.metadata.labels.items(): new_master_pod_node = pod.spec.node_name + # if both live on the same node, failover will happen to the other kind worker + if master_pod_node == new_master_pod_node: + nodes = k8s.core_v1.list_node() + for n in nodes.items: + if "node-role.kubernetes.io/master" not in n.metadata.labels & n.metadata.name != master_pod_node: + new_master_pod_node = n.metadata.name + break + + # patch node and test if master is failing over to the expected node k8s.core_v1.patch_node(master_pod_node, body) Utils.wait_for_master_failover(k8s, new_master_pod_node, self.RETRY_TIMEOUT_SEC) From f677714dabcb8c2022490e8e6f2d371fbc47f7ad Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 20 May 2019 15:57:11 +0200 Subject: [PATCH 63/92] consider setups with more replicas and kind workers --- e2e/tests/test_smoke.py | 63 +++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index ab7a0289f..82576fd09 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -97,8 +97,18 @@ def test_taint_based_eviction(self): """ k8s = K8sApi() labels = 'version=acid-minimal-cluster' - master_pod_node = '' - new_master_pod_node = '' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, failover_targets = get_spilo_nodes(k8s, labels) + num_replicas = len(failover_targets) + + # if all pods live on the same node, failover will happen to other worker(s) + failover_targets = [x for x in failover_targets if x != current_master_node] + if len(failover_targets) == 0: + nodes = k8s.core_v1.list_node() + for n in nodes.items: + if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != current_master_node: + failover_targets.append(n.metadata.name) # taint node with postgres=:NoExecute to force failover body = { @@ -112,27 +122,13 @@ def test_taint_based_eviction(self): } } - # get nodes of master and replica - podsList = k8s.core_v1.list_namespaced_pod("default", label_selector=labels) - for pod in podsList.items: - if ('spilo-role', 'master') in pod.metadata.labels.items(): - master_pod_node = pod.spec.node_name - elif ('spilo-role', 'replica') in pod.metadata.labels.items(): - new_master_pod_node = pod.spec.node_name + # patch node and test if master is failing over to one of the expected nodes + k8s.core_v1.patch_node(current_master_node, body) + Utils.wait_for_master_failover(k8s, failover_targets, self.RETRY_TIMEOUT_SEC) - # if both live on the same node, failover will happen to the other kind worker - if master_pod_node == new_master_pod_node: - nodes = k8s.core_v1.list_node() - for n in nodes.items: - if "node-role.kubernetes.io/master" not in n.metadata.labels & n.metadata.name != master_pod_node: - new_master_pod_node = n.metadata.name - break - - # patch node and test if master is failing over to the expected node - k8s.core_v1.patch_node(master_pod_node, body) - Utils.wait_for_master_failover(k8s, new_master_pod_node, self.RETRY_TIMEOUT_SEC) - - self.assertTrue(master_pod_node != new_master_pod_node, "Master on {} did not fail over to {}".format(master_pod_node, new_master_pod_node)) + new_master_node, new_replica_nodes = get_spilo_nodes(k8s, labels) + self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to {}".format(current_master_node, failover_targets)) + self.assertTrue(num_replicas == len(new_replica_nodes), "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) # undo the tainting body = { @@ -140,7 +136,7 @@ def test_taint_based_eviction(self): "taints": [] } } - k8s.core_v1.patch_node(new_master_pod_node, body) + k8s.core_v1.patch_node(new_master_node, body) class K8sApi: @@ -159,6 +155,19 @@ def __init__(self): class Utils: + @staticmethod + def get_spilo_nodes(k8s_api, pod_labels): + master_pod_node = '' + replica_pod_nodes = [] + podsList = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels) + for pod in podsList.items: + if ('spilo-role', 'master') in pod.metadata.labels.items(): + master_pod_node = pod.spec.node_name + elif ('spilo-role', 'replica') in pod.metadata.labels.items(): + replica_pod_nodes.append(pod.spec.node_name) + + return master_pod_node, replica_pod_nodes + @staticmethod def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' @@ -188,13 +197,13 @@ def count_pods_with_label(k8s_api, labels): return len(k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items) @staticmethod - def wait_for_master_failover(k8s_api, expected_master_pod_node, retry_timeout_sec): + def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): pod_phase = 'Failing over' - new_master_pod_node = '' - while (pod_phase != 'Running') & (new_master_pod_node != expected_master_pod_node): + new_master_node = '' + while (pod_phase != 'Running') and (new_master_node not in expected_master_nodes): pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items if pods: - new_master_pod_node = pods[0].spec.node_name + new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase time.sleep(retry_timeout_sec) From 1a3acd0eab71c9e929fd5bf4faeb67838d6941d8 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 20 May 2019 17:44:35 +0200 Subject: [PATCH 64/92] minor fix --- e2e/tests/test_smoke.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 82576fd09..462089d1f 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -99,7 +99,7 @@ def test_taint_based_eviction(self): labels = 'version=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) - current_master_node, failover_targets = get_spilo_nodes(k8s, labels) + current_master_node, failover_targets = Utils.get_spilo_nodes(k8s, labels) num_replicas = len(failover_targets) # if all pods live on the same node, failover will happen to other worker(s) @@ -126,8 +126,8 @@ def test_taint_based_eviction(self): k8s.core_v1.patch_node(current_master_node, body) Utils.wait_for_master_failover(k8s, failover_targets, self.RETRY_TIMEOUT_SEC) - new_master_node, new_replica_nodes = get_spilo_nodes(k8s, labels) - self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to {}".format(current_master_node, failover_targets)) + new_master_node, new_replica_nodes = Utils.get_spilo_nodes(k8s, labels) + self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) self.assertTrue(num_replicas == len(new_replica_nodes), "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) # undo the tainting From 7ca8662307cfc6869a608f8da273288f3aac8911 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 21 May 2019 12:17:30 +0200 Subject: [PATCH 65/92] minor fix --- e2e/tests/test_smoke.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 462089d1f..727fcb2ca 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -200,7 +200,8 @@ def count_pods_with_label(k8s_api, labels): def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): pod_phase = 'Failing over' new_master_node = '' - while (pod_phase != 'Running') and (new_master_node not in expected_master_nodes): + + while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items if pods: new_master_node = pods[0].spec.node_name From 6a16daf17256926fe0c76202eb80048f0b91038c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 21 May 2019 12:44:39 +0200 Subject: [PATCH 66/92] wait also for replica to be up and running --- e2e/tests/test_smoke.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 727fcb2ca..42522631d 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -125,6 +125,7 @@ def test_taint_based_eviction(self): # patch node and test if master is failing over to one of the expected nodes k8s.core_v1.patch_node(current_master_node, body) Utils.wait_for_master_failover(k8s, failover_targets, self.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'spilo-role=replica', self.RETRY_TIMEOUT_SEC) new_master_node, new_replica_nodes = Utils.get_spilo_nodes(k8s, labels) self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) From 948b03fffc6adf2691c5da68fc575e568792a4fc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 22 May 2019 16:26:50 +0200 Subject: [PATCH 67/92] be nice to flake8 --- e2e/tests/test_smoke.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 42522631d..38b6ca6f3 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -128,8 +128,10 @@ def test_taint_based_eviction(self): Utils.wait_for_pod_start(k8s, 'spilo-role=replica', self.RETRY_TIMEOUT_SEC) new_master_node, new_replica_nodes = Utils.get_spilo_nodes(k8s, labels) - self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertTrue(num_replicas == len(new_replica_nodes), "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) + self.assertTrue(current_master_node != new_master_node, + "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) + self.assertTrue(num_replicas == len(new_replica_nodes), + "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) # undo the tainting body = { @@ -201,9 +203,11 @@ def count_pods_with_label(k8s_api, labels): def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): pod_phase = 'Failing over' new_master_node = '' + labels = 'spilo-role=master,version=acid-minimal-cluster' while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector='spilo-role=master,version=acid-minimal-cluster').items + pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items + if pods: new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase From da2e251b5cc46c04d5315f6f35652178f741aae1 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 22 May 2019 17:51:11 +0200 Subject: [PATCH 68/92] 1st commit for logical backup e2e test --- e2e/tests/test_smoke.py | 68 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 42522631d..8d26bbaa8 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -65,7 +65,7 @@ def setUpClass(cls): Utils.wait_for_pod_start(k8s_api, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_master_is_unique(self): + def master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ @@ -76,7 +76,7 @@ def test_master_is_unique(self): self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): + def scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ @@ -91,7 +91,7 @@ def test_scaling(self): self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): + def taint_based_eviction(self): """ Add taint "postgres=:NoExecute" to node with master. """ @@ -140,6 +140,54 @@ def test_taint_based_eviction(self): k8s.core_v1.patch_node(new_master_node, body) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_logical_backup_cron_job(self): + """ + Ensure we can (a) create the cron job at user request for a specific PG cluster + (b) update the cluster-wide image of the logical backup pod + (c) delete the job at user request + + Limitations: + (a) Does not run the actual batch job because there is no S3 mock to upload backups to + (b) Assumes 'acid-minimal-cluster' exists as defined in setUp + """ + + k8s = K8sApi() + + # create the cron job + pg_patch_enable_backup = { + "spec": { + "enableLogicalBackup" : True, + "logicalBackupSchedule" : "7 7 7 7 *" + } + } + k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + Utils.wait_for_logical_backup_job_creation(k8s, self.RETRY_TIMEOUT_SEC) + + # update the cluster-wide image of the logical backup pod + config_map_patch = { + "data": { + "logical_backup_docker_image" : "test-image-name", + } + } + k8s.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + + operator_pod = k8s.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name + k8s.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf + Utils.wait_for_pod_start(k8s, 'name=postgres-operator', self.RETRY_TIMEOUT_SEC) + #TODO replace this timeout with a meaningful condition to avodi dropping a delete event + time.sleep(30) + + # delete the logical backup cron job + pg_patch_disable_backup = { + "spec": { + "enableLogicalBackup" : False, + } + } + k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + Utils.wait_for_logical_backup_job_deletion(k8s, self.RETRY_TIMEOUT_SEC) + + class K8sApi: def __init__(self): @@ -152,6 +200,8 @@ def __init__(self): self.core_v1 = client.CoreV1Api() self.crd = client.CustomObjectsApi() self.apps_v1 = client.AppsV1Api() + self.custom_objects_api = client.CustomObjectsApi() + self.batch_v1_beta1 = client.BatchV1beta1Api() class Utils: @@ -209,6 +259,18 @@ def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): pod_phase = pods[0].status.phase time.sleep(retry_timeout_sec) + @staticmethod + def wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs): + while (len(k8s_api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo").items) != expected_num_of_jobs): + time.sleep(retry_timeout_sec) + + @staticmethod + def wait_for_logical_backup_job_deletion(k8s_api, retry_timeout_sec): + Utils.wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs = 0) + + @staticmethod + def wait_for_logical_backup_job_creation(k8s_api, retry_timeout_sec): + Utils.wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs = 1) if __name__ == '__main__': unittest.main() From 0ec37397320c761bcd0eb5feccd96c7f3cf82b86 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 12:19:21 +0200 Subject: [PATCH 69/92] avoid dropping DELETE event for cron job --- docs/developer.md | 2 ++ e2e/tests/test_smoke.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index cbbf63dc1..a7c7a2f33 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -323,6 +323,8 @@ The tests utilize examples from `/manifests` (ConfigMap is used for the operator End-to-end tests are executed automatically during builds; to invoke them locally use `make e2e-run` from the project's top directory. Run `make e2e-tools e2e-build` to install `kind` and build the tests' image locally before the first run. +End-to-end tests are written in Python and use `flake8` for code quality. Please run flake8 [before submitting a PR](http://flake8.pycqa.org/en/latest/user/using-hooks.html). + ## Introduce additional configuration parameters In the case you want to add functionality to the operator that shall be diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 8d26bbaa8..60844ad2b 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -65,7 +65,7 @@ def setUpClass(cls): Utils.wait_for_pod_start(k8s_api, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def master_is_unique(self): + def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ @@ -76,7 +76,7 @@ def master_is_unique(self): self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def scaling(self): + def test_scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ @@ -91,7 +91,7 @@ def scaling(self): self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def taint_based_eviction(self): + def test_taint_based_eviction(self): """ Add taint "postgres=:NoExecute" to node with master. """ @@ -175,8 +175,8 @@ def test_logical_backup_cron_job(self): operator_pod = k8s.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name k8s.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf Utils.wait_for_pod_start(k8s, 'name=postgres-operator', self.RETRY_TIMEOUT_SEC) - #TODO replace this timeout with a meaningful condition to avodi dropping a delete event - time.sleep(30) + #HACK avoid dropping a delete event when the operator pod has the label but is still starting + time.sleep(10) # delete the logical backup cron job pg_patch_disable_backup = { From 45896d503f3829f22e9ba9f167be84f07465dd8e Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 14:16:35 +0200 Subject: [PATCH 70/92] add assertions to logical backup test --- e2e/tests/test_smoke.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index d94d6aedd..939e197f2 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -157,19 +157,31 @@ def test_logical_backup_cron_job(self): k8s = K8sApi() # create the cron job + schedule = "7 7 7 7 *" pg_patch_enable_backup = { "spec": { "enableLogicalBackup" : True, - "logicalBackupSchedule" : "7 7 7 7 *" + "logicalBackupSchedule" : schedule } } k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) Utils.wait_for_logical_backup_job_creation(k8s, self.RETRY_TIMEOUT_SEC) + + jobs = Utils.get_logical_backup_job(k8s).items + self.assertTrue(1 == len(jobs), + "Expected 1 logical backup job, found {}".format(len(jobs))) + + job = jobs[0] + self.assertTrue(job.metadata.name == "logical-backup-acid-minimal-cluster", + "Expected job name {}, found {}".format("logical-backup-acid-minimal-cluster", job.metadata.name)) + self.assertTrue(job.spec.schedule == schedule, + "Expected {} schedule, found {}".format(schedule, job.spec.schedule)) # update the cluster-wide image of the logical backup pod + image = "test-image-name" config_map_patch = { "data": { - "logical_backup_docker_image" : "test-image-name", + "logical_backup_docker_image" : image, } } k8s.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) @@ -180,6 +192,11 @@ def test_logical_backup_cron_job(self): #HACK avoid dropping a delete event when the operator pod has the label but is still starting time.sleep(10) + jobs = Utils.get_logical_backup_job(k8s).items + actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + self.assertTrue(actual_image == image, + "Expected job image {}, found {}".format(image, actual_image)) + # delete the logical backup cron job pg_patch_disable_backup = { "spec": { @@ -188,6 +205,9 @@ def test_logical_backup_cron_job(self): } k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) Utils.wait_for_logical_backup_job_deletion(k8s, self.RETRY_TIMEOUT_SEC) + jobs = Utils.get_logical_backup_job(k8s).items + self.assertTrue(0 == len(jobs), + "Expected 0 logical backup jobs, found {}".format(len(jobs))) class K8sApi: @@ -263,9 +283,13 @@ def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): pod_phase = pods[0].status.phase time.sleep(retry_timeout_sec) + @staticmethod + def get_logical_backup_job(k8s_api): + return k8s_api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") + @staticmethod def wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs): - while (len(k8s_api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo").items) != expected_num_of_jobs): + while (len(Utils.get_logical_backup_job(k8s_api).items) != expected_num_of_jobs): time.sleep(retry_timeout_sec) @staticmethod From 7b71e0b1e7e665d584862aaed1041457e23c9e6b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 14:44:44 +0200 Subject: [PATCH 71/92] resolve first TODOs --- e2e/tests/test_smoke.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 939e197f2..ac204325f 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -13,6 +13,7 @@ class SmokeTestCase(unittest.TestCase): Test the most basic functions of the operator. ''' + # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout TEST_TIMEOUT_SEC = 600 RETRY_TIMEOUT_SEC = 5 @@ -30,10 +31,7 @@ def setUpClass(cls): k8s_api = K8sApi() - # HACK - # 1. creating RBAC entites with a separate client fails - # with "AttributeError: object has no attribute 'select_header_accept'" - # 2. utils.create_from_yaml cannot create multiple entites from a single file + # k8s python client fails with certain resources; we resort to kubectl subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) for filename in ["configmap.yaml", "postgres-operator.yaml"]: @@ -58,12 +56,13 @@ def setUpClass(cls): # TODO check if CRD is registered instead Utils.wait_for_pod_start(k8s_api, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) + actual_operator_image = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image + print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - # HACK around the lack of Python client for the acid.zalan.do resource subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - Utils.wait_for_pod_start(k8s_api, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_master_is_unique(self): """ @@ -86,7 +85,6 @@ def test_scaling(self): Utils.wait_for_pg_to_scale(k8s, 3, self.RETRY_TIMEOUT_SEC) self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) - # TODO `kind` pods may stuck in the Terminating state for a few minutes; reproduce/file a bug report Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) From a8cdd8bd9de5c568888242197b17ef2449958557 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 16:14:38 +0200 Subject: [PATCH 72/92] introduce explicit timeout for operator pod --- e2e/tests/test_smoke.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index ac204325f..636f98201 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -16,6 +16,8 @@ class SmokeTestCase(unittest.TestCase): # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout TEST_TIMEOUT_SEC = 600 RETRY_TIMEOUT_SEC = 5 + # labels may be assigned before a pod becomes fully operational; so wait a few seconds more + OPERATOR_POD_START_PERIOD_SEC = 5 @classmethod @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -54,8 +56,10 @@ def setUpClass(cls): } k8s_api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - # TODO check if CRD is registered instead Utils.wait_for_pod_start(k8s_api, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) + # reason: CRD may take time to register + time.sleep(OPERATOR_POD_START_PERIOD_SEC) + actual_operator_image = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish @@ -187,8 +191,8 @@ def test_logical_backup_cron_job(self): operator_pod = k8s.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name k8s.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf Utils.wait_for_pod_start(k8s, 'name=postgres-operator', self.RETRY_TIMEOUT_SEC) - #HACK avoid dropping a delete event when the operator pod has the label but is still starting - time.sleep(10) + # reason: patch below is otherwise dropped during pod restart + time.sleep(OPERATOR_POD_START_PERIOD_SEC) jobs = Utils.get_logical_backup_job(k8s).items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image From 7b9e8521ea0cbdb125bdbbfbcb21872217a25707 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 16:17:57 +0200 Subject: [PATCH 73/92] rename k8s_api to k8s everywhere --- e2e/tests/test_smoke.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 636f98201..716618c11 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -31,13 +31,13 @@ def setUpClass(cls): next invocation of "make e2e" will re-create it. ''' - k8s_api = K8sApi() + k8s = K8sApi() # k8s python client fails with certain resources; we resort to kubectl subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) for filename in ["configmap.yaml", "postgres-operator.yaml"]: - utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) + utils.create_from_yaml(k8s.k8s_client, "manifests/" + filename) # submit the most recent operator image built on the Docker host body = { @@ -54,17 +54,17 @@ def setUpClass(cls): } } } - k8s_api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) + k8s.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - Utils.wait_for_pod_start(k8s_api, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) # reason: CRD may take time to register time.sleep(OPERATOR_POD_START_PERIOD_SEC) - actual_operator_image = k8s_api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image + actual_operator_image = k8s.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - Utils.wait_for_pod_start(k8s_api, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -231,10 +231,10 @@ def __init__(self): class Utils: @staticmethod - def get_spilo_nodes(k8s_api, pod_labels): + def get_spilo_nodes(k8s, pod_labels): master_pod_node = '' replica_pod_nodes = [] - podsList = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels) + podsList = k8s.core_v1.list_namespaced_pod('default', label_selector=pod_labels) for pod in podsList.items: if ('spilo-role', 'master') in pod.metadata.labels.items(): master_pod_node = pod.spec.node_name @@ -244,41 +244,41 @@ def get_spilo_nodes(k8s_api, pod_labels): return master_pod_node, replica_pod_nodes @staticmethod - def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): + def wait_for_pod_start(k8s, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' while pod_phase != 'Running': - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + pods = k8s.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase time.sleep(retry_timeout_sec) @staticmethod - def wait_for_pg_to_scale(k8s_api, number_of_instances, retry_timeout_sec): + def wait_for_pg_to_scale(k8s, number_of_instances, retry_timeout_sec): body = { "spec": { "numberOfInstances": number_of_instances } } - _ = k8s_api.crd.patch_namespaced_custom_object("acid.zalan.do", + _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' - while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: + while Utils.count_pods_with_label(k8s, labels) != number_of_instances: time.sleep(retry_timeout_sec) @staticmethod - def count_pods_with_label(k8s_api, labels): - return len(k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items) + def count_pods_with_label(k8s, labels): + return len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) @staticmethod - def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): + def wait_for_master_failover(k8s, expected_master_nodes, retry_timeout_sec): pod_phase = 'Failing over' new_master_node = '' labels = 'spilo-role=master,version=acid-minimal-cluster' while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): - pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=labels).items + pods = k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items if pods: new_master_node = pods[0].spec.node_name @@ -286,21 +286,21 @@ def wait_for_master_failover(k8s_api, expected_master_nodes, retry_timeout_sec): time.sleep(retry_timeout_sec) @staticmethod - def get_logical_backup_job(k8s_api): - return k8s_api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") + def get_logical_backup_job(k8s): + return k8s.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") @staticmethod - def wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs): - while (len(Utils.get_logical_backup_job(k8s_api).items) != expected_num_of_jobs): + def wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs): + while (len(Utils.get_logical_backup_job(k8s).items) != expected_num_of_jobs): time.sleep(retry_timeout_sec) @staticmethod - def wait_for_logical_backup_job_deletion(k8s_api, retry_timeout_sec): - Utils.wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs = 0) + def wait_for_logical_backup_job_deletion(k8s, retry_timeout_sec): + Utils.wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs = 0) @staticmethod - def wait_for_logical_backup_job_creation(k8s_api, retry_timeout_sec): - Utils.wait_for_logical_backup_job(k8s_api, retry_timeout_sec, expected_num_of_jobs = 1) + def wait_for_logical_backup_job_creation(k8s, retry_timeout_sec): + Utils.wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs = 1) if __name__ == '__main__': unittest.main() From 63aa6e60fbc969277207e7adddbca9c539752bd1 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 16:57:02 +0200 Subject: [PATCH 74/92] move retry timeout to utils --- e2e/tests/test_smoke.py | 49 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 716618c11..21bd34815 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -15,7 +15,6 @@ class SmokeTestCase(unittest.TestCase): # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout TEST_TIMEOUT_SEC = 600 - RETRY_TIMEOUT_SEC = 5 # labels may be assigned before a pod becomes fully operational; so wait a few seconds more OPERATOR_POD_START_PERIOD_SEC = 5 @@ -56,15 +55,15 @@ def setUpClass(cls): } k8s.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - Utils.wait_for_pod_start(k8s, 'name=postgres-operator', cls.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'name=postgres-operator') # reason: CRD may take time to register - time.sleep(OPERATOR_POD_START_PERIOD_SEC) + time.sleep(cls.OPERATOR_POD_START_PERIOD_SEC) actual_operator_image = k8s.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - Utils.wait_for_pod_start(k8s, 'spilo-role=master', cls.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'spilo-role=master') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -86,10 +85,10 @@ def test_scaling(self): k8s = K8sApi() labels = "version=acid-minimal-cluster" - Utils.wait_for_pg_to_scale(k8s, 3, self.RETRY_TIMEOUT_SEC) + Utils.wait_for_pg_to_scale(k8s, 3) self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) - Utils.wait_for_pg_to_scale(k8s, 2, self.RETRY_TIMEOUT_SEC) + Utils.wait_for_pg_to_scale(k8s, 2) self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -126,8 +125,8 @@ def test_taint_based_eviction(self): # patch node and test if master is failing over to one of the expected nodes k8s.core_v1.patch_node(current_master_node, body) - Utils.wait_for_master_failover(k8s, failover_targets, self.RETRY_TIMEOUT_SEC) - Utils.wait_for_pod_start(k8s, 'spilo-role=replica', self.RETRY_TIMEOUT_SEC) + Utils.wait_for_master_failover(k8s, failover_targets) + Utils.wait_for_pod_start(k8s, 'spilo-role=replica') new_master_node, new_replica_nodes = Utils.get_spilo_nodes(k8s, labels) self.assertTrue(current_master_node != new_master_node, @@ -167,7 +166,7 @@ def test_logical_backup_cron_job(self): } } k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - Utils.wait_for_logical_backup_job_creation(k8s, self.RETRY_TIMEOUT_SEC) + Utils.wait_for_logical_backup_job_creation(k8s) jobs = Utils.get_logical_backup_job(k8s).items self.assertTrue(1 == len(jobs), @@ -190,9 +189,9 @@ def test_logical_backup_cron_job(self): operator_pod = k8s.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name k8s.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf - Utils.wait_for_pod_start(k8s, 'name=postgres-operator', self.RETRY_TIMEOUT_SEC) + Utils.wait_for_pod_start(k8s, 'name=postgres-operator') # reason: patch below is otherwise dropped during pod restart - time.sleep(OPERATOR_POD_START_PERIOD_SEC) + time.sleep(self.OPERATOR_POD_START_PERIOD_SEC) jobs = Utils.get_logical_backup_job(k8s).items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image @@ -206,7 +205,7 @@ def test_logical_backup_cron_job(self): } } k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - Utils.wait_for_logical_backup_job_deletion(k8s, self.RETRY_TIMEOUT_SEC) + Utils.wait_for_logical_backup_job_deletion(k8s) jobs = Utils.get_logical_backup_job(k8s).items self.assertTrue(0 == len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) @@ -230,6 +229,8 @@ def __init__(self): class Utils: + RETRY_TIMEOUT_SEC = 5 + @staticmethod def get_spilo_nodes(k8s, pod_labels): master_pod_node = '' @@ -244,16 +245,16 @@ def get_spilo_nodes(k8s, pod_labels): return master_pod_node, replica_pod_nodes @staticmethod - def wait_for_pod_start(k8s, pod_labels, retry_timeout_sec): + def wait_for_pod_start(k8s, pod_labels): pod_phase = 'No pod running' while pod_phase != 'Running': pods = k8s.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - time.sleep(retry_timeout_sec) + time.sleep(Utils.RETRY_TIMEOUT_SEC) @staticmethod - def wait_for_pg_to_scale(k8s, number_of_instances, retry_timeout_sec): + def wait_for_pg_to_scale(k8s, number_of_instances): body = { "spec": { @@ -265,14 +266,14 @@ def wait_for_pg_to_scale(k8s, number_of_instances, retry_timeout_sec): labels = 'version=acid-minimal-cluster' while Utils.count_pods_with_label(k8s, labels) != number_of_instances: - time.sleep(retry_timeout_sec) + time.sleep(Utils.RETRY_TIMEOUT_SEC) @staticmethod def count_pods_with_label(k8s, labels): return len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) @staticmethod - def wait_for_master_failover(k8s, expected_master_nodes, retry_timeout_sec): + def wait_for_master_failover(k8s, expected_master_nodes): pod_phase = 'Failing over' new_master_node = '' labels = 'spilo-role=master,version=acid-minimal-cluster' @@ -283,24 +284,24 @@ def wait_for_master_failover(k8s, expected_master_nodes, retry_timeout_sec): if pods: new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase - time.sleep(retry_timeout_sec) + time.sleep(Utils.RETRY_TIMEOUT_SEC) @staticmethod def get_logical_backup_job(k8s): return k8s.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") @staticmethod - def wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs): + def wait_for_logical_backup_job(k8s, expected_num_of_jobs): while (len(Utils.get_logical_backup_job(k8s).items) != expected_num_of_jobs): - time.sleep(retry_timeout_sec) + time.sleep(Utils.RETRY_TIMEOUT_SEC) @staticmethod - def wait_for_logical_backup_job_deletion(k8s, retry_timeout_sec): - Utils.wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs = 0) + def wait_for_logical_backup_job_deletion(k8s): + Utils.wait_for_logical_backup_job(k8s, expected_num_of_jobs = 0) @staticmethod - def wait_for_logical_backup_job_creation(k8s, retry_timeout_sec): - Utils.wait_for_logical_backup_job(k8s, retry_timeout_sec, expected_num_of_jobs = 1) + def wait_for_logical_backup_job_creation(k8s): + Utils.wait_for_logical_backup_job(k8s, expected_num_of_jobs = 1) if __name__ == '__main__': unittest.main() From 7cee3701fd7085603e4579cd8eab5af67cdf1420 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 18:03:38 +0200 Subject: [PATCH 75/92] switch to K8s class instead of Util --- e2e/tests/test_smoke.py | 155 ++++++++++++++++++++++++++++++---------- 1 file changed, 117 insertions(+), 38 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 21bd34815..da62006bd 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -30,13 +30,14 @@ def setUpClass(cls): next invocation of "make e2e" will re-create it. ''' - k8s = K8sApi() + # set a single k8s wrapper for all tests + k8s = cls.k8s = K8s() - # k8s python client fails with certain resources; we resort to kubectl + # k8s python client fails with multiple resources in a single file; we resort to kubectl subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) for filename in ["configmap.yaml", "postgres-operator.yaml"]: - utils.create_from_yaml(k8s.k8s_client, "manifests/" + filename) + utils.create_from_yaml(k8s.api.k8s_client, "manifests/" + filename) # submit the most recent operator image built on the Docker host body = { @@ -53,60 +54,62 @@ def setUpClass(cls): } } } - k8s.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) + k8s.api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - Utils.wait_for_pod_start(k8s, 'name=postgres-operator') + k8s.wait_for_pod_start('name=postgres-operator') # reason: CRD may take time to register time.sleep(cls.OPERATOR_POD_START_PERIOD_SEC) - actual_operator_image = k8s.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image + actual_operator_image = k8s.api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) - Utils.wait_for_pod_start(k8s, 'spilo-role=master') + k8s.wait_for_pod_start('spilo-role=master') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_master_is_unique(self): + def master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ - k8s = K8sApi() + + k8s = self.k8s labels = 'spilo-role=master,version=acid-minimal-cluster' - num_of_master_pods = Utils.count_pods_with_label(k8s, labels) + num_of_master_pods = k8s.count_pods_with_label(labels) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): + def scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ - k8s = K8sApi() + + k8s = self.k8s labels = "version=acid-minimal-cluster" - Utils.wait_for_pg_to_scale(k8s, 3) - self.assertEqual(3, Utils.count_pods_with_label(k8s, labels)) + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) - Utils.wait_for_pg_to_scale(k8s, 2) - self.assertEqual(2, Utils.count_pods_with_label(k8s, labels)) + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): """ Add taint "postgres=:NoExecute" to node with master. """ - k8s = K8sApi() + k8s = self.k8s labels = 'version=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) - current_master_node, failover_targets = Utils.get_spilo_nodes(k8s, labels) + current_master_node, failover_targets = k8s.get_spilo_nodes(labels) num_replicas = len(failover_targets) # if all pods live on the same node, failover will happen to other worker(s) failover_targets = [x for x in failover_targets if x != current_master_node] if len(failover_targets) == 0: - nodes = k8s.core_v1.list_node() + nodes = k8s.api.core_v1.list_node() for n in nodes.items: if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != current_master_node: failover_targets.append(n.metadata.name) @@ -124,11 +127,11 @@ def test_taint_based_eviction(self): } # patch node and test if master is failing over to one of the expected nodes - k8s.core_v1.patch_node(current_master_node, body) - Utils.wait_for_master_failover(k8s, failover_targets) - Utils.wait_for_pod_start(k8s, 'spilo-role=replica') + k8s.api.core_v1.patch_node(current_master_node, body) + k8s.wait_for_master_failover(failover_targets) + k8s.wait_for_pod_start('spilo-role=replica') - new_master_node, new_replica_nodes = Utils.get_spilo_nodes(k8s, labels) + new_master_node, new_replica_nodes = k8s.get_spilo_nodes(labels) self.assertTrue(current_master_node != new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) self.assertTrue(num_replicas == len(new_replica_nodes), @@ -140,7 +143,7 @@ def test_taint_based_eviction(self): "taints": [] } } - k8s.core_v1.patch_node(new_master_node, body) + k8s.api.core_v1.patch_node(new_master_node, body) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -155,7 +158,7 @@ def test_logical_backup_cron_job(self): (b) Assumes 'acid-minimal-cluster' exists as defined in setUp """ - k8s = K8sApi() + k8s = self.k8s # create the cron job schedule = "7 7 7 7 *" @@ -165,10 +168,10 @@ def test_logical_backup_cron_job(self): "logicalBackupSchedule" : schedule } } - k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - Utils.wait_for_logical_backup_job_creation(k8s) + k8s.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + k8s.wait_for_logical_backup_job_creation() - jobs = Utils.get_logical_backup_job(k8s).items + jobs = k8s.get_logical_backup_job().items self.assertTrue(1 == len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) @@ -185,15 +188,15 @@ def test_logical_backup_cron_job(self): "logical_backup_docker_image" : image, } } - k8s.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) + k8s.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - operator_pod = k8s.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name - k8s.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf - Utils.wait_for_pod_start(k8s, 'name=postgres-operator') + operator_pod = k8s.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name + k8s.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf + k8s.wait_for_pod_start('name=postgres-operator') # reason: patch below is otherwise dropped during pod restart time.sleep(self.OPERATOR_POD_START_PERIOD_SEC) - jobs = Utils.get_logical_backup_job(k8s).items + jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image self.assertTrue(actual_image == image, "Expected job image {}, found {}".format(image, actual_image)) @@ -204,9 +207,9 @@ def test_logical_backup_cron_job(self): "enableLogicalBackup" : False, } } - k8s.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - Utils.wait_for_logical_backup_job_deletion(k8s) - jobs = Utils.get_logical_backup_job(k8s).items + k8s.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.wait_for_logical_backup_job_deletion() + jobs = k8s.get_logical_backup_job().items self.assertTrue(0 == len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) @@ -220,11 +223,87 @@ def __init__(self): self.config = config.load_kube_config() self.k8s_client = client.ApiClient() - self.core_v1 = client.CoreV1Api() + self.crd = client.CustomObjectsApi() + self.core_v1 = client.CoreV1Api() self.apps_v1 = client.AppsV1Api() - self.custom_objects_api = client.CustomObjectsApi() self.batch_v1_beta1 = client.BatchV1beta1Api() + self.custom_objects_api = client.CustomObjectsApi() + + + +class K8s: + + RETRY_TIMEOUT_SEC = 5 + + def __init__(self): + self.api = K8sApi() + + def get_spilo_nodes(self, pod_labels): + master_pod_node = '' + replica_pod_nodes = [] + podsList = self.api.core_v1.list_namespaced_pod('default', label_selector=pod_labels) + for pod in podsList.items: + if ('spilo-role', 'master') in pod.metadata.labels.items(): + master_pod_node = pod.spec.node_name + elif ('spilo-role', 'replica') in pod.metadata.labels.items(): + replica_pod_nodes.append(pod.spec.node_name) + + return master_pod_node, replica_pod_nodes + + def wait_for_pod_start(self, pod_labels): + pod_phase = 'No pod running' + while pod_phase != 'Running': + pods = self.api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + if pods: + pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pg_to_scale(self, number_of_instances): + + body = { + "spec": { + "numberOfInstances": number_of_instances + } + } + _ = self.api.crd.patch_namespaced_custom_object("acid.zalan.do", + "v1", "default", "postgresqls", "acid-minimal-cluster", body) + + labels = 'version=acid-minimal-cluster' + while self.count_pods_with_label(labels) != number_of_instances: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def count_pods_with_label(self, labels): + return len(self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items) + + def wait_for_master_failover(self, expected_master_nodes): + pod_phase = 'Failing over' + new_master_node = '' + labels = 'spilo-role=master,version=acid-minimal-cluster' + + while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): + pods = self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items + + if pods: + new_master_node = pods[0].spec.node_name + pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) + + def get_logical_backup_job(self): + return self.api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") + + def wait_for_logical_backup_job(self, expected_num_of_jobs): + while (len(self.api.get_logical_backup_job().items) != expected_num_of_jobs): + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_logical_backup_job_deletion(self): + Utils.wait_for_logical_backup_job(expected_num_of_jobs = 0) + + def wait_for_logical_backup_job_creation(self): + Utils.wait_for_logical_backup_job(expected_num_of_jobs = 1) + + def create_with_kubectl(self, path): + subprocess.run(["kubectl", "create", "-f", path]) class Utils: From 0580916ba05d7fcd77b4bdd0d4b1251da75807ce Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 18:04:21 +0200 Subject: [PATCH 76/92] drop Utils --- e2e/tests/test_smoke.py | 77 ----------------------------------------- 1 file changed, 77 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index da62006bd..3add9df15 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -305,82 +305,5 @@ def wait_for_logical_backup_job_creation(self): def create_with_kubectl(self, path): subprocess.run(["kubectl", "create", "-f", path]) - -class Utils: - - RETRY_TIMEOUT_SEC = 5 - - @staticmethod - def get_spilo_nodes(k8s, pod_labels): - master_pod_node = '' - replica_pod_nodes = [] - podsList = k8s.core_v1.list_namespaced_pod('default', label_selector=pod_labels) - for pod in podsList.items: - if ('spilo-role', 'master') in pod.metadata.labels.items(): - master_pod_node = pod.spec.node_name - elif ('spilo-role', 'replica') in pod.metadata.labels.items(): - replica_pod_nodes.append(pod.spec.node_name) - - return master_pod_node, replica_pod_nodes - - @staticmethod - def wait_for_pod_start(k8s, pod_labels): - pod_phase = 'No pod running' - while pod_phase != 'Running': - pods = k8s.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items - if pods: - pod_phase = pods[0].status.phase - time.sleep(Utils.RETRY_TIMEOUT_SEC) - - @staticmethod - def wait_for_pg_to_scale(k8s, number_of_instances): - - body = { - "spec": { - "numberOfInstances": number_of_instances - } - } - _ = k8s.crd.patch_namespaced_custom_object("acid.zalan.do", - "v1", "default", "postgresqls", "acid-minimal-cluster", body) - - labels = 'version=acid-minimal-cluster' - while Utils.count_pods_with_label(k8s, labels) != number_of_instances: - time.sleep(Utils.RETRY_TIMEOUT_SEC) - - @staticmethod - def count_pods_with_label(k8s, labels): - return len(k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items) - - @staticmethod - def wait_for_master_failover(k8s, expected_master_nodes): - pod_phase = 'Failing over' - new_master_node = '' - labels = 'spilo-role=master,version=acid-minimal-cluster' - - while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): - pods = k8s.core_v1.list_namespaced_pod('default', label_selector=labels).items - - if pods: - new_master_node = pods[0].spec.node_name - pod_phase = pods[0].status.phase - time.sleep(Utils.RETRY_TIMEOUT_SEC) - - @staticmethod - def get_logical_backup_job(k8s): - return k8s.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") - - @staticmethod - def wait_for_logical_backup_job(k8s, expected_num_of_jobs): - while (len(Utils.get_logical_backup_job(k8s).items) != expected_num_of_jobs): - time.sleep(Utils.RETRY_TIMEOUT_SEC) - - @staticmethod - def wait_for_logical_backup_job_deletion(k8s): - Utils.wait_for_logical_backup_job(k8s, expected_num_of_jobs = 0) - - @staticmethod - def wait_for_logical_backup_job_creation(k8s): - Utils.wait_for_logical_backup_job(k8s, expected_num_of_jobs = 1) - if __name__ == '__main__': unittest.main() From 2984872dfb639ca00038a7559e621d7263f4acb7 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Thu, 23 May 2019 18:16:16 +0200 Subject: [PATCH 77/92] create with kubectl --- e2e/tests/test_smoke.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 3add9df15..698f06529 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -33,11 +33,10 @@ def setUpClass(cls): # set a single k8s wrapper for all tests k8s = cls.k8s = K8s() - # k8s python client fails with multiple resources in a single file; we resort to kubectl - subprocess.run(["kubectl", "create", "-f", "manifests/operator-service-account-rbac.yaml"]) - - for filename in ["configmap.yaml", "postgres-operator.yaml"]: - utils.create_from_yaml(k8s.api.k8s_client, "manifests/" + filename) + for filename in ["operator-service-account-rbac.yaml", + "configmap.yaml", + "postgres-operator.yaml"]: + k8s.create_with_kubectl("manifests/" + filename) # submit the most recent operator image built on the Docker host body = { @@ -63,12 +62,12 @@ def setUpClass(cls): actual_operator_image = k8s.api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - subprocess.run(["kubectl", "create", "-f", "manifests/minimal-postgres-manifest.yaml"]) + k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def master_is_unique(self): + def test_master_is_unique(self): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master". """ @@ -80,7 +79,7 @@ def master_is_unique(self): self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def scaling(self): + def test_scaling(self): """ Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. """ From 2ce7b5113fbcbe2e5c6a7f5bae501ff09130d4a2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 14:16:29 +0200 Subject: [PATCH 78/92] refactor --- e2e/requirements.txt | 1 + e2e/tests/{test_smoke.py => test_e2e.py} | 102 ++++++++++------------- 2 files changed, 46 insertions(+), 57 deletions(-) rename e2e/tests/{test_smoke.py => test_e2e.py} (83%) mode change 100755 => 100644 diff --git a/e2e/requirements.txt b/e2e/requirements.txt index c46b334ad..562e34c3a 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,2 +1,3 @@ kubernetes==9.0.0 timeout_decorator==0.4.1 +pyyaml diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_e2e.py old mode 100755 new mode 100644 similarity index 83% rename from e2e/tests/test_smoke.py rename to e2e/tests/test_e2e.py index 698f06529..b61d70933 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_e2e.py @@ -4,19 +4,18 @@ import subprocess import warnings import os +import yaml from kubernetes import client, config, utils -class SmokeTestCase(unittest.TestCase): +class EndToEndTestCase(unittest.TestCase): ''' - Test the most basic functions of the operator. + Test interaction of the operator with multiple k8s components. ''' # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout TEST_TIMEOUT_SEC = 600 - # labels may be assigned before a pod becomes fully operational; so wait a few seconds more - OPERATOR_POD_START_PERIOD_SEC = 5 @classmethod @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -27,37 +26,24 @@ def setUpClass(cls): /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities. In the case of test failure the cluster will stay to enable manual examination; - next invocation of "make e2e" will re-create it. + next invocation of "make e2e-run" will re-create it. ''' # set a single k8s wrapper for all tests k8s = cls.k8s = K8s() + # submit the most recent operator image built on the Docker host + with open("manifests/postgres-operator.yaml", 'r+') as f: + operator_deployment = yaml.load(f, Loader=yaml.Loader) + operator_deployment["spec"]["template"]["spec"]["containers"][0]["image"] = os.environ['OPERATOR_IMAGE'] + yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) + for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml"]: k8s.create_with_kubectl("manifests/" + filename) - # submit the most recent operator image built on the Docker host - body = { - "spec": { - "template": { - "spec": { - "containers": [ - { - "name": "postgres-operator", - "image": os.environ['OPERATOR_IMAGE'] - } - ] - } - } - } - } - k8s.api.apps_v1.patch_namespaced_deployment("postgres-operator", "default", body) - - k8s.wait_for_pod_start('name=postgres-operator') - # reason: CRD may take time to register - time.sleep(cls.OPERATOR_POD_START_PERIOD_SEC) + k8s.wait_for_operator_pod_start() actual_operator_image = k8s.api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish @@ -66,22 +52,10 @@ def setUpClass(cls): k8s.wait_for_pod_start('spilo-role=master') - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_master_is_unique(self): - """ - Check that there is a single pod in the k8s cluster with the label "spilo-role=master". - """ - - k8s = self.k8s - labels = 'spilo-role=master,version=acid-minimal-cluster' - - num_of_master_pods = k8s.count_pods_with_label(labels) - self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): """ - Scale up from 2 to 3 pods and back to 2 by updating the Postgres manifest at runtime. + Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. """ k8s = self.k8s @@ -89,14 +63,16 @@ def test_scaling(self): k8s.wait_for_pg_to_scale(3) self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() k8s.wait_for_pg_to_scale(2) self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): """ - Add taint "postgres=:NoExecute" to node with master. + Add taint "postgres=:NoExecute" to node with master. This must cause a failover. """ k8s = self.k8s labels = 'version=acid-minimal-cluster' @@ -135,6 +111,7 @@ def test_taint_based_eviction(self): "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) self.assertTrue(num_replicas == len(new_replica_nodes), "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) + self.assert_master_is_unique() # undo the tainting body = { @@ -149,7 +126,7 @@ def test_taint_based_eviction(self): def test_logical_backup_cron_job(self): """ Ensure we can (a) create the cron job at user request for a specific PG cluster - (b) update the cluster-wide image of the logical backup pod + (b) update the cluster-wide image for the logical backup pod (c) delete the job at user request Limitations: @@ -191,9 +168,7 @@ def test_logical_backup_cron_job(self): operator_pod = k8s.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name k8s.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf - k8s.wait_for_pod_start('name=postgres-operator') - # reason: patch below is otherwise dropped during pod restart - time.sleep(self.OPERATOR_POD_START_PERIOD_SEC) + k8s.wait_for_operator_pod_start() jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image @@ -212,6 +187,18 @@ def test_logical_backup_cron_job(self): self.assertTrue(0 == len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) + def assert_master_is_unique(self): + """ + Check that there is a single pod in the k8s cluster with the label "spilo-role=master" + To be called manually after operations that affect pods + """ + + k8s = self.k8s + labels = 'spilo-role=master,version=acid-minimal-cluster' + + num_of_master_pods = k8s.count_pods_with_label(labels) + self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) + class K8sApi: @@ -223,17 +210,16 @@ def __init__(self): self.config = config.load_kube_config() self.k8s_client = client.ApiClient() - self.crd = client.CustomObjectsApi() self.core_v1 = client.CoreV1Api() self.apps_v1 = client.AppsV1Api() self.batch_v1_beta1 = client.BatchV1beta1Api() self.custom_objects_api = client.CustomObjectsApi() - class K8s: - - RETRY_TIMEOUT_SEC = 5 + ''' + Wraps around K8 api client and helper methods. + ''' def __init__(self): self.api = K8sApi() @@ -250,13 +236,17 @@ def get_spilo_nodes(self, pod_labels): return master_pod_node, replica_pod_nodes + def wait_for_operator_pod_start(self): + self. wait_for_pod_start("name=postgres-operator") + # HACK operator must register CRD / add existing PG clusters after pod start up + time.sleep(10) + def wait_for_pod_start(self, pod_labels): pod_phase = 'No pod running' while pod_phase != 'Running': pods = self.api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - time.sleep(self.RETRY_TIMEOUT_SEC) def wait_for_pg_to_scale(self, number_of_instances): @@ -265,12 +255,12 @@ def wait_for_pg_to_scale(self, number_of_instances): "numberOfInstances": number_of_instances } } - _ = self.api.crd.patch_namespaced_custom_object("acid.zalan.do", + _ = self.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: - time.sleep(self.RETRY_TIMEOUT_SEC) + pass def count_pods_with_label(self, labels): return len(self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items) @@ -282,25 +272,23 @@ def wait_for_master_failover(self, expected_master_nodes): while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): pods = self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items - if pods: new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase - time.sleep(self.RETRY_TIMEOUT_SEC) def get_logical_backup_job(self): return self.api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") def wait_for_logical_backup_job(self, expected_num_of_jobs): - while (len(self.api.get_logical_backup_job().items) != expected_num_of_jobs): - time.sleep(self.RETRY_TIMEOUT_SEC) + while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): + pass def wait_for_logical_backup_job_deletion(self): - Utils.wait_for_logical_backup_job(expected_num_of_jobs = 0) + self.wait_for_logical_backup_job(expected_num_of_jobs = 0) def wait_for_logical_backup_job_creation(self): - Utils.wait_for_logical_backup_job(expected_num_of_jobs = 1) - + self.wait_for_logical_backup_job(expected_num_of_jobs = 1) + def create_with_kubectl(self, path): subprocess.run(["kubectl", "create", "-f", path]) From 3f32870d75fbe387ab58ae75b3009ded768f7ee2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 14:37:08 +0200 Subject: [PATCH 79/92] fix flake8 violations --- e2e/tests/test_e2e.py | 84 ++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index b61d70933..3298b6e46 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -6,7 +6,7 @@ import os import yaml -from kubernetes import client, config, utils +from kubernetes import client, config class EndToEndTestCase(unittest.TestCase): @@ -38,20 +38,20 @@ def setUpClass(cls): operator_deployment["spec"]["template"]["spec"]["containers"][0]["image"] = os.environ['OPERATOR_IMAGE'] yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) - for filename in ["operator-service-account-rbac.yaml", - "configmap.yaml", + for filename in ["operator-service-account-rbac.yaml", + "configmap.yaml", "postgres-operator.yaml"]: k8s.create_with_kubectl("manifests/" + filename) k8s.wait_for_operator_pod_start() - actual_operator_image = k8s.api.core_v1.list_namespaced_pod('default', label_selector='name=postgres-operator').items[0].spec.containers[0].image - print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish + actual_operator_image = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image + print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): """ @@ -121,7 +121,6 @@ def test_taint_based_eviction(self): } k8s.api.core_v1.patch_node(new_master_node, body) - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): """ @@ -140,54 +139,58 @@ def test_logical_backup_cron_job(self): schedule = "7 7 7 7 *" pg_patch_enable_backup = { "spec": { - "enableLogicalBackup" : True, - "logicalBackupSchedule" : schedule + "enableLogicalBackup": True, + "logicalBackupSchedule": schedule } } - k8s.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) k8s.wait_for_logical_backup_job_creation() - + jobs = k8s.get_logical_backup_job().items - self.assertTrue(1 == len(jobs), - "Expected 1 logical backup job, found {}".format(len(jobs))) + self.assertTrue(1 == len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) job = jobs[0] self.assertTrue(job.metadata.name == "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}".format("logical-backup-acid-minimal-cluster", job.metadata.name)) + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) self.assertTrue(job.spec.schedule == schedule, - "Expected {} schedule, found {}".format(schedule, job.spec.schedule)) + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) # update the cluster-wide image of the logical backup pod image = "test-image-name" config_map_patch = { "data": { - "logical_backup_docker_image" : image, + "logical_backup_docker_image": image, } } k8s.api.core_v1.patch_namespaced_config_map("postgres-operator", "default", config_map_patch) - operator_pod = k8s.api.core_v1.list_namespaced_pod('default', label_selector="name=postgres-operator").items[0].metadata.name - k8s.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf + operator_pod = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector="name=postgres-operator").items[0].metadata.name + k8s.api.core_v1.delete_namespaced_pod(operator_pod, "default") # restart reloads the conf k8s.wait_for_operator_pod_start() jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image self.assertTrue(actual_image == image, - "Expected job image {}, found {}".format(image, actual_image)) + "Expected job image {}, found {}".format(image, actual_image)) # delete the logical backup cron job pg_patch_disable_backup = { "spec": { - "enableLogicalBackup" : False, + "enableLogicalBackup": False, } } - k8s.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) k8s.wait_for_logical_backup_job_deletion() jobs = k8s.get_logical_backup_job().items self.assertTrue(0 == len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + "Expected 0 logical backup jobs, found {}".format(len(jobs))) - def assert_master_is_unique(self): + def assert_master_is_unique(self, namespace='default'): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods @@ -196,7 +199,7 @@ def assert_master_is_unique(self): k8s = self.k8s labels = 'spilo-role=master,version=acid-minimal-cluster' - num_of_master_pods = k8s.count_pods_with_label(labels) + num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) @@ -224,10 +227,10 @@ class K8s: def __init__(self): self.api = K8sApi() - def get_spilo_nodes(self, pod_labels): + def get_spilo_nodes(self, pod_labels, namespace='default'): master_pod_node = '' replica_pod_nodes = [] - podsList = self.api.core_v1.list_namespaced_pod('default', label_selector=pod_labels) + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels) for pod in podsList.items: if ('spilo-role', 'master') in pod.metadata.labels.items(): master_pod_node = pod.spec.node_name @@ -238,59 +241,60 @@ def get_spilo_nodes(self, pod_labels): def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") - # HACK operator must register CRD / add existing PG clusters after pod start up + # HACK operator must register CRD / add existing PG clusters after pod start up time.sleep(10) - def wait_for_pod_start(self, pod_labels): + def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' while pod_phase != 'Running': - pods = self.api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase - def wait_for_pg_to_scale(self, number_of_instances): + def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): body = { "spec": { "numberOfInstances": number_of_instances } } - _ = self.api.custom_objects_api.patch_namespaced_custom_object("acid.zalan.do", - "v1", "default", "postgresqls", "acid-minimal-cluster", body) + _ = self.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: pass - def count_pods_with_label(self, labels): - return len(self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items) + def count_pods_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) - def wait_for_master_failover(self, expected_master_nodes): + def wait_for_master_failover(self, expected_master_nodes, namespace='default'): pod_phase = 'Failing over' new_master_node = '' labels = 'spilo-role=master,version=acid-minimal-cluster' while (pod_phase != 'Running') or (new_master_node not in expected_master_nodes): - pods = self.api.core_v1.list_namespaced_pod('default', label_selector=labels).items + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items if pods: new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase - def get_logical_backup_job(self): - return self.api.batch_v1_beta1.list_namespaced_cron_job("default", label_selector="application=spilo") + def get_logical_backup_job(self, namespace='default'): + return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") def wait_for_logical_backup_job(self, expected_num_of_jobs): while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): pass def wait_for_logical_backup_job_deletion(self): - self.wait_for_logical_backup_job(expected_num_of_jobs = 0) + self.wait_for_logical_backup_job(expected_num_of_jobs=0) def wait_for_logical_backup_job_creation(self): - self.wait_for_logical_backup_job(expected_num_of_jobs = 1) + self.wait_for_logical_backup_job(expected_num_of_jobs=1) def create_with_kubectl(self, path): subprocess.run(["kubectl", "create", "-f", path]) + if __name__ == '__main__': unittest.main() From 46a4342a2f46c96329eb1451945aad1f2ca69842 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 14:51:36 +0200 Subject: [PATCH 80/92] pin versions --- e2e/Dockerfile | 6 +++++- e2e/requirements.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 3d569b2fb..50c496239 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -7,7 +7,11 @@ COPY manifests ./manifests COPY e2e/requirements.txt e2e/tests ./ RUN apt-get update \ - && apt-get install --no-install-recommends -y python3 python3-setuptools python3-pip curl \ + && apt-get install --no-install-recommends -y \ + python3=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + python3-pip=9.0.1-2.3~ubuntu1 \ + curl=7.58.0-2ubuntu3.7 \ && pip3 install --no-cache-dir -r requirements.txt \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 562e34c3a..68a8775ff 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,3 +1,3 @@ kubernetes==9.0.0 timeout_decorator==0.4.1 -pyyaml +pyyaml==5.1 From 424ad7677c9e8f3b890d328c7cc7f97a2fc282a8 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 14:56:30 +0200 Subject: [PATCH 81/92] lint run.sh --- e2e/run.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index a711a467f..d907502a2 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -21,21 +21,22 @@ function start_kind(){ fi kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml - kind-linux-amd64 load docker-image ${operator_image} --name ${cluster_name} - export KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" + kind-linux-amd64 load docker-image "${operator_image}" --name ${cluster_name} + KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})" + export KUBECONFIG } function set_kind_api_server_ip(){ # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network - cp ${KUBECONFIG} /tmp + cp "${KUBECONFIG}" /tmp readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase - readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) - sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} + readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) + sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}" } function run_tests(){ - docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} + docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" } function clean_up(){ From 4bb20da4f5033048e8dfe05255971dbf7f3524c6 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 15:05:35 +0200 Subject: [PATCH 82/92] unset Kubeconfig env var --- e2e/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/run.sh b/e2e/run.sh index d907502a2..c18ee6c95 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -40,6 +40,7 @@ function run_tests(){ } function clean_up(){ + unset KUBECONFIG kind-linux-amd64 delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} } From 7075d11975ccd2a6d49d99e983f94b691bfa3ce7 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 15:53:36 +0200 Subject: [PATCH 83/92] add a test for multiple namespaces --- e2e/tests/test_e2e.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 3298b6e46..0db4080db 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -32,6 +32,12 @@ def setUpClass(cls): # set a single k8s wrapper for all tests k8s = cls.k8s = K8s() + # operator deploys pod service account there on start up + # needed for test_multi_namespace_support() + cls.namespace = "test" + v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.namespace)) + k8s.api.core_v1.create_namespace(v1_namespace) + # submit the most recent operator image built on the Docker host with open("manifests/postgres-operator.yaml", 'r+') as f: operator_deployment = yaml.load(f, Loader=yaml.Loader) @@ -52,6 +58,22 @@ def setUpClass(cls): k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") k8s.wait_for_pod_start('spilo-role=master') + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_multi_namespace_support(self): + ''' + Create a customized Postgres cluster in a non-default namespace. + ''' + k8s = self.k8s + + with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + pg_manifest = yaml.load(f, Loader=yaml.Loader) + pg_manifest["metadata"]["namespace"] = self.namespace + yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, version="acid-test-cluster") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): """ @@ -190,14 +212,14 @@ def test_logical_backup_cron_job(self): self.assertTrue(0 == len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) - def assert_master_is_unique(self, namespace='default'): + def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): """ Check that there is a single pod in the k8s cluster with the label "spilo-role=master" To be called manually after operations that affect pods """ k8s = self.k8s - labels = 'spilo-role=master,version=acid-minimal-cluster' + labels = 'spilo-role=master,version=' + version num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) From a64c35598566e94a252f5cc80a6d107a4bd2a754 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 16:08:08 +0200 Subject: [PATCH 84/92] bump up waiting period for the operator pod --- e2e/tests/test_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 0db4080db..8d65f71ce 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -264,7 +264,8 @@ def get_spilo_nodes(self, pod_labels, namespace='default'): def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") # HACK operator must register CRD / add existing PG clusters after pod start up - time.sleep(10) + # for local execution 10 suffices + time.sleep(30) def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' From 9d1939d1ce4976ce6f0651cf69d65d0e97759e21 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 24 May 2019 16:21:11 +0200 Subject: [PATCH 85/92] bump up kind versions --- Makefile | 2 +- e2e/tests/test_e2e.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 643158061..5d97c817a 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ e2e-build: e2e-tools: # install pinned version of 'kind' # leave the name as is to avoid overwriting official binary named `kind` - wget https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64 + wget https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64 chmod +x kind-linux-amd64 mv kind-linux-amd64 $(KIND_PATH) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 8d65f71ce..a7415e3de 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -264,7 +264,7 @@ def get_spilo_nodes(self, pod_labels, namespace='default'): def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") # HACK operator must register CRD / add existing PG clusters after pod start up - # for local execution 10 suffices + # for local execution ~ 10 seconds suffices time.sleep(30) def wait_for_pod_start(self, pod_labels, namespace='default'): From bc05589712ef991bda69c8d33f72c5ac0cebff6e Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 29 May 2019 13:06:44 +0200 Subject: [PATCH 86/92] remove pinned package versions --- e2e/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 50c496239..bd646b677 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -8,10 +8,10 @@ COPY e2e/requirements.txt e2e/tests ./ RUN apt-get update \ && apt-get install --no-install-recommends -y \ - python3=3.6.7-1~18.04 \ - python3-setuptools=39.0.1-2 \ - python3-pip=9.0.1-2.3~ubuntu1 \ - curl=7.58.0-2ubuntu3.7 \ + python3 \ + python3-setuptools \ + python3-pip \ + curl \ && pip3 install --no-cache-dir -r requirements.txt \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ From 711598956a4e819d8a2fd5d20b9cfa4f4b695f8e Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 31 May 2019 17:27:59 +0200 Subject: [PATCH 87/92] move clean_up to trap --- e2e/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/run.sh b/e2e/run.sh index c18ee6c95..3ee272979 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -46,10 +46,12 @@ function clean_up(){ } function main(){ + + trap "clean_up" QUIT TERM EXIT + start_kind set_kind_api_server_ip run_tests - clean_up exit 0 } From 11f2a66f49557c7ac496ee712e5f9cc6dfce37d4 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 31 May 2019 17:42:03 +0200 Subject: [PATCH 88/92] use specific asserts and yaml.safe_load --- e2e/tests/test_e2e.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index a7415e3de..339161177 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -40,7 +40,7 @@ def setUpClass(cls): # submit the most recent operator image built on the Docker host with open("manifests/postgres-operator.yaml", 'r+') as f: - operator_deployment = yaml.load(f, Loader=yaml.Loader) + operator_deployment = yaml.safe_load(f) operator_deployment["spec"]["template"]["spec"]["containers"][0]["image"] = os.environ['OPERATOR_IMAGE'] yaml.dump(operator_deployment, f, Dumper=yaml.Dumper) @@ -66,7 +66,7 @@ def test_multi_namespace_support(self): k8s = self.k8s with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - pg_manifest = yaml.load(f, Loader=yaml.Loader) + pg_manifest = yaml.safe_load(f) pg_manifest["metadata"]["namespace"] = self.namespace yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) @@ -129,9 +129,9 @@ def test_taint_based_eviction(self): k8s.wait_for_pod_start('spilo-role=replica') new_master_node, new_replica_nodes = k8s.get_spilo_nodes(labels) - self.assertTrue(current_master_node != new_master_node, + self.assertNotEqual(current_master_node, new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertTrue(num_replicas == len(new_replica_nodes), + self.assertEqual(num_replicas, len(new_replica_nodes), "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) self.assert_master_is_unique() @@ -170,13 +170,13 @@ def test_logical_backup_cron_job(self): k8s.wait_for_logical_backup_job_creation() jobs = k8s.get_logical_backup_job().items - self.assertTrue(1 == len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) job = jobs[0] - self.assertTrue(job.metadata.name == "logical-backup-acid-minimal-cluster", + self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", "Expected job name {}, found {}" .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertTrue(job.spec.schedule == schedule, + self.assertEqual(job.spec.schedule, schedule, "Expected {} schedule, found {}" .format(schedule, job.spec.schedule)) @@ -196,7 +196,7 @@ def test_logical_backup_cron_job(self): jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertTrue(actual_image == image, + self.assertEqual(actual_image, image, "Expected job image {}, found {}".format(image, actual_image)) # delete the logical backup cron job @@ -209,7 +209,7 @@ def test_logical_backup_cron_job(self): "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) k8s.wait_for_logical_backup_job_deletion() jobs = k8s.get_logical_backup_job().items - self.assertTrue(0 == len(jobs), + self.assertEqual(0, len(jobs), "Expected 0 logical backup jobs, found {}".format(len(jobs))) def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): From 8e16e3267d10689934946eed0c5ab7997f58dee3 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 31 May 2019 17:46:20 +0200 Subject: [PATCH 89/92] re-write condition in get_spilo_nodes --- e2e/tests/test_e2e.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 339161177..cffa95f32 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -254,9 +254,9 @@ def get_spilo_nodes(self, pod_labels, namespace='default'): replica_pod_nodes = [] podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels) for pod in podsList.items: - if ('spilo-role', 'master') in pod.metadata.labels.items(): + if pod.metadata.labels.get('spilo-role') == 'master': master_pod_node = pod.spec.node_name - elif ('spilo-role', 'replica') in pod.metadata.labels.items(): + elif pod.metadata.labels.get('spilo-role') == 'replica': replica_pod_nodes.append(pod.spec.node_name) return master_pod_node, replica_pod_nodes From f287f550e8a224b39ed06c3857d6daccf460b8e3 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 31 May 2019 18:16:10 +0200 Subject: [PATCH 90/92] introduce timeouts for individual tests --- e2e/tests/test_e2e.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index cffa95f32..4f9cd77ee 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -246,6 +246,8 @@ class K8s: Wraps around K8 api client and helper methods. ''' + RETRY_TIMEOUT_SEC = 5 + def __init__(self): self.api = K8sApi() @@ -265,7 +267,7 @@ def wait_for_operator_pod_start(self): self. wait_for_pod_start("name=postgres-operator") # HACK operator must register CRD / add existing PG clusters after pod start up # for local execution ~ 10 seconds suffices - time.sleep(30) + time.sleep(60) def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' @@ -273,6 +275,7 @@ def wait_for_pod_start(self, pod_labels, namespace='default'): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): @@ -286,7 +289,7 @@ def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): labels = 'version=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: - pass + time.sleep(self.RETRY_TIMEOUT_SEC) def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) @@ -301,13 +304,14 @@ def wait_for_master_failover(self, expected_master_nodes, namespace='default'): if pods: new_master_node = pods[0].spec.node_name pod_phase = pods[0].status.phase + time.sleep(self.RETRY_TIMEOUT_SEC) def get_logical_backup_job(self, namespace='default'): return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") def wait_for_logical_backup_job(self, expected_num_of_jobs): while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): - pass + time.sleep(self.RETRY_TIMEOUT_SEC) def wait_for_logical_backup_job_deletion(self): self.wait_for_logical_backup_job(expected_num_of_jobs=0) From 3433b7c988046962e8c1c54832506b079f910824 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Tue, 4 Jun 2019 15:24:49 +0200 Subject: [PATCH 91/92] fix flake8 violations --- e2e/tests/test_e2e.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 4f9cd77ee..ec3f6c6e8 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -130,9 +130,9 @@ def test_taint_based_eviction(self): new_master_node, new_replica_nodes = k8s.get_spilo_nodes(labels) self.assertNotEqual(current_master_node, new_master_node, - "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) + "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) self.assertEqual(num_replicas, len(new_replica_nodes), - "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) + "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) self.assert_master_is_unique() # undo the tainting @@ -174,11 +174,11 @@ def test_logical_backup_cron_job(self): job = jobs[0] self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) # update the cluster-wide image of the logical backup pod image = "test-image-name" @@ -197,7 +197,7 @@ def test_logical_backup_cron_job(self): jobs = k8s.get_logical_backup_job().items actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) + "Expected job image {}, found {}".format(image, actual_image)) # delete the logical backup cron job pg_patch_disable_backup = { @@ -210,7 +210,7 @@ def test_logical_backup_cron_job(self): k8s.wait_for_logical_backup_job_deletion() jobs = k8s.get_logical_backup_job().items self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + "Expected 0 logical backup jobs, found {}".format(len(jobs))) def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"): """ From 89163e704056526c34770a52061219ba31d97596 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 5 Jun 2019 16:39:24 +0200 Subject: [PATCH 92/92] update signature for getting nodes of pg cluster --- e2e/tests/test_e2e.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index ec3f6c6e8..c232ba7ac 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -97,10 +97,10 @@ def test_taint_based_eviction(self): Add taint "postgres=:NoExecute" to node with master. This must cause a failover. """ k8s = self.k8s - labels = 'version=acid-minimal-cluster' + cluster_label = 'version=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) - current_master_node, failover_targets = k8s.get_spilo_nodes(labels) + current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label) num_replicas = len(failover_targets) # if all pods live on the same node, failover will happen to other worker(s) @@ -128,7 +128,7 @@ def test_taint_based_eviction(self): k8s.wait_for_master_failover(failover_targets) k8s.wait_for_pod_start('spilo-role=replica') - new_master_node, new_replica_nodes = k8s.get_spilo_nodes(labels) + new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) self.assertNotEqual(current_master_node, new_master_node, "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) self.assertEqual(num_replicas, len(new_replica_nodes), @@ -251,10 +251,10 @@ class K8s: def __init__(self): self.api = K8sApi() - def get_spilo_nodes(self, pod_labels, namespace='default'): + def get_pg_nodes(self, pg_cluster_name, namespace='default'): master_pod_node = '' replica_pod_nodes = [] - podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels) + podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pg_cluster_name) for pod in podsList.items: if pod.metadata.labels.get('spilo-role') == 'master': master_pod_node = pod.spec.node_name