Skip to content
This repository has been archived by the owner on Jan 18, 2024. It is now read-only.

Commit

Permalink
*: tooling alignment with tobs
Browse files Browse the repository at this point in the history
* removal of artisian python scripts
* introduction if chart-testing tool
* improvement in shellcheck make target
* simplification of Makefile

Signed-off-by: Paweł Krupa (paulfantom) <[email protected]>
  • Loading branch information
paulfantom committed Aug 16, 2022
1 parent f6f09ff commit c216dfa
Show file tree
Hide file tree
Showing 13 changed files with 236 additions and 143 deletions.
12 changes: 11 additions & 1 deletion .github/renovate.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,23 @@
"matchStrings": ["helm-version:\\s(?<currentValue>.*?)\\n"],
"datasourceTemplate": "github-tags",
"depNameTemplate": "helm/helm"
},
{
"fileMatch": ["^\\.github\\/workflows\\/[^/]+\\.ya?ml$"],
"matchStrings": ["go-version:\\s(?<currentValue>.*?)\\n"],
"datasourceTemplate": "golang-version",
"depNameTemplate": "golang"
}
],
"packageRules": [
{
"addLabels": ["github_actions"],
"groupName": "github actions",
"matchPaths": [".github/**"]
},
{
"groupName": "helm charts",
"matchManagers": ["helmv3","helm-values"]
}
]
}
}
25 changes: 1 addition & 24 deletions .github/workflows/push.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "Test Helm Charts"
name: "Test [old]"
on:
- push
- pull_request
Expand All @@ -7,29 +7,6 @@ env:
helm-version: v3.8.1

jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Copy example values files to the expected Helm Linter locations
run: make refresh-ci-values

- name: Run linter on Helm Charts
id: lint
uses: helm/[email protected]
with:
command: lint

- name: Install shellcheck
run: sudo apt-get install shellcheck

- name: Run shellcheck on all scripts
run: make shellcheck

test:
runs-on: ubuntu-latest
steps:
Expand Down
91 changes: 91 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
name: "Test Helm Charts"
on:
- push
- pull_request

env:
go-version: 1.19.0

jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0

- uses: actions/setup-go@v3
with:
go-version: ${{ env.go-version }}

- name: Install gojsontoyaml
run: go install github.com/brancz/gojsontoyaml@latest

- name: Set up chart-testing
uses: helm/[email protected]

- name: Run chart-testing (lint)
run: make lint

- name: Run shellcheck
run: make shellcheck

generate:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0

- uses: actions/setup-go@v3
with:
go-version: ${{ env.go-version }}

- name: Install gojsontoyaml
run: go install github.com/brancz/gojsontoyaml@latest

- name: Generate JSON Schema
run: make json-schema && git diff --exit-code

test:
runs-on: ubuntu-latest
strategy:
matrix:
kube:
- "1.23"
- "1.24"
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set up chart-testing
uses: helm/[email protected]

- name: Start kuberenetes cluster
env:
KUBE_VERSION: ${{ matrix.kube }}
run: |
make start-kind
- name: Wait for cluster to finish bootstraping
run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s

- name: Run e2e chart-testing
run: make e2e

test-result:
name: End-to-End Test Results
if: always()
needs:
- test
runs-on: ubuntu-latest
steps:
- name: Mark the job as a success
if: needs.test.result == 'success'
run: exit 0
- name: Mark the job as a failure
if: needs.test.result != 'success'
run: exit 1
112 changes: 48 additions & 64 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,86 +1,68 @@
SINGLE_CHART_DIR := charts/timescaledb-single
MULTI_CHART_DIR := charts/timescaledb-multinode
CI_SINGLE_DIR := $(SINGLE_CHART_DIR)/ci/
SINGLE_VALUES_FILES := $(SINGLE_CHART_DIR)/values.yaml $(wildcard $(SINGLE_CHART_DIR)/values/*.yaml)
DEPLOYMENTS := $(SINGLE_VALUES_FILES)
K8S_NAMESPACE ?= citest
SHELL=/bin/bash -euo pipefail

export PULL_TIMEOUT ?= 3000s
export DEPLOYMENT_TIMEOUT ?= 600s
KUBE_VERSION ?= 1.23
KIND_CONFIG ?= ./testdata/kind-$(KUBE_VERSION).yaml

.PHONY: publish
publish: publish-multinode publish-single
TMP_DIR=tmp

.PHONY: publish-multinode
publish-multinode:
helm package charts/timescaledb-multinode --destination charts/repo
helm repo index charts/repo
.PHONY: json-schema
json-schema:
find charts/ -name values.schema.yaml -printf 'cat %p | gojsontoyaml -yamltojson | jq -r > $$(dirname %p)/values.schema.json' | sh

.PHONY: publish-single
publish-single:
helm package charts/timescaledb-single --destination charts/repo
helm repo index charts/repo
.PHONY: lint
lint: ## Lint helm chart using ct (chart-testing).
ct lint --config ct.yaml

.PHONY: clean
clean: clean-ci
clean:
rm -rf $(TMP_DIR)

.PHONY: assert-schema-equals
assert-schema-equals:
@cat $(SINGLE_CHART_DIR)/values.schema.yaml | python3 ./yaml2json.py | jq '.[0]' | git --no-pager diff --no-index - $(SINGLE_CHART_DIR)/values.schema.json
$(TMP_DIR):
mkdir -p $(TMP_DIR)

.PHONY: json-schema
json-schema:
cat $(SINGLE_CHART_DIR)/values.schema.yaml | python3 ./yaml2json.py | jq '.[0]' > $(SINGLE_CHART_DIR)/values.schema.json
.PHONY: extract-scripts
extract-scripts: $(TMP_DIR) ## Extract shell scripts from helm templates
./scripts/extract-scripts.sh

.PHONY: lint
lint: assert-schema-equals
@for file in $(SINGLE_VALUES_FILES); do \
echo "Linting timescaledb-single using file: $$file" ; \
helm lint $(SINGLE_CHART_DIR) -f "$$file" --set backup.enabled=true --set pgBouncer.enabled=true --set prometheus.enabled=true --set unsafe=true || exit 1 ; \
done
@helm lint $(MULTI_CHART_DIR)

# We're not symlinking the files, as that generates *a ton* of Helm noise
.PHONY: refresh-ci-values
refresh-ci-values: clean-ci
@mkdir -p ./$(CI_SINGLE_DIR)/
@for file in $(SINGLE_VALUES_FILES); do \
cp "$$file" "$(CI_SINGLE_DIR)/$$(basename $$file)-values.yaml"; \
.PHONY: shellcheck
shellcheck: extract-scripts
for f in $$(find scripts/ -name "*.sh" -type f) $$(find $(TMP_DIR)/ -name "*.sh" -type f); do \
shellcheck $$f --exclude=SC1090,SC1091,SC2148 ;\
done

.PHONY: clean-ci
clean-ci:
@rm -rf ./$(CI_SINGLE_DIR)
@mkdir -p ./$(CI_SINGLE_DIR)/
@kubectl delete namespace $(K8S_NAMESPACE) 2>/dev/null || true

.PHONY: shellcheck shellcheck-single
shellcheck: shellcheck-single
shellcheck-single:
@for vfile in $(SINGLE_VALUES_FILES); do \
helm template $(SINGLE_CHART_DIR) -s templates/configmap-scripts.yaml -f $$vfile > $(CI_SINGLE_DIR)/temp.yaml || exit 1 ; \
cat $(CI_SINGLE_DIR)/temp.yaml | python3 ./yaml2json.py > $(CI_SINGLE_DIR)/temp.json || exit 1; \
for script in $$(jq '.[0].data | keys | sort | .[] | select(endswith(".sh"))' -r $(CI_SINGLE_DIR)/temp.json); do \
echo "shellcheck - $$script (from values file $$vfile)" ; \
jq ".[0].data.\"$${script}\"" -r $(CI_SINGLE_DIR)/temp.json | shellcheck - --exclude=SC1090 || exit 1; \
done ; \
rm $(CI_SINGLE_DIR)/temp.* ; \
done
.PHONY: delete-kind
delete-kind: ## This is a phony target that is used to delete the local kubernetes kind cluster.
kind delete cluster && sleep 10

.PHONY: start-kind
start-kind: delete-kind ## This is a phony target that is used to create a local kubernetes kind cluster.
kind create cluster --config $(KIND_CONFIG)
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s

.PHONY: load-images
load-images: ## Load images into the local kubernetes kind cluster.
./scripts/load-images.sh

.PHONY: e2e
e2e: load-images ## Run e2e installation tests using ct (chart-testing).
# TODO(paulfantom): re-enable when it is possible to use `ct` to test the chart
# Chart is still tested using older method via a shell script
ct install --config ct.yaml --exclude-deprecated --excluded-charts=timescaledb-single


.PHONY: install-example
install-example: prepare-ci
DELETE_DEPLOYMENT=0 TEST_REPLICA=0 ./tests/verify_deployment.sh $(SINGLE_CHART_DIR)/values.yaml example
### TODO(paulfantom): remove this section once timescaledb-single is using `ct` and `helm test` for testing.

.PHONY: smoketest
smoketest: prepare-ci
./tests/verify_deployment.sh $(SINGLE_CHART_DIR)/values.yaml smoketest
SINGLE_CHART_DIR := charts/timescaledb-single
SINGLE_VALUES_FILES := $(SINGLE_CHART_DIR)/values.yaml $(wildcard $(SINGLE_CHART_DIR)/values/*.yaml)
DEPLOYMENTS := $(SINGLE_VALUES_FILES)
K8S_NAMESPACE ?= citest

# We test sequentially, as in GitHub actions we do not have a lot of CPU available,
# so scheduling the pods concurrently does not work
.PHONY: test
test: prepare-ci
for f in $(SINGLE_VALUES_FILES); do \
./tests/verify_deployment.sh $${f} || exit 1; \
./tests/verify_deployment.sh $${f}; \
done

.PHONY: prepare-ci
Expand All @@ -98,3 +80,5 @@ prepare-ci:
| kubectl create -f - ; \
done ; \
exit 0

### END_OF_SECTION
26 changes: 9 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
# TimescaleDB Kubernetes
[![Security Audit](https://github.com/timescale/helm-charts/actions/workflows/sec-audit.yml/badge.svg)](https://github.com/timescale/helm-charts/actions/workflows/sec-audit.yml)
[![Test Helm Charts](https://github.com/timescale/helm-charts/actions/workflows/tests.yml/badge.svg)](https://github.com/timescale/helm-charts/actions/workflows/tests.yml)
[![Commit activity](https://img.shields.io/github/commit-activity/m/timescale/helm-charts)](https://github.com/timescale/helm-charts/pulse/monthly)
[![License](https://img.shields.io/github/license/timescale/helm-charts)](https://github.com/timescale/helm-charts/blob/main/LICENSE)
[![Slack](https://img.shields.io/badge/chat-join%20slack-brightgreen.svg)](https://timescaledb.slack.com/)

# Timescale Helm Charts

This repository contains Helm charts to help with the deployment of
[TimescaleDB](https://github.com/timescale/timescaledb/) on Kubernetes. This
[TimescaleDB](https://github.com/timescale/timescaledb/) and [Promscale](https://github.com/timescale/promscale) on Kubernetes. This
project is currently in active development.

Supported Versions:
- Kubernetes: [AWS EKS](https://aws.amazon.com/eks/), [MicroK8s](https://microk8s.io/), and [minikube](https://github.com/kubernetes/minikube/releases)

# Recipes

| Recipe| TimescaleDB | Description |
|---|---|---|
| [TimescaleDB Single](charts/timescaledb-single) | Based on 1.x | TimescaleDB Single allows you to deploy a highly-available TimescaleDB database configuration. |
| [TimescaleDB Multinode](charts/timescaledb-multinode) | In Development | TimescaleDB Multinode allows you to deploy a multi-node, distributed version of TimescaleDB. |

# Repository

The Charts are available in a Helm Chart Repository, which is part of this project.
Expand All @@ -30,13 +26,9 @@ For more information, have a look at the [Using Helm](https://helm.sh/docs/intro
- [Migrating data to TimescaleDB](https://docs.timescale.com/getting-started/migrating-data)
- [Tutorials and sample data](https://docs.timescale.com/tutorials)

## Get Help
- [Slack Channel](https://slack.timescale.com/)
- [GitHub Issues](https://github.com/timescale/timescaledb-kubernetes/issues)

## License

TimescaleDB Kubernetes resources in this repository are released under the [Apache 2.0 license](LICENSE).
Resources in this repository are released under the [Apache 2.0 license](LICENSE).

## Contributing

Expand Down
19 changes: 0 additions & 19 deletions RELEASE.md

This file was deleted.

5 changes: 2 additions & 3 deletions ct.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@
remote: origin
target-branch: master
chart-dirs:
- deploy
- charts
chart-repos: []
# Some of tests require longer than default timeout
helm-extra-args: --timeout 600s
helm-extra-args: --timeout 15m --debug
# We do not want to bump the Chart version for every single commit,
# but only once we release
check-version-increment: false

20 changes: 20 additions & 0 deletions scripts/extract-scripts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

# Set this after CI detection to prevent "unbound variable" error
set -euo pipefail

cd "$(git rev-parse --show-toplevel)"

TMP_DIR=tmp

SINGLE_VALUES_FILES="charts/timescaledb-single/values.yaml $(ls charts/timescaledb-single/ci/*.yaml)"

for vfile in $SINGLE_VALUES_FILES; do \
scripts_dir="${TMP_DIR}/$(basename "${vfile}")"
mkdir -p "${scripts_dir}"; \
helm template charts/timescaledb-single -s templates/configmap-scripts.yaml -f "${vfile}" | gojsontoyaml -yamltojson > "${scripts_dir}/temp.json"
for script in $(jq '.data | keys | sort | .[]' -r "${scripts_dir}/temp.json"); do
echo "Extracting script: ${vfile} / ${script}"
jq -r --arg script "${script}" '.data[$script]' "${scripts_dir}/temp.json" > "${scripts_dir}/${script}"
done
done
Loading

0 comments on commit c216dfa

Please sign in to comment.