diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index cb7315c815dd1..e65dddb777006 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -480,6 +480,36 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') { }, ], }, + pipeline('helm-test-image') { + workspace: { + base: '/src', + path: 'loki', + }, + steps: [ + { + name: 'test-image', + image: 'plugins/docker', + when: onPRs + onPath('production/helm/loki/src/helm-test/**'), + settings: { + repo: 'grafana/loki-helm-test', + dockerfile: 'production/helm/loki/src/helm-test/Dockerfile', + dry_run: true, + }, + }, + { + name: 'push-image', + image: 'plugins/docker', + when: onTagOrMain + onPath('production/helm/loki/src/helm-test/**'), + settings: { + repo: 'grafana/loki-helm-test', + dockerfile: 'production/helm/loki/src/helm-test/Dockerfile', + username: { from_secret: docker_username_secret.name }, + password: { from_secret: docker_password_secret.name }, + dry_run: false, + }, + }, + ], + }, pipeline('check') { workspace: { base: '/src', diff --git a/.drone/drone.yml b/.drone/drone.yml index 57be3f88a920c..b6535fd712b2f 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -46,6 +46,46 @@ workspace: path: loki --- kind: pipeline +name: helm-test-image +steps: +- image: plugins/docker + name: test-image + settings: + dockerfile: production/helm/loki/src/helm-test/Dockerfile + dry_run: true + repo: grafana/loki-helm-test + when: + event: + - pull_request + paths: + - production/helm/loki/src/helm-test/** +- image: plugins/docker + name: push-image + settings: + dockerfile: production/helm/loki/src/helm-test/Dockerfile + dry_run: false + password: + from_secret: docker_password + repo: grafana/loki-helm-test + username: + from_secret: docker_username + when: + event: + - push + - tag + paths: + - production/helm/loki/src/helm-test/** +trigger: + ref: + - refs/heads/main + - refs/heads/k??? + - refs/tags/v* + - refs/pull/*/head +workspace: + base: /src + path: loki +--- +kind: pipeline name: check steps: - commands: @@ -1535,8 +1575,3 @@ get: path: infra/data/ci/packages-publish/gpg kind: secret name: gpg_private_key ---- -kind: signature -hmac: 8738700b68f651859a25c2c9bd9f70efb8092aa1eca2ad35238a85ec19f9fe31 - -... diff --git a/.github/workflows/helm-ci.yml b/.github/workflows/helm-ci.yml index 691af650b78aa..6db1ceb37fe35 100644 --- a/.github/workflows/helm-ci.yml +++ b/.github/workflows/helm-ci.yml @@ -3,17 +3,96 @@ name: helm-ci on: pull_request: paths: - - 'production/helm/**' + - "production/helm/**" + +env: + CT_CONFIGFILE: production/helm/ct.yaml jobs: call-lint: - uses: grafana/helm-charts/.github/workflows/linter.yml@main - with: - filter_regex_include: .*production/helm/.* - - call-lint-test: - uses: grafana/helm-charts/.github/workflows/lint-test.yaml@main - with: - ct_configfile: production/helm/ct.yaml - ct_check_version_increment: false - helm_version: v3.8.2 + name: Lint Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Check Docs + run: | + docker run --rm --volume "$(pwd):/helm-docs" -u "$(id -u)" jnorwood/helm-docs:v1.8.1 + if ! git diff --exit-code; then + echo "Documentation not up to date. Please run helm-docs and commit changes!" >&2 + exit 1 + fi + + - name: Lint Code Base + uses: docker://github/super-linter:v3.12.0 + env: + FILTER_REGEX_EXCLUDE: .*(README\.md|Chart\.yaml|NOTES.txt).* + FILTER_REGEX_INCLUDE: .*production/helm/.* + VALIDATE_ALL_CODEBASE: false + VALIDATE_KUBERNETES_KUBEVAL: false + VALIDATE_YAML: false + VALIDATE_GO: false + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + call-test: + name: Test Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.8.2 + + # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and + # yamllint (https://github.com/adrienverge/yamllint) which require Python + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.1.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config "${CT_CONFIGFILE}") + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run chart-testing (lint) + run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false + + - name: Create kind cluster + uses: helm/kind-action@v1.2.0 + if: steps.list-changed.outputs.changed == 'true' + + - name: Install prometheus operator + id: install-prometheus + if: steps.list-changed.outputs.changed == 'true' + run: | + kubectl create namespace prometheus + + helm install prometheus prometheus-community/kube-prometheus-stack \ + --namespace prometheus \ + --set grafana.enabled=false \ + --set prometheus.prometheusSpec.serviceMonitorSelector.matchLabels.release=prometheus + + kubectl --namespace prometheus get pods -l "release=prometheus" + kubectl --namespace prometheus get services -l "release=prometheus" + + - name: Run chart-testing (install) + run: | + changed=$(ct list-changed --config "${CT_CONFIGFILE}") + if [[ "$changed" == "charts/enterprise-metrics" ]]; then + # Do not run `ct install` for enterprise-metrics + exit 0 + fi + ct install --config "${CT_CONFIGFILE}" diff --git a/.gitignore b/.gitignore index 98fe88c25af90..a0cc2550cb204 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ cmd/loki/loki-debug clients/cmd/promtail/promtail-debug clients/cmd/docker-driver/docker-driver cmd/loki-canary/loki-canary +cmd/loki-canary-test/loki-canary-test clients/cmd/fluent-bit/out_grafana_loki.so clients/cmd/fluent-bit/out_grafana_loki.h cmd/migrate/migrate diff --git a/Makefile b/Makefile index 2dc36f42826ec..4dbc628bc9e06 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,16 @@ loki-canary: cmd/loki-canary/loki-canary cmd/loki-canary/loki-canary: CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) +############### +# Helm-Test # +############### +.PHONY: production/helm/loki/src/helm-test/helm-test +helm-test: production/helm/loki/src/helm-test/helm-test + +# Package Helm tests but do not run them. +production/helm/loki/src/helm-test/helm-test: + CGO_ENABLED=0 go test $(GO_FLAGS) --tags=helm_test -c -o $@ ./$(@D) + ################# # Loki-QueryTee # ################# @@ -485,7 +495,7 @@ push-bigtable-backup: bigtable-backup # Images # ########## -images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image fluentd-image +images: promtail-image loki-image loki-canary-image helm-test-image docker-driver fluent-bit-image fluentd-image # push(app, optional tag) # pushes the app, optionally tagging it differently before @@ -534,6 +544,10 @@ loki-canary-image-cross: $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) -f cmd/loki-canary/Dockerfile.cross . loki-canary-push: loki-canary-image-cross $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) +helm-test-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) -f production/helm/loki/src/helm-test/Dockerfile . +helm-test-push: helm-test-image + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-helm-test:$(IMAGE_TAG) # loki-querytee loki-querytee-image: diff --git a/flake.nix b/flake.nix index 5e6535925c163..4f905b756c6b8 100644 --- a/flake.nix +++ b/flake.nix @@ -79,18 +79,23 @@ config = { allowUnfree = true; }; }; in - with pkgs; { + { # The default package for 'nix build'. This makes sense if the # flake provides only one package or there is a clear "main" # package. - defaultPackage = loki; + defaultPackage = pkgs.loki; - packages = { inherit loki; }; + packages = with pkgs; { + inherit + loki + loki-helm-test + loki-helm-test-docker; + }; apps = { lint = { type = "app"; - program = "${ + program = with pkgs; "${ (writeShellScriptBin "lint.sh" '' ${nixpkgs-fmt}/bin/nixpkgs-fmt --check ${self}/flake.nix ${self}/nix/*.nix ${statix}/bin/statix check ${self} @@ -114,10 +119,14 @@ type = "app"; program = with pkgs; "${loki.overrideAttrs(old: rec { doCheck = false; })}/bin/loki-canary"; }; + loki-helm-test = { + type = "app"; + program = with pkgs; "${loki-helm-test}/bin/helm-test"; + }; }; - devShell = mkShell { - nativeBuildInputs = [ + devShell = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ gcc go systemd diff --git a/nix/default.nix b/nix/default.nix index c1d00204110eb..c80348c2b987d 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,45 +1,53 @@ { self, nixpkgs, system }: let buildVars = import ./build-vars.nix; -in { - overlay = final: prev: rec { - loki = - let - # self.rev is only set on a clean git tree - gitRevision = if (self ? rev) then self.rev else "dirty"; - shortGitRevsion = with prev.lib; - if (self ? rev) then - (strings.concatStrings - (lists.take 8 (strings.stringToCharacters gitRevision))) - else - "dirty"; +in +{ + overlay = final: prev: + let + # self.rev is only set on a clean git tree + gitRevision = if (self ? rev) then self.rev else "dirty"; + shortGitRevsion = with prev.lib; + if (self ? rev) then + (strings.concatStrings + (lists.take 8 (strings.stringToCharacters gitRevision))) + else + "dirty"; - # the image tag script is hard coded to take only 7 characters - imageTagVersion = with prev.lib; - if (self ? rev) then - (strings.concatStrings - (lists.take 8 (strings.stringToCharacters gitRevision))) - else - "dirty"; + # the image tag script is hard coded to take only 7 characters + imageTagVersion = with prev.lib; + if (self ? rev) then + (strings.concatStrings + (lists.take 8 (strings.stringToCharacters gitRevision))) + else + "dirty"; - imageTag = - if (self ? rev) then - "${buildVars.gitBranch}-${imageTagVersion}" - else - "${buildVars.gitBranch}-${imageTagVersion}-WIP"; - in - prev.callPackage ./loki.nix { + imageTag = + if (self ? rev) then + "${buildVars.gitBranch}-${imageTagVersion}" + else + "${buildVars.gitBranch}-${imageTagVersion}-WIP"; + + loki-helm-test = prev.callPackage ../production/helm/loki/src/helm-test { + inherit (prev) pkgs lib buildGoModule dockerTools; + rev = gitRevision; + }; + in + { + inherit (loki-helm-test) loki-helm-test loki-helm-test-docker; + + loki = prev.callPackage ./loki.nix { inherit imageTag; inherit (buildVars) gitBranch; version = shortGitRevsion; pkgs = prev; }; - faillint = prev.callPackage ./faillint.nix { - inherit (prev) lib buildGoModule fetchFromGitHub; - }; + faillint = prev.callPackage ./faillint.nix { + inherit (prev) lib buildGoModule fetchFromGitHub; + }; - chart-releaser = prev.callPackage ./chart-releaser.nix { - inherit (prev) pkgs lib buildGoModule fetchFromGitHub; + chart-releaser = prev.callPackage ./chart-releaser.nix { + inherit (prev) pkgs lib buildGoModule fetchFromGitHub; + }; }; - }; } diff --git a/production/helm/ct.yaml b/production/helm/ct.yaml index 3fdd21999527e..b3bb9ec213e5a 100644 --- a/production/helm/ct.yaml +++ b/production/helm/ct.yaml @@ -6,6 +6,7 @@ chart-dirs: chart-repos: - grafana=https://grafana.github.io/helm-charts - minio=https://charts.min.io + - prometheus-community=https://prometheus-community.github.io/helm-charts helm-extra-args: --timeout 600s check-version-increment: false validate-maintainers: false diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 1831801242ea6..d25f5b70f84f2 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -4,7 +4,7 @@ name: loki description: Helm chart for Grafana Loki in simple, scalable mode type: application appVersion: 2.6.1 -version: 3.2.2 +version: 3.3.0 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index b11b9e03c211a..697a8e60e487b 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 3.2.2](https://img.shields.io/badge/Version-3.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.6.1](https://img.shields.io/badge/AppVersion-2.6.1-informational?style=flat-square) +![Version: 3.3.0](https://img.shields.io/badge/Version-3.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.6.1](https://img.shields.io/badge/AppVersion-2.6.1-informational?style=flat-square) Helm chart for Grafana Loki in simple, scalable mode @@ -153,7 +153,7 @@ monitoring: | enterprise.image.repository | string | `"grafana/enterprise-logs"` | Docker image repository | | enterprise.image.tag | string | `"v1.4.0"` | Overrides the image tag whose default is the chart's appVersion | | enterprise.license | object | `{"contents":"NOTAVALIDLICENSE"}` | Grafana Enterprise Logs license In order to use Grafana Enterprise Logs features, you will need to provide the contents of your Grafana Enterprise Logs license, either by providing the contents of the license.jwt, or the name Kubernetes Secret that contains your license.jwt. To set the license contents, use the flag `--set-file 'license.contents=./license.jwt'` | -| enterprise.nginxConfig.file | string | `"worker_processes 5; ## Default: 1\nerror_log /dev/stderr;\npid /tmp/nginx.pid;\nworker_rlimit_nofile 8192;\n\nevents {\n worker_connections 4096; ## Default: 1024\n}\n\nhttp {\n client_body_temp_path /tmp/client_temp;\n proxy_temp_path /tmp/proxy_temp_path;\n fastcgi_temp_path /tmp/fastcgi_temp;\n uwsgi_temp_path /tmp/uwsgi_temp;\n scgi_temp_path /tmp/scgi_temp;\n\n proxy_http_version 1.1;\n\n default_type application/octet-stream;\n log_format {{ .Values.gateway.nginxConfig.logFormat }}\n\n {{- if .Values.gateway.verboseLogging }}\n access_log /dev/stderr main;\n {{- else }}\n\n map $status $loggable {\n ~^[23] 0;\n default 1;\n }\n access_log /dev/stderr main if=$loggable;\n {{- end }}\n\n sendfile on;\n tcp_nopush on;\n resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }};\n\n {{- with .Values.gateway.nginxConfig.httpSnippet }}\n {{ . | nindent 2 }}\n {{- end }}\n\n server {\n listen 8080;\n\n {{- if .Values.gateway.basicAuth.enabled }}\n auth_basic \"Loki\";\n auth_basic_user_file /etc/nginx/secrets/.htpasswd;\n {{- end }}\n\n location = / {\n return 200 'OK';\n auth_basic off;\n }\n\n location = /api/prom/push {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /api/prom/tail {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n\n location ~ /api/prom/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /prometheus/api/v1/alerts.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /prometheus/api/v1/rules.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /loki/api/v1/push {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /loki/api/v1/tail {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n\n location ~ /loki/api/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /admin/api/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /compactor/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /distributor/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ring {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ingester/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ruler/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /scheduler/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n {{- with .Values.gateway.nginxConfig.serverSnippet }}\n {{ . | nindent 4 }}\n {{- end }}\n }\n}\n"` | | +| enterprise.nginxConfig.file | string | `"worker_processes 5; ## Default: 1\nerror_log /dev/stderr;\npid /tmp/nginx.pid;\nworker_rlimit_nofile 8192;\n\nevents {\n worker_connections 4096; ## Default: 1024\n}\n\nhttp {\n client_body_temp_path /tmp/client_temp;\n proxy_temp_path /tmp/proxy_temp_path;\n fastcgi_temp_path /tmp/fastcgi_temp;\n uwsgi_temp_path /tmp/uwsgi_temp;\n scgi_temp_path /tmp/scgi_temp;\n\n proxy_http_version 1.1;\n\n default_type application/octet-stream;\n log_format {{ .Values.gateway.nginxConfig.logFormat }}\n\n {{- if .Values.gateway.verboseLogging }}\n access_log /dev/stderr main;\n {{- else }}\n\n map $status $loggable {\n ~^[23] 0;\n default 1;\n }\n access_log /dev/stderr main if=$loggable;\n {{- end }}\n\n sendfile on;\n tcp_nopush on;\n resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}.;\n\n {{- with .Values.gateway.nginxConfig.httpSnippet }}\n {{ . | nindent 2 }}\n {{- end }}\n\n server {\n listen 8080;\n\n {{- if .Values.gateway.basicAuth.enabled }}\n auth_basic \"Loki\";\n auth_basic_user_file /etc/nginx/secrets/.htpasswd;\n {{- end }}\n\n location = / {\n return 200 'OK';\n auth_basic off;\n }\n\n location = /api/prom/push {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /api/prom/tail {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n\n location ~ /api/prom/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /prometheus/api/v1/alerts.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /prometheus/api/v1/rules.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /loki/api/v1/push {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location = /loki/api/v1/tail {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n\n location ~ /loki/api/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /admin/api/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /compactor/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /distributor/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ring {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ingester/.* {\n proxy_pass http://{{ include \"loki.writeFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /ruler/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n location ~ /scheduler/.* {\n proxy_pass http://{{ include \"loki.readFullname\" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;\n }\n\n {{- with .Values.gateway.nginxConfig.serverSnippet }}\n {{ . | nindent 4 }}\n {{- end }}\n }\n}\n"` | | | enterprise.provisioner | object | `{"annotations":{},"enabled":true,"env":[],"image":{"pullPolicy":"IfNotPresent","registry":"docker.io","repository":"grafana/enterprise-logs-provisioner","tag":null},"labels":{},"priorityClassName":null,"provisionedSecretPrefix":"{{ include \"loki.name\" . }}-provisioned","securityContext":{"fsGroup":10001,"runAsGroup":10001,"runAsNonRoot":true,"runAsUser":10001},"tenants":[]}` | Configuration for `provisioner` target | | enterprise.provisioner.annotations | object | `{}` | Additional annotations for the `provisioner` Job | | enterprise.provisioner.enabled | bool | `true` | Whether the job should be part of the deployment | @@ -414,6 +414,16 @@ monitoring: | singleBinary.selectorLabels | object | `{}` | Additional selecto labels for each `single binary` pod | | singleBinary.terminationGracePeriodSeconds | int | `30` | Grace period to allow the single binary to shutdown before it is killed | | singleBinary.tolerations | list | `[]` | Tolerations for single binary pods | +| test | object | `{"annotations":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","registry":"docker.io","repository":"grafana/loki-helm-test","tag":null},"labels":{},"prometheusAddress":"http://prometheus:9090","timeout":"1m"}` | Section for configuring optional Helm test | +| test.annotations | object | `{}` | Additional annotations for test pods | +| test.image | object | `{"pullPolicy":"IfNotPresent","registry":"docker.io","repository":"grafana/loki-helm-test","tag":null}` | Image to use for loki canary | +| test.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | +| test.image.registry | string | `"docker.io"` | The Docker registry | +| test.image.repository | string | `"grafana/loki-helm-test"` | Docker image repository | +| test.image.tag | string | `nil` | Overrides the image tag whose default is the chart's appVersion | +| test.labels | object | `{}` | Additional labels for the test pods | +| test.prometheusAddress | string | `"http://prometheus:9090"` | Address of the prometheus server to query for the test | +| test.timeout | string | `"1m"` | Number of times to retry the test before failing | | tracing.jaegerAgentHost | string | `""` | | | write.affinity | string | Hard node and soft zone anti-affinity | Affinity for write pods. Passed through `tpl` and, thus, to be configured as string | | write.extraArgs | list | `[]` | Additional CLI args for the write | diff --git a/production/helm/loki/ci/default-values.yaml b/production/helm/loki/ci/default-values.yaml index cd0b536b6df6d..6fe3c1014a737 100644 --- a/production/helm/loki/ci/default-values.yaml +++ b/production/helm/loki/ci/default-values.yaml @@ -6,3 +6,9 @@ read: replicas: 1 write: replicas: 1 +monitoring: + serviceMonitor: + labels: + release: "prometheus" +test: + prometheusAddress: "http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local.:9090" diff --git a/production/helm/loki/ci/enterprise.yaml b/production/helm/loki/ci/enterprise.yaml index 30e90a0f65c96..98b95148d30eb 100644 --- a/production/helm/loki/ci/enterprise.yaml +++ b/production/helm/loki/ci/enterprise.yaml @@ -23,3 +23,9 @@ write: persistence: enabled: true size: 100Mi +monitoring: + serviceMonitor: + labels: + release: "prometheus" +test: + prometheusAddress: "http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local.:9090" diff --git a/production/helm/loki/ci/ingress-values.yaml b/production/helm/loki/ci/ingress-values.yaml index 6d0692579de56..b78242f0c1add 100644 --- a/production/helm/loki/ci/ingress-values.yaml +++ b/production/helm/loki/ci/ingress-values.yaml @@ -15,3 +15,9 @@ read: replicas: 1 write: replicas: 1 +monitoring: + selfMonitoring: + lokiCanary: + enabled: false +test: + enabled: false diff --git a/production/helm/loki/ci/persistence-values.yaml b/production/helm/loki/ci/persistence-values.yaml deleted file mode 100644 index 0221b170d580d..0000000000000 --- a/production/helm/loki/ci/persistence-values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -read: - replicas: 1 - persistence: - enabled: true - size: 100Mi - -write: - replicas: 1 - persistence: - enabled: true - size: 100Mi - -loki: - commonConfig: - replication_factor: 1 - -gateway: - nginxConfig: - httpSnippet: |- - client_max_body_size 100M; - serverSnippet: |- - client_max_body_size 100M; - - basicAuth: - enabled: true - username: user - password: pass diff --git a/production/helm/loki/src/helm-test/Dockerfile b/production/helm/loki/src/helm-test/Dockerfile new file mode 100644 index 0000000000000..5ffb228f70bc2 --- /dev/null +++ b/production/helm/loki/src/helm-test/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.18.5 as build + +# build via Makefile target helm-test-image in root +# Makefile. Building from this directory will not be +# able to access source needed in rest of repo. +COPY . /src/loki +WORKDIR /src/loki +RUN make clean && make BUILD_IN_CONTAINER=false helm-test + +FROM alpine:3.16.2 +RUN apk add --update --no-cache ca-certificates=20220614-r0 +COPY --from=build /src/loki/production/helm/loki/src/helm-test/helm-test /usr/bin/helm-test +ENTRYPOINT [ "/usr/bin/helm-test" ] diff --git a/production/helm/loki/src/helm-test/README.md b/production/helm/loki/src/helm-test/README.md new file mode 100644 index 0000000000000..68c9bfd9dcb06 --- /dev/null +++ b/production/helm/loki/src/helm-test/README.md @@ -0,0 +1,7 @@ +# Loki Helm Test + +This folder contains a collection of go tests that test if a Loki canary is running correctly. It's primary use it to test that the helm chart is working correctly by using metrics from the Loki canary. In the helm chart, the template for this test is only available if you are running both the Loki canary and have self monitoring enabled (as the Loki canary's logs need to be in Loki for it to work). However, the tests in this folder can be run against any running Loki canary using `go test`. + +## Instructions + +Run `go test .` from this directory, or use the Docker image published at `grafana/loki-helm-test`. diff --git a/production/helm/loki/src/helm-test/canary_test.go b/production/helm/loki/src/helm-test/canary_test.go new file mode 100644 index 0000000000000..baf92d3b9ee94 --- /dev/null +++ b/production/helm/loki/src/helm-test/canary_test.go @@ -0,0 +1,105 @@ +//go:build helm_test +// +build helm_test + +package test + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +func TestCanary(t *testing.T) { + totalEntriesQuery := "sum(loki_canary_entries_total)" + totalEntriesMissingQuery := "sum(loki_canary_missing_entries_total)" + + timeout := getEnv("CANARY_TEST_TIMEOUT", "1m") + timeoutDuration, err := time.ParseDuration(timeout) + require.NoError(t, err, "Failed to parse timeout. Please set CANARY_TEST_TIMEOUT to a valid duration.") + + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + + t.Cleanup(func() { + cancel() + }) + + t.Run("Canary should have entries", func(t *testing.T) { + client := newClient(t) + + eventually(t, func() error { + result, _, err := client.Query(ctx, totalEntriesQuery, time.Now(), v1.WithTimeout(timeoutDuration)) + if err != nil { + return err + } + return testResult(t, result, totalEntriesQuery, func(v model.SampleValue) bool { + return v > 0 + }, fmt.Sprintf("Expected %s to be greater than 0", totalEntriesQuery)) + }, timeoutDuration, "Expected Loki Canary to have entries") + }) + + t.Run("Canary should not have missed any entries", func(t *testing.T) { + client := newClient(t) + + eventually(t, func() error { + result, _, err := client.Query(ctx, totalEntriesMissingQuery, time.Now(), v1.WithTimeout(timeoutDuration)) + if err != nil { + return err + } + return testResult(t, result, totalEntriesMissingQuery, func(v model.SampleValue) bool { + return v == 0 + }, fmt.Sprintf("Expected %s to equal 0", totalEntriesMissingQuery)) + }, timeoutDuration, "Expected Loki Canary to not have any missing entries") + }) +} + +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +func testResult(t *testing.T, result model.Value, query string, test func(model.SampleValue) bool, msg string) error { + if v, ok := result.(model.Vector); ok { + for _, s := range v { + t.Logf("%s => %v\n", query, s.Value) + if !test(s.Value) { + return errors.New(msg) + } + } + + return nil + } + + return fmt.Errorf("unexpected Prometheus result type: %v ", result.Type()) +} + +func newClient(t *testing.T) v1.API { + address := os.Getenv("CANARY_PROMETHEUS_ADDRESS") + require.NotEmpty(t, address, "CANARY_PROMETHEUS_ADDRESS must be set to a valid prometheus address") + + client, err := api.NewClient(api.Config{ + Address: address, + }) + require.NoError(t, err, "Failed to create Loki Canary client") + + return v1.NewAPI(client) +} + +func eventually(t *testing.T, test func() error, timeoutDuration time.Duration , msg string) { + require.Eventually(t, func() bool { + queryError := test() + if queryError != nil { + t.Logf("Query failed\n%+v\n", queryError) + } + return queryError == nil + }, timeoutDuration, 1*time.Second, msg) +} diff --git a/production/helm/loki/src/helm-test/default.nix b/production/helm/loki/src/helm-test/default.nix new file mode 100644 index 0000000000000..5ebfa3e4e64e6 --- /dev/null +++ b/production/helm/loki/src/helm-test/default.nix @@ -0,0 +1,27 @@ +{ pkgs, lib, buildGoModule, dockerTools, rev }: +rec { + loki-helm-test = buildGoModule rec { + pname = "loki-helm-test"; + version = "0.1.0"; + + src = ./../../../../..; + vendorSha256 = null; + + buildPhase = '' + runHook preBuild + go test --tags=helm_test -c -o $out/bin/helm-test ./production/helm/loki/src/helm-test + runHook postBuild + ''; + + doCheck = false; + }; + + # by default, uses the nix hash as the tag, which can be retrieved with: + # basename "$(readlink result)" | cut -d - -f 1 + loki-helm-test-docker = dockerTools.buildImage { + name = "grafana/loki-helm-test"; + config = { + Entrypoint = [ "${loki-helm-test}/bin/helm-test" ]; + }; + }; +} diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 44cfbb55c29ea..bbb3e23266bb6 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -332,9 +332,9 @@ Create the service endpoint including port for MinIO. {{/* Determine the public host for the Loki cluster */}} {{- define "loki.host" -}} {{- $isSingleBinary := eq (include "loki.deployment.isSingleBinary" .) "true" -}} -{{- $url := printf "%s.%s.svc.%s" (include "loki.gatewayFullname" .) .Release.Namespace .Values.global.clusterDomain }} +{{- $url := printf "%s.%s.svc.%s." (include "loki.gatewayFullname" .) .Release.Namespace .Values.global.clusterDomain }} {{- if and $isSingleBinary (not .Values.gateway.enabled) }} - {{- $url = printf "%s.%s.svc.%s:3100" (include "loki.singleBinaryFullname" .) .Release.Namespace .Values.global.clusterDomain }} + {{- $url = printf "%s.%s.svc.%s.:3100" (include "loki.singleBinaryFullname" .) .Release.Namespace .Values.global.clusterDomain }} {{- end }} {{- printf "%s" $url -}} {{- end -}} diff --git a/production/helm/loki/templates/tests/_helpers.tpl b/production/helm/loki/templates/tests/_helpers.tpl new file mode 100644 index 0000000000000..b3152f908f094 --- /dev/null +++ b/production/helm/loki/templates/tests/_helpers.tpl @@ -0,0 +1,7 @@ +{{/* +Docker image name for loki helm test +*/}} +{{- define "loki.helm-test-image" -}} +{{- $dict := dict "service" .Values.test.image "global" .Values.global.image "defaultVersion" "latest" -}} +{{- include "loki.baseImage" $dict -}} +{{- end -}} diff --git a/production/helm/loki/templates/tests/test-canary.yaml b/production/helm/loki/templates/tests/test-canary.yaml new file mode 100644 index 0000000000000..ace291cb0e69e --- /dev/null +++ b/production/helm/loki/templates/tests/test-canary.yaml @@ -0,0 +1,33 @@ +{{- with .Values.test }} +{{- if and .enabled $.Values.monitoring.selfMonitoring.enabled $.Values.monitoring.selfMonitoring.lokiCanary.enabled }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "loki.name" $ }}-helm-test" + labels: + {{- include "loki.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": test +spec: + containers: + - name: loki-helm-test + image: {{ include "loki.helm-test-image" $ }} + env: + - name: CANARY_PROMETHEUS_ADDRESS + value: "{{ .prometheusAddress }}" + {{- with .timeout }} + - name: CANARY_TEST_TIMEOUT + value: "{{ . }}" + {{- end }} + args: + - -test.v + restartPolicy: Never +{{- end }} +{{- end }} diff --git a/production/helm/loki/templates/validate.yaml b/production/helm/loki/templates/validate.yaml new file mode 100644 index 0000000000000..cd1e85b34edeb --- /dev/null +++ b/production/helm/loki/templates/validate.yaml @@ -0,0 +1,24 @@ +{{- if .Values.config }} +{{- fail "Top level 'config' is not allowed. Most common configuration sections are exposed under the `loki` section. If you need to override the whole config, provide the configuration as a string that can contain template expressions under `loki.config`. Alternatively, you can provide the configuration as an external secret." }} +{{- end }} + +{{ with .Values.monitoring.selfMonitoring}} + +{{- if and (not .enabled) .lokiCanary.enabled }} +{{- fail "Loki Canary requires self monitoring to also be enabled"}} +{{- end }} + +{{- if and (not .enabled) $.Values.test.enabled }} +{{- fail "Helm test requires self monitoring to be enabled"}} +{{- end }} + +{{- if and (not .lokiCanary.enabled) $.Values.test.enabled }} +{{- fail "Helm test requires the Loki Canary to be enabled"}} +{{- end }} + +{{- end}} + +{{- if and .Values.test.enabled (not .Values.test.prometheusAddress) }} +{{- fail "Helm test requires a prometheusAddress for an instance scraping the Loki canary's metrics"}} +{{- end }} + diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 1dab18f9bb019..f96c7c3818d56 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -427,7 +427,7 @@ enterprise: sendfile on; tcp_nopush on; - resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}; + resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}.; {{- with .Values.gateway.nginxConfig.httpSnippet }} {{ . | nindent 2 }} @@ -546,6 +546,28 @@ rbac: # -- For OpenShift set pspEnabled to 'false' and sccEnabled to 'true' to use the SecurityContextConstraints. sccEnabled: false +# -- Section for configuring optional Helm test +test: + enabled: true + # -- Address of the prometheus server to query for the test + prometheusAddress: "http://prometheus:9090" + # -- Number of times to retry the test before failing + timeout: 1m + # -- Additional labels for the test pods + labels: {} + # -- Additional annotations for test pods + annotations: {} + # -- Image to use for loki canary + image: + # -- The Docker registry + registry: docker.io + # -- Docker image repository + repository: grafana/loki-helm-test + # -- Overrides the image tag whose default is the chart's appVersion + tag: null + # -- Docker image pull policy + pullPolicy: IfNotPresent + # Monitoring section determines which monitoring features to enable monitoring: # Dashboards for monitoring Loki @@ -1107,7 +1129,7 @@ gateway: sendfile on; tcp_nopush on; - resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}; + resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}.; {{- with .Values.gateway.nginxConfig.httpSnippet }} {{ . | nindent 2 }} diff --git a/tools/dev/k3d/environments/enterprise-logs/spec.json b/tools/dev/k3d/environments/enterprise-logs/spec.json index 278c6152a8458..98bcd23a2d1c7 100644 --- a/tools/dev/k3d/environments/enterprise-logs/spec.json +++ b/tools/dev/k3d/environments/enterprise-logs/spec.json @@ -6,7 +6,7 @@ "namespace": "environments/enterprise-logs/main.jsonnet" }, "spec": { - "apiServer": "https://0.0.0.0:44365", + "apiServer": "https://0.0.0.0:45985", "namespace": "k3d-enterprise-logs", "resourceDefaults": {}, "expectVersions": {} diff --git a/tools/dev/k3d/jsonnetfile.lock.json b/tools/dev/k3d/jsonnetfile.lock.json index 203daf6efbf90..f966588dec89d 100644 --- a/tools/dev/k3d/jsonnetfile.lock.json +++ b/tools/dev/k3d/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "consul" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "Po3c1Ic96ngrJCtOazic/7OsLkoILOKZWXWyZWl+od8=" }, { @@ -18,7 +18,7 @@ "subdir": "enterprise-metrics" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "hi2ZpHKl7qWXmSZ46sAycjWEQK6oGsoECuDKQT1dA+k=" }, { @@ -28,7 +28,7 @@ "subdir": "etcd-operator" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "duHm6wmUju5KHQurOe6dnXoKgl5gTUsfGplgbmAOsHw=" }, { @@ -38,7 +38,7 @@ "subdir": "grafana" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "Y5nheroSOIwmE+djEVPq4OvvTxKenzdHhpEwaR3Ebjs=" }, { @@ -48,7 +48,7 @@ "subdir": "jaeger-agent-mixin" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "nsukyr2SS8h97I2mxvBazXZp2fxu1i6eg+rKq3/NRwY=" }, { @@ -58,7 +58,7 @@ "subdir": "ksonnet-util" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "2++XoPslyz02LRgsxREWxjLgYgiCIqhAtXCyVSvYcoE=" }, { @@ -78,8 +78,8 @@ "subdir": "memcached" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", - "sum": "8hXTN4QOMkpad75LESkdfRD4/Sl81fMqZcD0ZPx2SNc=" + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", + "sum": "SWywAq4U0MRPMbASU0Ez8O9ArRNeoZzb75sEuReueow=" }, { "source": { @@ -88,7 +88,7 @@ "subdir": "tanka-util" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "9d68b0200c682fde2eca8b6b7cc738fc08d663cc", "sum": "ShSIissXdvCy1izTCDZX6tY7qxCoepE5L+WJ52Hw7ZQ=" }, {