diff --git a/config/flavors/air-gapped/sc-config.yaml b/config/flavors/air-gapped/sc-config.yaml index 97543f55b..ec0af1032 100644 --- a/config/flavors/air-gapped/sc-config.yaml +++ b/config/flavors/air-gapped/sc-config.yaml @@ -68,6 +68,7 @@ opensearch: min: 300 # 12 / day * 30 days = 360, subtract some to account for failures max: 500 ageSeconds: 2592000 # 30 days + retentionAge: 30d # 30 days backupSchedule: 30 */2 * * * # 30 min past every 2nd hour to avoid collision with retention harbor: diff --git a/config/flavors/dev/sc-config.yaml b/config/flavors/dev/sc-config.yaml index d7b4de0b7..67739d7c7 100644 --- a/config/flavors/dev/sc-config.yaml +++ b/config/flavors/dev/sc-config.yaml @@ -43,7 +43,6 @@ opensearch: snapshot: enabled: false backupSchedule: 0 */12 * * * # run twice/day - retentionSchedule: 0 1 * * * # 1am prometheus: alertmanagerSpec: diff --git a/config/flavors/prod/sc-config.yaml b/config/flavors/prod/sc-config.yaml index c40306bd7..289953656 100644 --- a/config/flavors/prod/sc-config.yaml +++ b/config/flavors/prod/sc-config.yaml @@ -64,6 +64,7 @@ opensearch: min: 300 # 12 / day * 30 days = 360, subtract some to account for failures max: 500 ageSeconds: 2592000 # 30 days + retentionAge: 30d # 30 days backupSchedule: 30 */2 * * * # 30 min past every 2nd hour to avoid collision with retention harbor: diff --git a/config/sc-config.yaml b/config/sc-config.yaml index 144f1de11..2d7440941 100644 --- a/config/sc-config.yaml +++ b/config/sc-config.yaml @@ -973,12 +973,13 @@ opensearch: repository: opensearch-snapshots # Uses the bucket set in `objectStorage.buckets.opensearch` min: 7 max: 14 + retentionAge: 10d + backupSchedule: 0 */2 * * * + retentionSchedule: 0 0 * * * + + # Needed while migration to Snapshot Management Policy ageSeconds: 864000 maxRequestSeconds: 1200 - backupSchedule: 0 */2 * * * - backupStartingDeadlineSeconds: 600 - backupActiveDeadlineSeconds: 600 - retentionSchedule: '@daily' retentionStartingDeadlineSeconds: 600 retentionActiveDeadlineSeconds: 2700 retentionResources: diff --git a/config/schemas/config.yaml b/config/schemas/config.yaml index 1e7b190b9..546ed566c 100644 --- a/config/schemas/config.yaml +++ b/config/schemas/config.yaml @@ -5606,7 +5606,7 @@ properties: snapshot: title: OpenSearch Snapshot description: |- - Configures the CronJob and repository to take snapshots in OpenSearch. + Configure OpenSearch snapshot creation and retention. This requires that `objectStorage` is configured, and will use the bucket or container set in `objectStorage.buckets.opensearch`. type: object @@ -5635,26 +5635,18 @@ properties: type: number default: 1200 backupSchedule: - title: OpenSearch Backup CronJob Schedule + title: OpenSearch Snapshot Backup Schedule description: |- Schedule to trigger Opensearch backups. Uses the Cron format, see https://en.wikipedia.org/wiki/Cron. default: 0 */2 * * * $ref: '#/$defs/cronSchedule' - backupStartingDeadlineSeconds: - title: OpenSearch Backup Job Starting Deadline - type: number - default: 600 - backupActiveDeadlineSeconds: - title: OpenSearch Backup Job Active Deadline - type: number - default: 600 retentionSchedule: - title: OpenSearch Retention CronJob Schedule + title: OpenSearch Snapshot Retention Schedule description: |- Schedule to check for and remove old snapshots. Uses the Cron format, see https://en.wikipedia.org/wiki/Cron. - default: '@daily' + default: '0 0 * * *' $ref: '#/$defs/cronSchedule' retentionStartingDeadlineSeconds: title: OpenSearch Retention Job Starting Deadline @@ -5666,6 +5658,10 @@ properties: default: 2700 retentionResources: $ref: '#/$defs/kubernetesResourceRequirements' + retentionAge: + title: OpenSearch Maximum Snapshot Age + type: string + default: '10d' securityadmin: title: OpenSearch Security Admin description: Configures the Job that initialises OpenSearch Security. diff --git a/docs/sbom.yaml b/docs/sbom.yaml index ddb336c3c..2bbf3bb3c 100644 --- a/docs/sbom.yaml +++ b/docs/sbom.yaml @@ -466,10 +466,6 @@ name: node-local-dns app_version: '' chart_version: 0.1.1 -- domain: Custom Helm Charts - name: opensearch-backup - app_version: 0.1.0 - chart_version: 0.1.0 - domain: Custom Helm Charts name: opensearch-configurer app_version: 0.1.0 @@ -780,7 +776,6 @@ - opensearch - init-harbor - opensearch-configurer - - opensearch-backup - domain: Container images name: "docker.io/opensearchproject/opensearch-dashboards" tag: "2.12.0" diff --git a/helmfile.d/charts/README.md b/helmfile.d/charts/README.md index e457b54f6..de00b682a 100644 --- a/helmfile.d/charts/README.md +++ b/helmfile.d/charts/README.md @@ -8,11 +8,11 @@ As part of our effort to make our Helm charts available for public and internal ```terminal cd helmfile.d/charts - helm pull --verify oci://ghcr.io/elastisys/opensearch-slm --version 0.1.0 --keyring public.gpg + helm pull --verify oci://ghcr.io/elastisys/opensearch-configurer --version 0.1.0 --keyring public.gpg ``` 1. Pull and install charts e.g: ```terminal - helm install opensearch-slm oci://ghcr.io/elastisys/opensearch-slm --version 0.1.0 --namespace + helm install opensearch-configurer oci://ghcr.io/elastisys/opensearch-configurer --version 0.1.0 --namespace ``` diff --git a/helmfile.d/charts/grafana-dashboards/dashboards/backup-dashboard.json b/helmfile.d/charts/grafana-dashboards/dashboards/backup-dashboard.json index 172a1a665..e0b572feb 100644 --- a/helmfile.d/charts/grafana-dashboards/dashboards/backup-dashboard.json +++ b/helmfile.d/charts/grafana-dashboards/dashboards/backup-dashboard.json @@ -218,7 +218,7 @@ "pluginVersion": "8.2.7", "targets": [ { - "expr": "min((time()-kube_job_status_completion_time{job_name=~\"opensearch-backup-.*\", cluster=~\"$cluster\"})/3600)", + "expr": "min((time()-elasticsearch_snapshot_stats_latest_snapshot_timestamp_seconds{cluster=~\"$cluster\"})/3600)", "instant": false, "interval": "", "legendFormat": "{{job_name}}", diff --git a/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/backup.yaml b/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/backup.yaml deleted file mode 100644 index c50c8c728..000000000 --- a/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/backup.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{ if .Values.opensearch.enabled }} -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: allow-opensearch-backup - namespace: opensearch-system -spec: - policyTypes: - - Egress - podSelector: - matchLabels: - app.kubernetes.io/instance: opensearch-backup - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/component: opensearch-master - ports: - - port: 9200 - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP -{{ end }} diff --git a/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/master.yaml b/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/master.yaml index 9aec5aa12..87988f9c5 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/master.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/templates/opensearch/master.yaml @@ -16,9 +16,6 @@ spec: - podSelector: matchLabels: app.kubernetes.io/instance: opensearch-curator - - podSelector: - matchLabels: - app.kubernetes.io/instance: opensearch-backup - podSelector: matchLabels: app.kubernetes.io/instance: opensearch-slm diff --git a/helmfile.d/charts/opensearch/backup/.helmignore b/helmfile.d/charts/opensearch/backup/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helmfile.d/charts/opensearch/backup/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helmfile.d/charts/opensearch/backup/Chart.yaml b/helmfile.d/charts/opensearch/backup/Chart.yaml deleted file mode 100644 index b020b3858..000000000 --- a/helmfile.d/charts/opensearch/backup/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: opensearch-backup -description: A Helm chart for Kubernetes -type: application -version: 0.1.0 -appVersion: 0.1.0 diff --git a/helmfile.d/charts/opensearch/backup/scripts/backup.sh b/helmfile.d/charts/opensearch/backup/scripts/backup.sh deleted file mode 100644 index 57098981c..000000000 --- a/helmfile.d/charts/opensearch/backup/scripts/backup.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -: "${OPENSEARCH_ENDPOINT:?Missing OPENSEARCH_ENDPOINT}" -: "${OPENSEARCH_USERNAME:?Missing OPENSEARCH_USERNAME}" -: "${OPENSEARCH_PASSWORD:?Missing OPENSEARCH_PASSWORD}" -: "${SNAPSHOT_REPOSITORY:?Missing SNAPSHOT_REPOSITORY}" -: "${INDICES:?Missing INDICES}" - -curl --insecure -s -i -u "${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}" \ - -XPUT "https://${OPENSEARCH_ENDPOINT}/_snapshot/${SNAPSHOT_REPOSITORY}/snapshot-$(date --utc +%Y%m%d_%H%M%Sz)" \ - -H "Content-Type: application/json" -d' - { - "indices": "'"${INDICES}"'", - "include_global_state": false - }' | - tee /dev/stderr | grep "200 OK" diff --git a/helmfile.d/charts/opensearch/backup/templates/_helpers.tpl b/helmfile.d/charts/opensearch/backup/templates/_helpers.tpl deleted file mode 100644 index f0ed2d73d..000000000 --- a/helmfile.d/charts/opensearch/backup/templates/_helpers.tpl +++ /dev/null @@ -1,52 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "opensearch-backup.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "opensearch-backup.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "opensearch-backup.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "opensearch-backup.labels" -}} -helm.sh/chart: {{ include "opensearch-backup.chart" . }} -{{ include "opensearch-backup.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - -{{/* -Selector labels -*/}} -{{- define "opensearch-backup.selectorLabels" -}} -app.kubernetes.io/name: {{ include "opensearch-backup.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} diff --git a/helmfile.d/charts/opensearch/backup/templates/configmap.yaml b/helmfile.d/charts/opensearch/backup/templates/configmap.yaml deleted file mode 100644 index 1131cf293..000000000 --- a/helmfile.d/charts/opensearch/backup/templates/configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "opensearch-backup.fullname" . }} - labels: - {{- include "opensearch-backup.labels" . | nindent 4 }} -data: - backup.sh: |- - {{- .Files.Get "scripts/backup.sh" | nindent 4 }} diff --git a/helmfile.d/charts/opensearch/backup/templates/cronjob.yaml b/helmfile.d/charts/opensearch/backup/templates/cronjob.yaml deleted file mode 100644 index 65e5e7a30..000000000 --- a/helmfile.d/charts/opensearch/backup/templates/cronjob.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ include "opensearch-backup.fullname" . }} - labels: - {{- include "opensearch-backup.labels" . | nindent 4 }} -spec: - schedule: {{ .Values.schedule | quote }} - concurrencyPolicy: Forbid - startingDeadlineSeconds: {{ .Values.startingDeadlineSeconds }} - jobTemplate: - spec: - activeDeadlineSeconds: {{ .Values.activeDeadlineSeconds }} - template: - metadata: - labels: - {{- include "opensearch-backup.labels" . | nindent 12 }} - spec: - restartPolicy: {{ .Values.restartPolicy }} - containers: - - name: snapshotter - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.resources }} - resources: {{- toYaml .Values.resources | nindent 16 }} - {{- end }} - volumeMounts: - - name: scripts - mountPath: /scripts - env: - - name: OPENSEARCH_ENDPOINT - value: {{ .Values.opensearch.clusterEndpoint | quote }} - - name: OPENSEARCH_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.opensearch.userSecret }} - key: username - - name: OPENSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.opensearch.userSecret }} - key: password - - name: SNAPSHOT_REPOSITORY - value: {{ .Values.snapshotRepository | quote }} - - name: INDICES - value: {{ .Values.indices | quote }} - command: ['/bin/bash'] - args: ['/scripts/backup.sh'] - volumes: - - name: scripts - configMap: - name: {{ include "opensearch-backup.fullname" . }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 12 }} - {{- end }} - securityContext: - runAsUser: 65534 diff --git a/helmfile.d/charts/opensearch/backup/values.yaml b/helmfile.d/charts/opensearch/backup/values.yaml deleted file mode 100644 index 2187a645e..000000000 --- a/helmfile.d/charts/opensearch/backup/values.yaml +++ /dev/null @@ -1,26 +0,0 @@ -image: - repository: ghcr.io/elastisys/curl-jq - tag: 1.0.0 - pullPolicy: IfNotPresent - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -opensearch: - userSecret: opensearch-snapshotter-user - clusterEndpoint: opensearch-cluster-master:9200 - -startingDeadlineSeconds: 300 -activeDeadlineSeconds: 300 - -schedule: "@daily" -snapshotRepository: "s3" - -indices: "*" - -restartPolicy: Never - -resources: - limits: {} - requests: {} diff --git a/helmfile.d/charts/opensearch/configurer/files/configurer.sh b/helmfile.d/charts/opensearch/configurer/files/configurer.sh index 9223508cf..156ffe00a 100644 --- a/helmfile.d/charts/opensearch/configurer/files/configurer.sh +++ b/helmfile.d/charts/opensearch/configurer/files/configurer.sh @@ -15,7 +15,7 @@ auth="${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}" os_url="https://{{ .Values.opensearch.clusterEndpoint }}" osd_url="http://{{ .Values.opensearch.dashboardsEndpoint }}" -snapshot_repository="{{ .Values.config.snapshotRepository }}" +snapshot_repository="{{ .Values.config.snapshots.repository }}" create_indices="{{ .Values.config.createIndices }}" @@ -267,6 +267,96 @@ create_user() { esac } +create_update_snapshot_policy() { + echo + echo "Checking if snapshot policy exists" + policy_resp=$(curl --insecure -X GET "${os_url}/_plugins/_sm/policies/snapshot_management_policy" -s -k -u "${auth}") + seq_no=$(echo "${policy_resp}" | grep "^{" | jq -r '._seq_no') + primary_term=$(echo "${policy_resp}" | grep "^{" | jq -r '._primary_term') + if [ "${seq_no}" != "null" ] && [ "${primary_term}" != "null" ]; then + echo "Updating snapshot policy" + resp=$(curl --insecure -X PUT "${os_url}/_plugins/_sm/policies/snapshot_management_policy?if_seq_no=${seq_no}&if_primary_term=${primary_term}" \ + -H 'Content-Type: application/json' \ + -s -k -u "${auth}" \ + -d '{ + "description": "Snapshot Management Policy", + "creation": { + "schedule": { + "cron": { + "expression": "{{ .Values.config.snapshots.backupSchedule }}", + "timezone": "UTC" + } + }, + "time_limit": "1h" + }, + "deletion": { + "schedule": { + "cron": { + "expression": "{{ .Values.config.snapshots.retentionSchedule }}", + "timezone": "UTC" + } + }, + "condition": { + "max_age": "{{ .Values.config.snapshots.retentionAge }}", + "min_count": {{ .Values.config.snapshots.min }}, + "max_count": {{ .Values.config.snapshots.max }} + }, + "time_limit": "1h" + }, + "snapshot_config": { + "repository": "{{ .Values.config.snapshots.repository }}", + "date_format": "yyyy-MM-dd-HH:mm:ss", + "timezone": "UTC", + "indices": "{{ .Values.config.snapshots.indices }}", + "include_global_state": "false" + } + }') + else + echo "Creating snapshot policy" + resp=$(curl --insecure -X POST "${os_url}/_plugins/_sm/policies/snapshot_management_policy" \ + -H 'Content-Type: application/json' \ + -s -k -u "${auth}" \ + -d '{ + "description": "Snapshot Management Policy", + "creation": { + "schedule": { + "cron": { + "expression": "{{ .Values.config.snapshots.backupSchedule }}", + "timezone": "UTC" + } + }, + "time_limit": "1h" + }, + "deletion": { + "schedule": { + "cron": { + "expression": "{{ .Values.config.snapshots.retentionSchedule }}", + "timezone": "UTC" + } + }, + "condition": { + "max_age": "{{ .Values.config.snapshots.retentionAge }}", + "min_count": {{ .Values.config.snapshots.min }}, + "max_count": {{ .Values.config.snapshots.max }} + }, + "time_limit": "1h" + }, + "snapshot_config": { + "repository": "{{ .Values.config.snapshots.repository }}", + "date_format": "yyyy-MM-dd-HH:mm:ss", + "timezone": "UTC", + "indices": "{{ .Values.config.snapshots.indices }}", + "include_global_state": "false" + } + }') + fi + + policy_id=$(echo "${resp}" | grep "^{" | jq -r '._id') + if [ "${policy_id}" == "null" ]; then + log_error_exit "Failed to create snapshot policy" "${resp}" + fi +} + wait_for_dashboards setup_dashboards @@ -330,5 +420,7 @@ for row in $(echo "${users}" | jq -r '.[] | @base64'); do create_user "$(_jq '.username')" "$(_jq '.definition')" done +create_update_snapshot_policy + echo echo "Done configuring OpenSearch and Dashboards" diff --git a/helmfile.d/charts/prometheus-alerts/templates/alerts/backup-status.yaml b/helmfile.d/charts/prometheus-alerts/templates/alerts/backup-status.yaml index ef81c6d95..3fa468173 100644 --- a/helmfile.d/charts/prometheus-alerts/templates/alerts/backup-status.yaml +++ b/helmfile.d/charts/prometheus-alerts/templates/alerts/backup-status.yaml @@ -65,30 +65,6 @@ spec: for: 1h labels: severity: warning - - alert: OpenSearchBackupHaveFailed24Hours - annotations: - description: The job daily backup job elasticsearch-backup have failed over 24 hours. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-elasticsearchbackup - summary: The daily backup job elasticsearch-backup have failed over 24 hours. - expr: |- - ( - min((time()-kube_job_status_completion_time{job_name=~"opensearch-backup-.*", cluster=~".*"})/3600) > 24 - ) - for: 1h - labels: - severity: warning - - alert: OpenSearchBackupHaveFailed48Hours - annotations: - description: The job daily backup job elasticsearch-backup have failed over 48 hours. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-elasticsearchbackup - summary: The daily backup job elasticsearch-backup have failed over 48 hours. - expr: |- - ( - min((time()-kube_job_status_completion_time{job_name=~"opensearch-backup-.*", cluster=~".*"})/3600) > 48 - ) - for: 1h - labels: - severity: warning - alert: OpenSearchSnapshotHaveFailed24Hours annotations: description: The job daily backup job {{`{{`}} $labels.repository {{`}}`}} have failed over 24 hours. diff --git a/helmfile.d/stacks/opensearch.yaml.gotmpl b/helmfile.d/stacks/opensearch.yaml.gotmpl index 808030c10..7efd999a3 100644 --- a/helmfile.d/stacks/opensearch.yaml.gotmpl +++ b/helmfile.d/stacks/opensearch.yaml.gotmpl @@ -128,17 +128,6 @@ templates: - values/opensearch/securityadmin.yaml.gotmpl - values/opensearch/configurer.yaml.gotmpl - opensearch-backup: - inherit: [ template: opensearch ] - installed: {{ and ( .Values | get "opensearch.enabled" false) (.Values | get "opensearch.snapshot.enabled" false) }} - chart: charts/opensearch/backup - version: 0.1.0 - name: opensearch-backup - needs: - - opensearch-system/opensearch-configurer - values: - - values/opensearch/backup.yaml.gotmpl - opensearch-slm: inherit: [ template: opensearch ] installed: {{ and ( .Values | get "opensearch.enabled" false) (.Values | get "opensearch.snapshot.enabled" false) }} diff --git a/helmfile.d/state.yaml b/helmfile.d/state.yaml index 160865ea1..28b08023d 100644 --- a/helmfile.d/state.yaml +++ b/helmfile.d/state.yaml @@ -162,7 +162,6 @@ releases: - inherit: [ template: opensearch-dashboards ] - inherit: [ template: opensearch-securityadmin ] - inherit: [ template: opensearch-configurer ] - - inherit: [ template: opensearch-backup ] - inherit: [ template: opensearch-slm ] - inherit: [ template: opensearch-curator ] - inherit: [ template: opensearch-exporter ] diff --git a/helmfile.d/values/opensearch/backup.yaml.gotmpl b/helmfile.d/values/opensearch/backup.yaml.gotmpl deleted file mode 100644 index 2e482cad4..000000000 --- a/helmfile.d/values/opensearch/backup.yaml.gotmpl +++ /dev/null @@ -1,26 +0,0 @@ -{{ if not (or (eq .Values.objectStorage.type "s3") (eq .Values.objectStorage.type "gcs") (eq .Values.objectStorage.type "azure") ) }} -{{ fail "\nERROR: OpenSearch backup requires S3 or GCS or Azure object storage, see Values.objectStorage.type" }} -{{ end }} - -opensearch: - userSecret: opensearch-snapshotter-user - clusterEndpoint: {{ .Values.opensearch.clusterName }}-master:9200 - -startingDeadlineSeconds: {{ .Values.opensearch.snapshot.backupStartingDeadlineSeconds }} -activeDeadlineSeconds: {{ .Values.opensearch.snapshot.backupActiveDeadlineSeconds }} - -schedule: {{ .Values.opensearch.snapshot.backupSchedule | quote }} - -snapshotRepository: {{ .Values.opensearch.snapshot.repository }} - -# Ignore security plugin index as per -# https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/snapshot-restore/#security-plugin-considerations -indices: "*,-.opendistro_security" - -resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 10m - memory: 32Mi diff --git a/helmfile.d/values/opensearch/configurer.yaml.gotmpl b/helmfile.d/values/opensearch/configurer.yaml.gotmpl index 8d0825ee3..4dd003f60 100644 --- a/helmfile.d/values/opensearch/configurer.yaml.gotmpl +++ b/helmfile.d/values/opensearch/configurer.yaml.gotmpl @@ -35,7 +35,16 @@ config: rolloverAge: {{ .Values.opensearch.ism.rolloverAgeDays }} {{- if .Values.opensearch.snapshot.enabled }} - snapshotRepository: {{ .Values.opensearch.snapshot.repository }} + snapshots: + repository: {{ .Values.opensearch.snapshot.repository }} + min: {{ .Values.opensearch.snapshot.min }} + max: {{ .Values.opensearch.snapshot.max }} + backupSchedule: {{ .Values.opensearch.snapshot.backupSchedule | quote }} + retentionSchedule: {{ .Values.opensearch.snapshot.retentionSchedule | quote }} + retentionAge: {{ .Values.opensearch.snapshot.retentionAge }} + # Ignore security plugin index as per + # https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/snapshot-restore/#security-plugin-considerations + indices: "*,-.opendistro_security" {{- if eq .Values.objectStorage.type "s3" }} s3: diff --git a/helmfile.d/values/opensearch/securityadmin.yaml.gotmpl b/helmfile.d/values/opensearch/securityadmin.yaml.gotmpl index 250f8dc80..3f8a66116 100644 --- a/helmfile.d/values/opensearch/securityadmin.yaml.gotmpl +++ b/helmfile.d/values/opensearch/securityadmin.yaml.gotmpl @@ -117,6 +117,12 @@ securityConfig: - "cluster:admin/repository/get" - "cluster_manage_index_templates" - "cluster:admin/opendistro/ism/policy/*" + - "cluster:admin/snapshot/get" + - "cluster:admin/snapshot/create" + - "cluster:admin/snapshot/delete" + - "cluster:admin/opensearch/snapshot_management/policy/get" + - "cluster:admin/opensearch/snapshot_management/policy/search" + - "cluster:admin/opensearch/snapshot_management/policy/write" - indices:admin/index_template/put index_permissions: - index_patterns: diff --git a/pipeline/test/services/service-cluster/testOpensearch.sh b/pipeline/test/services/service-cluster/testOpensearch.sh index 61b6babc2..63ef77e6d 100644 --- a/pipeline/test/services/service-cluster/testOpensearch.sh +++ b/pipeline/test/services/service-cluster/testOpensearch.sh @@ -101,48 +101,43 @@ check_opensearch_snapshots_status() { repo_name=$(yq4 -e '.opensearch.snapshot.repository' "${config['config_file_sc']}") repo_exists_status=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_snapshot/${repo_name}" | jq "select(.error)") if [[ -z "$repo_exists_status" ]]; then - if kubectl get "cronjob" -n "opensearch-system" "opensearch-backup" &>/dev/null; then - snapshots=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/snapshots/${repo_name}") - error=$(echo "$snapshots" | jq '.error' 2>/dev/null || true) - failed=$(echo "$snapshots" | grep 'FAILED' || true) - partial=$(echo "$snapshots" | grep 'PARTIAL' || true) + snapshots=$(curl -sk -u admin:"${adminPassword}" -X GET "https://opensearch.${opsDomain}/_cat/snapshots/${repo_name}") + error=$(echo "$snapshots" | jq '.error' 2>/dev/null || true) + failed=$(echo "$snapshots" | grep 'FAILED' || true) + partial=$(echo "$snapshots" | grep 'PARTIAL' || true) - if [[ "$error" != "" ]] && [[ "$error" != "null" ]]; then + if [[ "$error" != "" ]] && [[ "$error" != "null" ]]; then + no_error=false + debug_msg+="[ERROR] Error in snapshots output: \n $error\n" + else + if [[ "$failed" != "" ]]; then no_error=false - debug_msg+="[ERROR] Error in snapshots output: \n $error\n" - else - if [[ "$failed" != "" ]]; then - no_error=false - debug_msg+="[ERROR] We found some failed snapshots: \n $failed\n" - fi + debug_msg+="[ERROR] We found some failed snapshots: \n $failed\n" + fi - if [[ "$partial" != "" ]]; then - no_error=false - debug_msg+="[WARNING] We found some partial snapshots: \n $partial\n" - fi + if [[ "$partial" != "" ]]; then + no_error=false + debug_msg+="[WARNING] We found some partial snapshots: \n $partial\n" + fi - IFS=$'\n' readarray -t data < <(awk '{ print $1 " " $2 " " $3}' <<<"$snapshots") - IFS=" " read -ra last_snapshot <<<"${data[-1]}" + IFS=$'\n' readarray -t data < <(awk '{ print $1 " " $2 " " $3}' <<<"$snapshots") + IFS=" " read -ra last_snapshot <<<"${data[-1]}" - if [[ "${#last_snapshot[@]}" -gt 0 ]]; then - now_epoch=$(date +%s) - last_snapshot_epoch=${last_snapshot[2]} - ((diff = now_epoch - last_snapshot_epoch)) + if [[ "${#last_snapshot[@]}" -gt 0 ]]; then + now_epoch=$(date +%s) + last_snapshot_epoch=${last_snapshot[2]} + ((diff = now_epoch - last_snapshot_epoch)) - if [[ $diff -gt 86400 ]]; then - no_error=false - debug_msg+="[ERROR] The latest snapshot has not been created within the past 24 hours, with status: ${last_snapshot[1]}\n" - else - debug_msg+="[WARNING] The latest snapshot has been created within the past 24 hours, with status: ${last_snapshot[1]}\n" - fi - else + if [[ $diff -gt 86400 ]]; then no_error=false - debug_msg+="[ERROR] No snapshots found, if this is a brand new cluster this can safely be ignored\n" + debug_msg+="[ERROR] The latest snapshot has not been created within the past 24 hours, with status: ${last_snapshot[1]}\n" + else + debug_msg+="[WARNING] The latest snapshot has been created within the past 24 hours, with status: ${last_snapshot[1]}\n" fi + else + no_error=false + debug_msg+="[ERROR] No snapshots found, if this is a brand new cluster this can safely be ignored\n" fi - else - no_error=false - debug_msg+="[ERROR] opensearch-backup cronjob doesn't exist\n" fi else no_error=false diff --git a/pipeline/test/services/service-cluster/testPodsReady.sh b/pipeline/test/services/service-cluster/testPodsReady.sh index fa9ecc279..9207bf30c 100755 --- a/pipeline/test/services/service-cluster/testPodsReady.sh +++ b/pipeline/test/services/service-cluster/testPodsReady.sh @@ -197,7 +197,6 @@ if "${enable_harbor}" && "${enable_harbor_backup}"; then fi if "${enable_opensearch_snapshot}"; then cronjobs+=( - "opensearch-system opensearch-backup" "opensearch-system opensearch-slm" ) fi diff --git a/tests/unit/pods-ready.bats.yaml b/tests/unit/pods-ready.bats.yaml index ffb6ce9f6..712b0267d 100644 --- a/tests/unit/pods-ready.bats.yaml +++ b/tests/unit/pods-ready.bats.yaml @@ -122,8 +122,6 @@ tests: - target: opensearch-dashboards - function: test_cronjob tests: - - condition: .opensearch.snapshot.enabled - target: opensearch-backup - condition: .opensearch.snapshot.enabled target: opensearch-slm - condition: .opensearch.curator.enabled