From a41274ff9735d86397868063b2fd15ac4a4fdfe3 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 7 Jun 2022 14:56:15 +0300 Subject: [PATCH] docs: Add example section and dockerconfigjson encryption - remove the Go spec section - add basic usage example - fix SOPS CLI examples - explain how to generate image pull secrets from encrypted dockerconfigjson files Signed-off-by: Stefan Prodan --- docs/spec/v1beta2/kustomization.md | 356 +++++++++-------------------- 1 file changed, 112 insertions(+), 244 deletions(-) diff --git a/docs/spec/v1beta2/kustomization.md b/docs/spec/v1beta2/kustomization.md index e2d7f889..babf0831 100644 --- a/docs/spec/v1beta2/kustomization.md +++ b/docs/spec/v1beta2/kustomization.md @@ -1,247 +1,95 @@ # Kustomization -The `Kustomization` API defines a pipeline for fetching, decrypting, building, validating and applying Kubernetes manifests. - -## Specification - -A **Kustomization** object defines the source of Kubernetes manifests by referencing an object -managed by [source-controller](https://github.com/fluxcd/source-controller), -the path to the Kustomization file within that source, -and the interval at which the kustomize build output is applied on the cluster. - -```go -type KustomizationSpec struct { - // DependsOn may contain a dependency.CrossNamespaceDependencyReference slice - // with references to Kustomization resources that must be ready before this - // Kustomization can be reconciled. - // +optional - DependsOn []dependency.CrossNamespaceDependencyReference `json:"dependsOn,omitempty"` - - // Decrypt Kubernetes secrets before applying them on the cluster. - // +optional - Decryption *Decryption `json:"decryption,omitempty"` - - // The interval at which to reconcile the Kustomization. - // +required - Interval metav1.Duration `json:"interval"` - - // The interval at which to retry a previously failed reconciliation. - // When not specified, the controller uses the KustomizationSpec.Interval - // value to retry failures. - // +optional - RetryInterval *metav1.Duration `json:"retryInterval,omitempty"` - - // The KubeConfig for reconciling the Kustomization on a remote cluster. - // When specified, KubeConfig takes precedence over ServiceAccountName. - // +optional - KubeConfig *KubeConfig `json:"kubeConfig,omitempty"` - - // Path to the directory containing the kustomization.yaml file, or the - // set of plain YAMLs a kustomization.yaml should be generated for. - // Defaults to 'None', which translates to the root path of the SourceRef. - // +optional - Path string `json:"path,omitempty"` - - // PostBuild describes which actions to perform on the YAML manifest - // generated by building the kustomize overlay. - // +optional - PostBuild *PostBuild `json:"postBuild,omitempty"` - - // Enables garbage collection. - // +required - Prune bool `json:"prune"` - - // Force instructs the controller to recreate resources - // when patching fails due to an immutable field change. - // +kubebuilder:default:=false - // +optional - Force bool `json:"force,omitempty"` - - // Strategic merge and JSON patches, defined as inline YAML objects, - // capable of targeting objects based on kind, label and annotation selectors. - // +optional - Patches []kustomize.Patch `json:"patches,omitempty"` - - // Images is a list of (image name, new name, new tag or digest) - // for changing image names, tags or digests. This can also be achieved with a - // patch, but this operator is simpler to specify. - // +optional - Images []kustomize.Image `json:"images,omitempty"` - - // The name of the Kubernetes service account to impersonate - // when reconciling this Kustomization. - // +optional - ServiceAccountName string `json:"serviceAccountName,omitempty"` - - // Reference of the source where the kustomization file is. - // +required - SourceRef CrossNamespaceSourceReference `json:"sourceRef"` - - // This flag tells the controller to suspend subsequent kustomize executions, - // it does not apply to already started executions. Defaults to false. - // +optional - Suspend bool `json:"suspend,omitempty"` - - // TargetNamespace sets or overrides the namespace in the - // kustomization.yaml file. - // +optional - TargetNamespace string `json:"targetNamespace,omitempty"` - - // Timeout for apply and health checking operations. - // Defaults to 'Interval' duration. - // +optional - Timeout *metav1.Duration `json:"timeout,omitempty"` - - // A list of resources to be included in the health assessment. - // +optional - HealthChecks []meta.NamespacedObjectKindReference `json:"healthChecks,omitempty"` - - // Wait instructs the controller to check the health of all the reconciled resources. - // When enabled, the HealthChecks are ignored. Defaults to false. - // +optional - Wait bool `json:"wait,omitempty"` -} -``` - -The decryption section defines how decryption is handled for Kubernetes manifests: - -```go -type Decryption struct { - // Provider is the name of the decryption engine. - // +kubebuilder:validation:Enum=sops - // +required - Provider string `json:"provider"` - - // The secret name containing the private OpenPGP keys used for decryption. - // +optional - SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"` -} -``` - -KubeConfig references a Kubernetes Secret for applying to another cluster. -This can be used with Cluster API: - -```go -type KubeConfig struct { - // SecretRef holds the name of a secret that contains a key with - // the kubeconfig file as the value. If no key is set, the key will default - // to 'value'. The secret must be in the same namespace as - // the Kustomization. - // It is recommended that the kubeconfig is self-contained, and the secret - // is regularly updated if credentials such as a cloud-access-token expire. - // Cloud specific `cmd-path` auth helpers will not function without adding - // binaries and credentials to the Pod that is responsible for reconciling - // the Kustomization. - // +required - SecretRef meta.SecretKeyReference `json:"secretRef,omitempty"` -} -``` - -The post-build section defines which actions to perform on the YAML manifest after kustomize build: - -```go -type PostBuild struct { - // Substitute holds a map of key/value pairs. - // The variables defined in your YAML manifests - // that match any of the keys defined in the map - // will be substituted with the set value. - // Includes support for bash string replacement functions - // e.g. ${var:=default}, ${var:position} and ${var/substring/replacement}. - // +optional - Substitute map[string]string `json:"substitute,omitempty"` - - // SubstituteFrom holds references to ConfigMaps and Secrets containing - // the variables and their values to be substituted in the YAML manifests. - // The ConfigMap and the Secret data keys represent the var names and they - // must match the vars declared in the manifests for the substitution to happen. - // +optional - SubstituteFrom []SubstituteReference `json:"substituteFrom,omitempty"` -} -``` - -The status sub-resource records the result of the last reconciliation: - -```go -type KustomizationStatus struct { - // ObservedGeneration is the last reconciled generation. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` +The `Kustomization` API defines a pipeline for fetching, decrypting, building, +validating and applying Kustomize overlays or plain Kubernetes manifests. +The `Kustomization` Custom Resource Definition is the +counterpart of Kustomize' `kustomization.yaml` config file. - // The last successfully applied revision. - // The revision format for Git sources is /. - // +optional - LastAppliedRevision string `json:"lastAppliedRevision,omitempty"` +## Example - // LastAttemptedRevision is the revision of the last reconciliation attempt. - // +optional - LastAttemptedRevision string `json:"lastAttemptedRevision,omitempty"` +The following is an example of a Flux Kustomization that reconciles on the +cluster the Kubernetes manifests stored in a Git repository. - // LastHandledReconcileAt is the last manual reconciliation request (by - // annotating the Kustomization) handled by the reconciler. - // +optional - LastHandledReconcileAt string `json:"lastHandledReconcileAt,omitempty"` - - // Inventory contains the list of Kubernetes resource object references - // that have been successfully applied. - // +optional - Inventory *ResourceInventory `json:"inventory,omitempty"` -} -``` - -Status condition types: - -```go -const ( - // ReadyCondition is the name of the condition that - // records the readiness status of a Kustomization. - ReadyCondition string = "Ready" - - // HealthyCondition is the condition type used - // to record the last health assessment result. - HealthyCondition string = "Healthy" -) +```yaml +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 5m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: podinfo + namespace: default +spec: + interval: 10m + targetNamespace: default + sourceRef: + kind: GitRepository + name: podinfo + path: "./kustomize" + prune: true ``` -Status condition reasons: - -```go -const ( - // ReconciliationSucceededReason represents the fact that the - // reconciliation of the Kustomization has succeeded. - ReconciliationSucceededReason string = "ReconciliationSucceeded" - - // ReconciliationFailedReason represents the fact that the - // reconciliation of the Kustomization has failed. - ReconciliationFailedReason string = "ReconciliationFailed" - - // ProgressingReason represents the fact that the - // reconciliation of the Kustomization is underway. - ProgressingReason string = "Progressing" - - // DependencyNotReady represents the fact that - // one of the dependencies of the Kustomization is not ready. - DependencyNotReadyReason string = "DependencyNotReady" - - // PruneFailedReason represents the fact that the - // pruning of the Kustomization failed. - PruneFailedReason string = "PruneFailed" - - // ArtifactFailedReason represents the fact that the - // artifact download of the kustomization failed. - ArtifactFailedReason string = "ArtifactFailed" - - // BuildFailedReason represents the fact that the - // kustomize build of the Kustomization failed. - BuildFailedReason string = "BuildFailed" - - // HealthCheckFailedReason represents the fact that - // one of the health checks of the Kustomization failed. - HealthCheckFailedReason string = "HealthCheckFailed" -) -``` +In the above example: + +- A Flux GitRepository named `podinfo` is created that clones the `master` branch + and makes the repository content available as an Artifact inside the cluster. +- A Flux Kustomization named `podinfo` is created that watches the GitRepository for artifact changes. +- The Kustomization builds the YAML manifests located at the specified `spec.path`, + sets the namespace of all objects to the `spec.targetNamespace`, + validates the objects against the Kubernetes API, and finally applies them on the cluster. +- Every ten minutes, the Kustomization runs a server-side apply dry-run to detect and correct drift inside the cluster. +- When the Git revision changes, the manifests are reconciled automatically. If previously applied objects + are missing from the current revision, these objects are deleted from the cluster when `spec.pune` is enabled. + +You can run this example by saving the manifest into `podinfo.yaml`. + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f podinfo.yaml + ``` + +2. Run `kubectl get gitrepositories` to see the source status: + + ```console + NAME URL READY STATUS + podinfo https://github.com/stefanprodan/podinfo True stored artifact for revision 'master/450796ddb2ab6724ee1cc32a4be56da032d1cca0' + ``` + +3. Run `kubectl get kustomizations` to see the reconciliation status: + + ```console + NAME READY STATUS + podinfo True Applied revision: master/450796ddb2ab6724ee1cc32a4be56da032d1cca0 + ``` + +4. Run `kubectl describe kustomization podinfo` to see the reconciliation status conditions and events: + + ```console + ... + Status: + Conditions: + Last Transition Time: 2022-06-07T11:14:41Z + Message: Applied revision: master/450796ddb2ab6724ee1cc32a4be56da032d1cca0 + Reason: ReconciliationSucceeded + Status: True + Type: Ready + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Progressing 1m48s kustomize-controller Service/default/podinfo created + Deployment/default/podinfo created + HorizontalPodAutoscaler/default/podinfo created + Normal ReconciliationSucceeded 1m48s kustomize-controller Reconciliation finished in 176.163666ms, next run in 10m0s + ``` ## Recommended settings @@ -290,7 +138,7 @@ spec: For security and performance reasons, it is advised to disallow the usage of [remote bases](https://github.com/kubernetes-sigs/kustomize/blob/a7f4db7fb41e17b2c826a524f545e6174b4dc6ac/examples/remoteBuild.md) -in Kustomize overlays. To enforce this setting, platform admins can use the `--no-remote-bases=true` flag. +in Kustomize overlays. To enforce this setting, platform admins can use the `--no-remote-bases=true` controller flag. ## Source reference @@ -1449,12 +1297,12 @@ The kustomize-controller scans the values of Kubernetes Secrets, and when it detects that the values are SOPS encrypted, it decrypts them before applying them on the cluster. -For secrets in `.json`, `.yaml` and `.env` format, make sure you specify the input type when encrypting them with SOPS: +For secrets in `.json`, `.yaml` `.ini` and `.env` format, make sure you specify the input type when encrypting them with SOPS: ```sh -cat config.json | sops -e --input-type=json > config.json.encrypted -cat config.yaml | sops -e --input-type=yaml > config.yaml.encrypted -cat config.env | sops -e --input-type=env > config.env.encrypted +sops -e --input-type=json config.json > config.json.encrypted +sops -e --input-type=yaml config.yaml > config.yaml.encrypted +sops -e --input-type=env config.env > config.env.encrypted ``` For kustomize-controller to be able to decrypt a JSON config, you need to set the file extension to `.json`: @@ -1470,7 +1318,6 @@ secretGenerator: For dotenv files, use the `envs` directive: ```yaml -apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization secretGenerator: - name: config @@ -1478,9 +1325,26 @@ secretGenerator: - config.env.encrypted ``` +For Docker config files, you need to specify both input and output type as JSON: + +```sh +sops -e --input-type=json --output-type=json ghcr.dockerconfigjson > ghcr.dockerconfigjson.encrypted +``` + +To generate an image pull secret, use the `.dockerconfigjson` as the secret key: + +```yaml +kind: Kustomization +secretGenerator: + - name: ghcr-auth + type: kubernetes.io/dockerconfigjson + files: + - .dockerconfigjson=ghcr.dockerconfigjson.encrypted +``` + ## Status -When the controller completes a Kustomization apply, reports the result in the `status` sub-resource. +When the controller completes a Kustomization reconciliation, reports the result in the `status` sub-resource. A successful reconciliation sets the ready condition to `true` and updates the revision field: @@ -1496,6 +1360,10 @@ status: lastAttemptedRevision: master/a1afe267b54f38b46b487f6e938a6fd508278c07 ``` +If `spec.wait` or `spec.healthChecks` is enabled, the health assessment result +is reported under the `Healthy` condition. A failed health check will set both +`Ready` and `Healthy` conditions to `False`. + You can wait for the kustomize controller to complete a reconciliation with: ```bash