diff --git a/crds/backup.yaml b/crds/backup.yaml index 4654fc17..20cef5e2 100644 --- a/crds/backup.yaml +++ b/crds/backup.yaml @@ -3,6 +3,22 @@ kind: CustomResourceDefinition metadata: name: backups.resources.cattle.io spec: + additionalPrinterColumns: + - JSONPath: .status.storageLocation + name: Storage-Location + type: string + - JSONPath: .status.backupType + name: Backup-Type + type: string + - JSONPath: .status.numSnapshots + name: Backups-Saved + type: string + - JSONPath: .status.prefix + name: Backupfile-Prefix + type: string + - JSONPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string group: resources.cattle.io names: kind: Backup @@ -16,12 +32,12 @@ spec: spec: properties: encryptionConfigName: - description: 'Name of secret containing the encryption config, must + description: 'Name of the Secret containing the encryption config, must be in the namespace of the chart: cattle-resources-system' type: string resourceSetName: - description: Name of resourceSet CR to use for backup, must be in the - same namespace + description: Name of the ResourceSet CR to use for backup, must be in + the same namespace type: string retentionCount: type: integer @@ -55,6 +71,8 @@ spec: type: object status: properties: + backupType: + type: string conditions: items: properties: @@ -73,6 +91,8 @@ spec: type: object nullable: true type: array + filename: + type: string lastSnapshotTs: type: string nextSnapshotAt: @@ -81,6 +101,10 @@ spec: type: integer observedGeneration: type: integer + prefix: + type: string + storageLocation: + type: string summary: type: string type: object diff --git a/crds/faketest.yaml b/crds/faketest.yaml deleted file mode 100644 index 33470444..00000000 --- a/crds/faketest.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: faketests.backupper.cattle.io -spec: - group: backupper.cattle.io - names: - kind: FakeTest - plural: faketests - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - valInt: - type: integer - valStr: - type: string - type: object - status: - properties: - generation: - type: integer - version: - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true diff --git a/crds/restore.yaml b/crds/restore.yaml index fc08d448..a89f74a0 100644 --- a/crds/restore.yaml +++ b/crds/restore.yaml @@ -3,6 +3,16 @@ kind: CustomResourceDefinition metadata: name: restores.resources.cattle.io spec: + additionalPrinterColumns: + - JSONPath: .status.numRetries + name: Retries + type: string + - JSONPath: .status.backupSource + name: Backup-Source + type: string + - JSONPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string group: resources.cattle.io names: kind: Restore @@ -18,7 +28,8 @@ spec: backupFilename: type: string deleteTimeout: - maximum: 5 + maximum: 10 + deleteTimeoutSeconds: type: integer encryptionConfigName: type: string @@ -52,6 +63,8 @@ spec: type: object status: properties: + backupSource: + type: string conditions: items: properties: diff --git a/examples/create-deflocation-backup.yaml b/examples/create-deflocation-backup.yaml index 95eb55ae..daa4c351 100644 --- a/examples/create-deflocation-backup.yaml +++ b/examples/create-deflocation-backup.yaml @@ -2,9 +2,6 @@ apiVersion: resources.cattle.io/v1 kind: Backup metadata: name: test-default-location-recurring-backup - namespace: default spec: resourceSetName: ecm-resource-set - encryptionConfigName: test-encryptionconfig - schedule: "@every 1m" - retention: "4m" \ No newline at end of file + encryptionConfigName: test-encryptionconfig \ No newline at end of file diff --git a/examples/create-deflocation-restore.yaml b/examples/create-deflocation-restore.yaml new file mode 100644 index 00000000..c5b42b7a --- /dev/null +++ b/examples/create-deflocation-restore.yaml @@ -0,0 +1,8 @@ +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo + namespace: default +spec: + backupFilename: backup-ns-test-default-location-recurring-backup-6a9c60bf-73ff-461f-93d8-506f174b1d65-2020-08-28T00#39#05Z.tar.gz + encryptionConfigName: test-encryptionconfig \ No newline at end of file diff --git a/examples/create-local-backup.yaml b/examples/create-local-backup.yaml deleted file mode 100644 index 8fbae850..00000000 --- a/examples/create-local-backup.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: resources.cattle.io/v1 -kind: Backup -metadata: - name: test-ecm-backup - namespace: default -spec: - storageLocation: - local: test-backup-location-local - resourceSetName: ecm-resource-set - encryptionConfigName: test-encryptionconfig \ No newline at end of file diff --git a/examples/create-local-recurring-backup.yaml b/examples/create-local-recurring-backup.yaml deleted file mode 100644 index b4a786d9..00000000 --- a/examples/create-local-recurring-backup.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: resources.cattle.io/v1 -kind: Backup -metadata: - name: test-local-recurring-backup - namespace: default -spec: - resourceSetName: ecm-resource-set - encryptionConfigName: test-encryptionconfig - schedule: "@every 1m" - retentionCount: 5 \ No newline at end of file diff --git a/examples/create-local-restore.yaml b/examples/create-local-restore.yaml deleted file mode 100644 index 2452317e..00000000 --- a/examples/create-local-restore.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: resources.cattle.io/v1 -kind: Restore -metadata: - name: my-restore -spec: - backupFilename: default-test-ecm-backup-061e9637-6671-468e-abf5-2cee7e2fd817-2020-08-19T13#43#37-07#00.tar.gz - encryptionConfigName: test-encryptionconfig - deleteTimeout: 5 - storageLocation: - local: test-backup-location-local \ No newline at end of file diff --git a/examples/create-s3-backup.yaml b/examples/create-s3-backup.yaml index c6c57248..2685801b 100644 --- a/examples/create-s3-backup.yaml +++ b/examples/create-s3-backup.yaml @@ -2,7 +2,6 @@ apiVersion: resources.cattle.io/v1 kind: Backup metadata: name: s3-backup-demo - namespace: default spec: storageLocation: s3: diff --git a/examples/create-s3-restore.yaml b/examples/create-s3-restore.yaml index 804f0184..c1e97f31 100644 --- a/examples/create-s3-restore.yaml +++ b/examples/create-s3-restore.yaml @@ -4,7 +4,7 @@ metadata: name: restore-s3-demo namespace: default spec: - backupFilename: default-s3-backup-demo-84bf8dd8-0ef3-4240-8ad1-fc7ec308e216-2020-08-24T12#49#25-07#00.tar.gz + backupFilename: default-s3-backup-demo-df56e4f4-d5f6-4b9d-8a78-b8c89757df30-2020-08-27T09#59#53-07#00.tar.gz storageLocation: s3: credentialSecretName: creds diff --git a/main.go b/main.go index eeba8c35..0f95288e 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,7 @@ var ( func init() { flag.StringVar(&KubeConfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.Parse() - OperatorPVCName = os.Getenv("DEFAULT_PVC_BACKUP_STORAGE_LOCATION") + OperatorPVCName = os.Getenv("DEFAULT_PVC_NAME") OperatorS3BackupStorageLocation = os.Getenv("DEFAULT_S3_BACKUP_STORAGE_LOCATION") ChartNamespace = os.Getenv("CHART_NAMESPACE") } @@ -107,26 +107,14 @@ func main() { logrus.Fatalf("Error setting default location %v: %v", dmPath, err) } logrus.Infof("No temporary backup location provided, saving backups at %v", dmPath) - OperatorPVCName = dmPath + defaultMountPath = dmPath } } else { // else, this log tells user that each backup needs to contain StorageLocation details logrus.Infof("No PVC or S3 details provided for storing backups by default. User must specify storageLocation" + "on each Backup CR") - // TODO: add to the operator status, - // TODO: add printer columns for backup indicate storageLocation - // TODO: document this, and add to chart notes } } else if OperatorPVCName != "" { - // Get the PVC using the name, and it should be in the chart's namespace - pvc, err := core.Core().V1().PersistentVolumeClaim().Get(ChartNamespace, OperatorPVCName, k8sv1.GetOptions{}) - if err != nil { - logrus.Fatalf("Error getting pvc details %v: %v", OperatorPVCName, err) - } - // pvc must have only one access mode, that is RWO - if !(len(pvc.Spec.AccessModes) == 1 && pvc.Spec.AccessModes[0] == "ReadWriteOnce") { - logrus.Fatalf("PVC %v access mode must be ReadWriteOnce", OperatorPVCName) - } defaultMountPath = LocalBackupStorageLocation } else if OperatorS3BackupStorageLocation != "" { // read the secret from chart's namespace, with OperatorS3BackupStorageLocation as the name diff --git a/pkg/apis/resources.cattle.io/v1/types.go b/pkg/apis/resources.cattle.io/v1/types.go index e1fb8e5b..da02d658 100644 --- a/pkg/apis/resources.cattle.io/v1/types.go +++ b/pkg/apis/resources.cattle.io/v1/types.go @@ -40,6 +40,10 @@ type BackupStatus struct { NextSnapshotAt string `json:"nextSnapshotAt"` NumSnapshots int `json:"numSnapshots"` ObservedGeneration int64 `json:"observedGeneration"` + StorageLocation string `json:"storageLocation"` + BackupType string `json:"backupType"` + Filename string `json:"filename"` + Prefix string `json:"prefix"` Summary string `json:"summary"` } @@ -103,7 +107,7 @@ type RestoreSpec struct { BackupFilename string `json:"backupFilename"` StorageLocation *StorageLocation `json:"storageLocation"` Prune *bool `json:"prune"` //prune by default - DeleteTimeout int `json:"deleteTimeout,omitempty"` + DeleteTimeoutSeconds int `json:"deleteTimeoutSeconds,omitempty"` EncryptionConfigName string `json:"encryptionConfigName,omitempty"` } @@ -112,5 +116,6 @@ type RestoreStatus struct { RestoreCompletionTS string `json:"restoreCompletionTs"` ObservedGeneration int64 `json:"observedGeneration"` NumRetries int `json:"numRetries"` - Summary string `json:"summary,omitempty"` + BackupSource string `json:"backupSource"` + Summary string `json:"summary"` } diff --git a/pkg/controllers/backup/controller.go b/pkg/controllers/backup/controller.go index 0ec0e563..b98bb0e9 100644 --- a/pkg/controllers/backup/controller.go +++ b/pkg/controllers/backup/controller.go @@ -112,7 +112,7 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error } } - backupFileName, err := h.generateBackupFilename(backup) + backupFileName, prefix, err := h.generateBackupFilename(backup) if err != nil { return h.setReconcilingCondition(backup, err) } @@ -148,14 +148,15 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error return h.setReconcilingCondition(backup, err) } } - + storageLocationType := backup.Status.StorageLocation updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { var err error backup, err = h.backups.Get(backup.Namespace, backup.Name, k8sv1.GetOptions{}) if err != nil { return err } - + condition.Cond(v1.BackupConditionReady).SetStatusBool(backup, true) + condition.Cond(v1.BackupConditionReady).Message(backup, "Completed") condition.Cond(v1.BackupConditionUploaded).SetStatusBool(backup, true) backup.Status.LastSnapshotTS = time.Now().Format(time.RFC3339) backup.Status.NumSnapshots++ @@ -164,9 +165,14 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error backup.Status.NextSnapshotAt = nextBackupAt.Format(time.RFC3339) after := nextBackupAt.Sub(time.Now().Round(time.Minute)) h.backups.EnqueueAfter(backup.Namespace, backup.Name, after) + backup.Status.BackupType = "Recurring" + } else { + backup.Status.BackupType = "One-time" } backup.Status.ObservedGeneration = backup.Generation - + backup.Status.StorageLocation = storageLocationType + backup.Status.Filename = backupFileName + ".tar.gz" + backup.Status.Prefix = prefix _, err = h.backups.UpdateStatus(backup) return err }) @@ -248,15 +254,18 @@ func (h *handler) performBackup(backup *v1.Backup, tmpBackupPath, backupFileName if err := CreateTarAndGzip(tmpBackupPath, h.defaultBackupMountPath, gzipFile, backup.Name); err != nil { return err } + backup.Status.StorageLocation = util.PVCBackup } else if h.defaultS3BackupLocation != nil { // not checking for nil, since if this wasn't provided, the default local location would get used if err := h.uploadToS3(backup, h.defaultS3BackupLocation, util.ChartNamespace, tmpBackupPath, gzipFile); err != nil { return err } + backup.Status.StorageLocation = util.S3Backup } else { return fmt.Errorf("backup %v needs to specify S3 details, or configure storage location at the operator level", backup.Name) } } else if storageLocation.S3 != nil { + backup.Status.StorageLocation = util.S3Backup if err := h.uploadToS3(backup, storageLocation.S3, backup.Namespace, tmpBackupPath, gzipFile); err != nil { return err } @@ -264,12 +273,13 @@ func (h *handler) performBackup(backup *v1.Backup, tmpBackupPath, backupFileName return nil } -func (h *handler) generateBackupFilename(backup *v1.Backup) (string, error) { +func (h *handler) generateBackupFilename(backup *v1.Backup) (string, string, error) { currSnapshotTS := time.Now().Format(time.RFC3339) // on OS X writing file with `:` converts colon to forward slash currTSForFilename := strings.Replace(currSnapshotTS, ":", "#", -1) backupFileName := fmt.Sprintf("%s-%s-%s-%s", backup.Namespace, backup.Name, h.kubeSystemNS, currTSForFilename) - return backupFileName, nil + prefix := fmt.Sprintf("%s-%s-%s", backup.Namespace, backup.Name, h.kubeSystemNS) + return backupFileName, prefix, nil } // https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus @@ -284,6 +294,7 @@ func (h *handler) setReconcilingCondition(backup *v1.Backup, originalErr error) condition.Cond(v1.BackupConditionReconciling).SetStatusBool(updBackup, true) condition.Cond(v1.BackupConditionReconciling).SetError(updBackup, "", originalErr) + condition.Cond(v1.BackupConditionReady).Message(updBackup, "Retrying") _, err = h.backups.UpdateStatus(updBackup) return err diff --git a/pkg/controllers/restore/controller.go b/pkg/controllers/restore/controller.go index 9c3e0482..1d962015 100644 --- a/pkg/controllers/restore/controller.go +++ b/pkg/controllers/restore/controller.go @@ -106,6 +106,7 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e return restore, nil } + var backupSource string backupName := restore.Spec.BackupFilename logrus.Infof("Restoring from backup %v", restore.Spec.BackupFilename) if restore.Status.NumRetries > 0 { @@ -150,12 +151,14 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e return restore, removeFileErr } foundBackup = true + backupSource = util.S3Backup } else if h.defaultBackupMountPath != "" { backupFilePath := filepath.Join(h.defaultBackupMountPath, backupName) if err = h.LoadFromTarGzip(backupFilePath, transformerMap); err != nil { return h.setReconcilingCondition(restore, err) } foundBackup = true + backupSource = util.PVCBackup } } else if backupLocation.S3 != nil { backupFilePath, err := h.downloadFromS3(restore, restore.Spec.StorageLocation.S3, restore.Namespace) @@ -171,6 +174,7 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e return restore, removeFileErr } foundBackup = true + backupSource = util.S3Backup } if !foundBackup { return h.setReconcilingCondition(restore, fmt.Errorf("Backup location not specified on the restore CR, and not configured at the operator level")) @@ -202,7 +206,7 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e // prune by default if restore.Spec.Prune == nil || *restore.Spec.Prune == true { logrus.Infof("Pruning resources that are not part of the backup for restore CR %v", restore.Name) - if err := h.prune(h.backupResourceSet.ResourceSelectors, transformerMap, restore.Spec.DeleteTimeout); err != nil { + if err := h.prune(h.backupResourceSet.ResourceSelectors, transformerMap, restore.Spec.DeleteTimeoutSeconds); err != nil { return h.setReconcilingCondition(restore, fmt.Errorf("error pruning during restore: %v", err)) } } @@ -215,9 +219,10 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e } condition.Cond(v1.RestoreConditionReady).SetStatusBool(restore, true) + condition.Cond(v1.BackupConditionReady).Message(restore, "Completed") restore.Status.RestoreCompletionTS = time.Now().Format(time.RFC3339) restore.Status.ObservedGeneration = restore.Generation - + restore.Status.BackupSource = backupSource _, err = h.restores.UpdateStatus(restore) return err }) @@ -579,6 +584,8 @@ func getGVR(resourceGVR string) schema.GroupVersionResource { // https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus // Reconciling and Stalled conditions are present and with a value of true whenever something unusual happens. func (h *handler) setReconcilingCondition(restore *v1.Restore, originalErr error) (*v1.Restore, error) { + // scale back up controller before returning error + h.scaleUpControllersFromResourceSet() time.Sleep(2 * time.Second) err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var err error @@ -590,6 +597,7 @@ func (h *handler) setReconcilingCondition(restore *v1.Restore, originalErr error updRestore.Status.NumRetries++ condition.Cond(v1.RestoreConditionReconciling).SetStatusBool(updRestore, true) condition.Cond(v1.RestoreConditionReconciling).SetError(updRestore, "", originalErr) + condition.Cond(v1.BackupConditionReady).Message(updRestore, "Retrying") _, err = h.restores.UpdateStatus(updRestore) return err diff --git a/pkg/crds/crd.go b/pkg/crds/crd.go index 4ac83bce..c3efcc72 100644 --- a/pkg/crds/crd.go +++ b/pkg/crds/crd.go @@ -45,12 +45,20 @@ func WriteCRD() error { func List() []crd.CRD { return []crd.CRD{ newCRD(&resources.Backup{}, func(c crd.CRD) crd.CRD { - return c - }), - newCRD(&resources.ResourceSet{}, func(c crd.CRD) crd.CRD { - return c + return c. + WithColumn("Storage-Location", ".status.storageLocation"). + WithColumn("Backup-Type", ".status.backupType"). + WithColumn("Backups-Saved", ".status.numSnapshots"). + WithColumn("Backupfile-Prefix", ".status.prefix"). + WithColumn("Status", ".status.conditions[?(@.type==\"Ready\")].message") }), newCRD(&resources.Restore{}, func(c crd.CRD) crd.CRD { + return c. + WithColumn("Retries", ".status.numRetries"). + WithColumn("Backup-Source", ".status.backupSource"). + WithColumn("Status", ".status.conditions[?(@.type==\"Ready\")].message") + }), + newCRD(&resources.ResourceSet{}, func(c crd.CRD) crd.CRD { return c }), } @@ -61,10 +69,10 @@ func customizeBackup(backup *apiext.CustomResourceDefinition) { spec := properties["spec"] spec.Required = []string{"resourceSetName"} resourceSetName := spec.Properties["resourceSetName"] - resourceSetName.Description = "Name of resourceSet CR to use for backup, must be in the same namespace" + resourceSetName.Description = "Name of the ResourceSet CR to use for backup, must be in the same namespace" spec.Properties["resourceSetName"] = resourceSetName encryptionConfig := spec.Properties["encryptionConfigName"] - encryptionConfig.Description = "Name of secret containing the encryption config, must be in the namespace of the chart: cattle-resources-system" + encryptionConfig.Description = "Name of the Secret containing the encryption config, must be in the namespace of the chart: cattle-resources-system" spec.Properties["encryptionConfigName"] = encryptionConfig schedule := spec.Properties["schedule"] schedule.Description = "Cron schedule for recurring backups" @@ -81,7 +89,7 @@ func customizeResourceSet(resourceSetCRD *apiext.CustomResourceDefinition) { } func customizeRestore(restore *apiext.CustomResourceDefinition) { - maxDeleteTimeout := float64(5) + maxDeleteTimeout := float64(10) properties := restore.Spec.Validation.OpenAPIV3Schema.Properties spec := properties["spec"] spec.Required = []string{"backupFilename"} diff --git a/pkg/resourcesets/collector.go b/pkg/resourcesets/collector.go index a68aa484..f4be15ec 100644 --- a/pkg/resourcesets/collector.go +++ b/pkg/resourcesets/collector.go @@ -56,7 +56,7 @@ func (h *ResourceHandler) GatherResources(ctx context.Context, resourceSelectors for _, resourceSelector := range resourceSelectors { resourceList, err := h.gatherResourcesForGroupVersion(resourceSelector) if err != nil { - return resourcesWithStatusSubresource, err + return resourcesWithStatusSubresource, fmt.Errorf("error gathering resouce for %v: %v", resourceSelector.APIVersion, err) } gv, err := schema.ParseGroupVersion(resourceSelector.APIVersion) if err != nil { diff --git a/pkg/util/util.go b/pkg/util/util.go index 397ba709..5e2d288b 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -15,6 +15,8 @@ import ( const ( WorkerThreads = 25 + S3Backup = "S3" + PVCBackup = "PVC" ) var ChartNamespace string