From 624f681ce6f366975faf800ac167f008474ee56f Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 28 Jun 2024 11:27:46 +0200 Subject: [PATCH 1/6] feat(ocis): add purge-revision cli Signed-off-by: jkoberg --- changelog/unreleased/remove-revisions-cli.md | 5 + ocis/pkg/command/revisions.go | 109 +++++++++++++++ ocis/pkg/revisions/revisions.go | 131 +++++++++++++++++++ 3 files changed, 245 insertions(+) create mode 100644 changelog/unreleased/remove-revisions-cli.md create mode 100644 ocis/pkg/command/revisions.go create mode 100644 ocis/pkg/revisions/revisions.go diff --git a/changelog/unreleased/remove-revisions-cli.md b/changelog/unreleased/remove-revisions-cli.md new file mode 100644 index 00000000000..15afc95510d --- /dev/null +++ b/changelog/unreleased/remove-revisions-cli.md @@ -0,0 +1,5 @@ +Enhancement: Add cli to purge revisions + +Adds a cli that allows removing all revisions for a storage-provider. + +https://github.com/owncloud/ocis/pull/9497 diff --git a/ocis/pkg/command/revisions.go b/ocis/pkg/command/revisions.go new file mode 100644 index 00000000000..7de6e3367be --- /dev/null +++ b/ocis/pkg/command/revisions.go @@ -0,0 +1,109 @@ +package command + +import ( + "errors" + "fmt" + + ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/owncloud/ocis/v2/ocis/pkg/revisions" + "github.com/urfave/cli/v2" +) + +// RevisionsCommand is the entrypoint for the revisions command. +func RevisionsCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "revisions", + Usage: "ocis revisions functionality", + Subcommands: []*cli.Command{ + PurgeRevisionsCommand(cfg), + }, + Before: func(c *cli.Context) error { + return configlog.ReturnError(parser.ParseConfig(cfg, true)) + }, + Action: func(_ *cli.Context) error { + fmt.Println("Read the docs") + return nil + }, + } +} + +// PurgeRevisionsCommand allows removing all revisions from a storage provider. +func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "purge", + Usage: "purge all revisions", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "basepath", + Aliases: []string{"p"}, + Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)", + Required: true, + }, + &cli.StringFlag{ + Name: "blobstore", + Aliases: []string{"b"}, + Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", + Value: "ocis", + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "do not delete anything, just print what would be deleted", + Value: true, + }, + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "print verbose output", + Value: false, + }, + }, + Action: func(c *cli.Context) error { + basePath := c.String("basepath") + if basePath == "" { + fmt.Println("basepath is required") + return cli.ShowCommandHelp(c, "revisions") + } + + var ( + bs revisions.DelBlobstore + err error + ) + switch c.String("blobstore") { + case "s3ng": + bs, err = s3bs.New( + cfg.StorageUsers.Drivers.S3NG.Endpoint, + cfg.StorageUsers.Drivers.S3NG.Region, + cfg.StorageUsers.Drivers.S3NG.Bucket, + cfg.StorageUsers.Drivers.S3NG.AccessKey, + cfg.StorageUsers.Drivers.S3NG.SecretKey, + s3bs.Options{}, + ) + case "ocis": + bs, err = ocisbs.New(basePath) + case "none": + bs = nil + default: + err = errors.New("blobstore type not supported") + } + if err != nil { + fmt.Println(err) + return err + } + if err := revisions.PurgeRevisions(basePath, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil { + fmt.Printf("❌ Error purging revisions: %s", err) + return err + } + + return nil + }, + } +} + +func init() { + register.AddCommand(RevisionsCommand) +} diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go new file mode 100644 index 00000000000..f30eca457db --- /dev/null +++ b/ocis/pkg/revisions/revisions.go @@ -0,0 +1,131 @@ +// Package revisions allows manipulating revisions in a storage provider. +package revisions + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +var ( + // _nodesGlobPattern is the glob pattern to find all nodes + _nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*" + // regex to determine if a node versioned. Examples: + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock + _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`) +) + +// DelBlobstore is the interface for a blobstore that can delete blobs. +type DelBlobstore interface { + Delete(node *node.Node) error +} + +// PurgeRevisions removes all revisions from a storage provider. +func PurgeRevisions(p string, bs DelBlobstore, dryRun bool, verbose bool) error { + pattern := filepath.Join(p, _nodesGlobPattern) + if verbose { + fmt.Println("Looking for nodes in", pattern) + } + + nodes, err := filepath.Glob(pattern) + if err != nil { + return err + } + + if len(nodes) == 0 { + return errors.New("no nodes found, double check storage path") + } + + countFiles := 0 + countBlobs := 0 + countRevisions := 0 + for _, d := range nodes { + if !_versionRegex.MatchString(d) { + continue + } + + var blobID string + e := filepath.Ext(d) + switch e { + case ".mpk": + blobID, err = getBlobID(d) + if err != nil { + fmt.Printf("error getting blobID from %s: %v\n", d, err) + continue + } + + countBlobs++ + case ".mlock": + // no extra action on .mlock files + default: + countRevisions++ + } + + if !dryRun { + if blobID != "" { + // TODO: needs spaceID for s3ng + if err := bs.Delete(&node.Node{BlobID: blobID}); err != nil { + fmt.Printf("error deleting blob %s: %v\n", blobID, err) + continue + } + } + + if err := os.Remove(d); err != nil { + fmt.Printf("error removing %s: %v\n", d, err) + continue + } + + } + + countFiles++ + + if verbose { + if dryRun { + fmt.Println("Would delete", d) + if blobID != "" { + fmt.Println("Would delete blob", blobID) + } + } else { + fmt.Println("Deleted", d) + if blobID != "" { + fmt.Println("Deleted blob", blobID) + } + } + } + } + + switch { + case countFiles == 0 && countRevisions == 0 && countBlobs == 0: + fmt.Println("❎ No revisions found. Storage provider is clean.") + case !dryRun: + fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + default: + fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + } + return nil +} + +func getBlobID(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", err + } + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + return string(bid), nil + } + + return "", nil +} From 077e56e2c7a3f5dff69dc3b702ce64b7dfea6dfe Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 2 Jul 2024 15:33:52 +0200 Subject: [PATCH 2/6] feat(ocis): make purge-revision work per resource Signed-off-by: jkoberg --- ocis/pkg/command/revisions.go | 49 +++++++++++++++++++++++++++++++-- ocis/pkg/revisions/revisions.go | 42 ++++++++++++++++++++++------ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/ocis/pkg/command/revisions.go b/ocis/pkg/command/revisions.go index 7de6e3367be..174a9df3a6a 100644 --- a/ocis/pkg/command/revisions.go +++ b/ocis/pkg/command/revisions.go @@ -3,9 +3,12 @@ package command import ( "errors" "fmt" + "path/filepath" ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" + "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/owncloud/ocis/v2/ocis-pkg/config" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" @@ -14,6 +17,11 @@ import ( "github.com/urfave/cli/v2" ) +var ( + // _nodesGlobPattern is the glob pattern to find all nodes + _nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*" +) + // RevisionsCommand is the entrypoint for the revisions command. func RevisionsCommand(cfg *config.Config) *cli.Command { return &cli.Command{ @@ -36,7 +44,7 @@ func RevisionsCommand(cfg *config.Config) *cli.Command { func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "purge", - Usage: "purge all revisions", + Usage: "purge revisions", Flags: []cli.Flag{ &cli.StringFlag{ Name: "basepath", @@ -47,7 +55,7 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { &cli.StringFlag{ Name: "blobstore", Aliases: []string{"b"}, - Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", + Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service", Value: "ocis", }, &cli.BoolFlag{ @@ -61,6 +69,11 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { Usage: "print verbose output", Value: false, }, + &cli.StringFlag{ + Name: "resource-id", + Aliases: []string{"r"}, + Usage: "purge all revisions of this file/space. If not set, all revisions will be purged", + }, }, Action: func(c *cli.Context) error { basePath := c.String("basepath") @@ -94,7 +107,14 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { fmt.Println(err) return err } - if err := revisions.PurgeRevisions(basePath, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil { + + p, err := generatePath(basePath, c.String("resource-id")) + if err != nil { + fmt.Printf("❌ Error parsing resourceID: %s", err) + return err + } + + if err := revisions.PurgeRevisions(p, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil { fmt.Printf("❌ Error purging revisions: %s", err) return err } @@ -104,6 +124,29 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { } } +func generatePath(basePath string, resourceID string) (string, error) { + if resourceID == "" { + return filepath.Join(basePath, _nodesGlobPattern), nil + } + + rid, err := storagespace.ParseID(resourceID) + if err != nil { + return "", err + } + + sid := lookup.Pathify(rid.GetSpaceId(), 1, 2) + if sid == "" { + sid = "*/*" + } + + nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2) + if nid == "" { + nid = "*/*/*/*/" + } + + return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*"), nil +} + func init() { register.AddCommand(RevisionsCommand) } diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go index f30eca457db..249dbf37e1c 100644 --- a/ocis/pkg/revisions/revisions.go +++ b/ocis/pkg/revisions/revisions.go @@ -7,14 +7,13 @@ import ( "os" "path/filepath" "regexp" + "strings" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/shamaton/msgpack/v2" ) var ( - // _nodesGlobPattern is the glob pattern to find all nodes - _nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*" // regex to determine if a node versioned. Examples: // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk @@ -28,8 +27,7 @@ type DelBlobstore interface { } // PurgeRevisions removes all revisions from a storage provider. -func PurgeRevisions(p string, bs DelBlobstore, dryRun bool, verbose bool) error { - pattern := filepath.Join(p, _nodesGlobPattern) +func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) error { if verbose { fmt.Println("Looking for nodes in", pattern) } @@ -87,15 +85,22 @@ func PurgeRevisions(p string, bs DelBlobstore, dryRun bool, verbose bool) error countFiles++ if verbose { + spaceID, nodeID := getIDsFromPath(d) if dryRun { - fmt.Println("Would delete", d) + fmt.Println("Would delete") + fmt.Println("\tResourceID:", spaceID+"!"+nodeID) + fmt.Println("\tSpaceID:", spaceID) + fmt.Println("\tPath:", d) if blobID != "" { - fmt.Println("Would delete blob", blobID) + fmt.Println("\tBlob:", blobID) } } else { - fmt.Println("Deleted", d) + fmt.Println("Deleted") + fmt.Println("\tResourceID:", spaceID+"!"+nodeID) + fmt.Println("\tSpaceID:", spaceID) + fmt.Println("\tPath:", d) if blobID != "" { - fmt.Println("Deleted blob", blobID) + fmt.Println("\tBlob:", blobID) } } } @@ -129,3 +134,24 @@ func getBlobID(path string) (string, error) { return "", nil } + +func getIDsFromPath(path string) (string, string) { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return "", "" + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return "", "" + } + + n := strings.Split(rawIDs[1], ".REV.") + if len(n) != 2 { + return "", "" + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(n[0], "/", "", -1) + return spaceID, filepath.Base(nodeID) +} From f8e5175df526aaa3288650cc2d0309b8681cc49f Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 4 Jul 2024 14:59:02 +0200 Subject: [PATCH 3/6] docu(ocis): document the new commands Signed-off-by: jkoberg --- ocis/README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ocis/README.md b/ocis/README.md index 4bd86c67aae..15ef1e8650a 100644 --- a/ocis/README.md +++ b/ocis/README.md @@ -23,4 +23,60 @@ To authenticate the connection to the etcd registry, you have to set `ETCD_USERN ## Memory limits -oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide). \ No newline at end of file +oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide). + +## Cli commands + +The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common parameter: `--basePath` (or `-p`). This needs to point to a storage provider. Examples are: +```bash +.ocis/storage/users # bare metal installation +/var/tmp/ocis/storage/users # docker installation +... +``` +This value can vary depending on your ocis installation. + +### Backup Cli + +The backup command allows inspecting the consistency of an ocis storage: +``` +ocis backup consistency -p /base/path/storage/users +``` + +This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be: +* Orphaned Blobs: A blob in the blobstore that is not referenced by any file metadata +* Missing Blobs: A blob referenced by file metadata that is not present in the blobstore +* Missing Nodes: A node that is referenced by a symlink but doesn't exist +* Missing Link: A node that is not referenced by any symlink but should be +* Missing Files: A node that is missing essential files (such as the `.mpk` metadata file) +* Missing/Malformed Metadata: A node that doesn't have any (or malformed) metadata + +This command provides additional options: +* `-b`/`--blobstore` allows specifying the blobstore to use. Defaults to `ocis`. If empty blobs will not be checked. Can also be switched to `s3ng` but needs addtional envvar configuration (see storage-users service). +* `--fail` exists with non-zero exit code if inconsistencies are found. Useful for automation. + +### Revisions Cli + +The revisions command allows removing the revisions of files in the storage +``` +ocis revisions purge -p /base/path/storage/users +``` + +It takes the `--resource-id` (or `--r`) parameter which specify the scope of the command: +* An empty string (default) removes all revisions from all spaces. +* A spaceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space. +* A resourceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49\!e8a73d49-2e00-4322-9f34-9d7f178577b2`) removes all revisions from that specific file. + +This command provides additional options: +* `--dry-run` (default: `true`) does not remove any revisions but prints the revisions that would be removed. +* `-b` / `--blobstore` allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see storage-users service). +* `-v` / `--verbose` prints additional information about the revisions that are removed. + +### Trash Cli + +The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations. +``` +ocis trash purge-empty-dirs -p /base/path/storage/users +``` + +This command provides additional options: +* `--dry-run` (default: `true`) does not remove any empty folders but prints the empty folders that would be removed. From 315f52328d757db3339b9bef33c2a61201d4bf60 Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 5 Jul 2024 10:13:14 +0200 Subject: [PATCH 4/6] feat(docu): improve docu Co-authored-by: Martin Signed-off-by: jkoberg --- ocis/README.md | 72 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/ocis/README.md b/ocis/README.md index 15ef1e8650a..a1ef4bcb637 100644 --- a/ocis/README.md +++ b/ocis/README.md @@ -25,58 +25,80 @@ To authenticate the connection to the etcd registry, you have to set `ETCD_USERN oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide). -## Cli commands +## CLI Commands + +The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common mandatory parameter: `--basePath` (or `-p`) which needs to point to a storage provider. Example paths are: -The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common parameter: `--basePath` (or `-p`). This needs to point to a storage provider. Examples are: ```bash -.ocis/storage/users # bare metal installation -/var/tmp/ocis/storage/users # docker installation +.ocis/storage/users # bare metal installation +/var/tmp/ocis/storage/users # docker installation ... ``` -This value can vary depending on your ocis installation. -### Backup Cli +These paths can vary depending on your ocis installation. + +### Backup CLI The backup command allows inspecting the consistency of an ocis storage: -``` + +```bash ocis backup consistency -p /base/path/storage/users ``` This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be: -* Orphaned Blobs: A blob in the blobstore that is not referenced by any file metadata -* Missing Blobs: A blob referenced by file metadata that is not present in the blobstore -* Missing Nodes: A node that is referenced by a symlink but doesn't exist -* Missing Link: A node that is not referenced by any symlink but should be -* Missing Files: A node that is missing essential files (such as the `.mpk` metadata file) -* Missing/Malformed Metadata: A node that doesn't have any (or malformed) metadata + +* **Orphaned Blobs**\ +A blob in the blobstore that is not referenced by any file metadata. +* **Missing Blobs**\ +A blob referenced by file metadata that is not present in the blobstore. +* **Missing Nodes**\ +A node that is referenced by a symlink but doesn't exist. +* **Missing Link**\ +A node that is not referenced by any symlink but should be. +* **Missing Files**\ +A node that is missing essential files (such as the `.mpk` metadata file). +* **Missing/Malformed Metadata**\ +A node that doesn't have any (or malformed) metadata. This command provides additional options: -* `-b`/`--blobstore` allows specifying the blobstore to use. Defaults to `ocis`. If empty blobs will not be checked. Can also be switched to `s3ng` but needs addtional envvar configuration (see storage-users service). -* `--fail` exists with non-zero exit code if inconsistencies are found. Useful for automation. -### Revisions Cli +* `-b` / `--blobstore`\ +Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details). +* `--fail`\ +Exits with non-zero exit code if inconsistencies are found. Useful for automation. -The revisions command allows removing the revisions of files in the storage -``` +### Revisions CLI + +The revisions command allows removing the revisions of files in the storage. + +```bash ocis revisions purge -p /base/path/storage/users ``` It takes the `--resource-id` (or `--r`) parameter which specify the scope of the command: + * An empty string (default) removes all revisions from all spaces. -* A spaceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space. +* A spaceID (like `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space. * A resourceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49\!e8a73d49-2e00-4322-9f34-9d7f178577b2`) removes all revisions from that specific file. This command provides additional options: -* `--dry-run` (default: `true`) does not remove any revisions but prints the revisions that would be removed. -* `-b` / `--blobstore` allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see storage-users service). -* `-v` / `--verbose` prints additional information about the revisions that are removed. -### Trash Cli +* `--dry-run` (default: `true`)\ +Do not remove any revisions but print the revisions that would be removed. +* `-b` / `--blobstore`\ +Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details). +* `-v` / `--verbose`\ +Prints additional information about the revisions that are removed. + +### Trash CLI The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations. -``` + +```bash ocis trash purge-empty-dirs -p /base/path/storage/users ``` This command provides additional options: -* `--dry-run` (default: `true`) does not remove any empty folders but prints the empty folders that would be removed. + +* `--dry-run` (default: `true`)\ +Do not remove any empty folders but print the empty folders that would be removed. From 53553a9bfd13012c1caa54e994e7c99f7145487c Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Tue, 9 Jul 2024 14:45:36 +0200 Subject: [PATCH 5/6] disable vuln-check temporarily Signed-off-by: Christian Richter --- .drone.star | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index b93ab99943e..80715758f4a 100644 --- a/.drone.star +++ b/.drone.star @@ -321,11 +321,19 @@ def buildWebCache(ctx): def testOcisAndUploadResults(ctx): pipeline = testOcis(ctx) + ###################################################################### + # The triggers have been disabled for now, since the govulncheck can # + # not silence single, acceptable vulnerabilities. # + # See https://github.com/owncloud/ocis/issues/9527 for more details. # + # FIXME: RE-ENABLE THIS ASAP!!! # + ###################################################################### + scan_result_upload = uploadScanResults(ctx) scan_result_upload["depends_on"] = getPipelineNames([pipeline]) - security_scan = scanOcis(ctx) - return [security_scan, pipeline, scan_result_upload] + #security_scan = scanOcis(ctx) + #return [security_scan, pipeline, scan_result_upload] + return [pipeline, scan_result_upload] def testPipelines(ctx): pipelines = [] From 9355cc59128834712a51830866b236ba72f4e030 Mon Sep 17 00:00:00 2001 From: Christian Richter Date: Tue, 9 Jul 2024 14:47:36 +0200 Subject: [PATCH 6/6] remove not backported readme references Signed-off-by: Christian Richter --- ocis/README.md | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/ocis/README.md b/ocis/README.md index a1ef4bcb637..c081047791c 100644 --- a/ocis/README.md +++ b/ocis/README.md @@ -37,36 +37,6 @@ The ocis package offers a variety of cli commands to monitor or repair ocis inst These paths can vary depending on your ocis installation. -### Backup CLI - -The backup command allows inspecting the consistency of an ocis storage: - -```bash -ocis backup consistency -p /base/path/storage/users -``` - -This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be: - -* **Orphaned Blobs**\ -A blob in the blobstore that is not referenced by any file metadata. -* **Missing Blobs**\ -A blob referenced by file metadata that is not present in the blobstore. -* **Missing Nodes**\ -A node that is referenced by a symlink but doesn't exist. -* **Missing Link**\ -A node that is not referenced by any symlink but should be. -* **Missing Files**\ -A node that is missing essential files (such as the `.mpk` metadata file). -* **Missing/Malformed Metadata**\ -A node that doesn't have any (or malformed) metadata. - -This command provides additional options: - -* `-b` / `--blobstore`\ -Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details). -* `--fail`\ -Exits with non-zero exit code if inconsistencies are found. Useful for automation. - ### Revisions CLI The revisions command allows removing the revisions of files in the storage. @@ -90,15 +60,3 @@ Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to ` * `-v` / `--verbose`\ Prints additional information about the revisions that are removed. -### Trash CLI - -The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations. - -```bash -ocis trash purge-empty-dirs -p /base/path/storage/users -``` - -This command provides additional options: - -* `--dry-run` (default: `true`)\ -Do not remove any empty folders but print the empty folders that would be removed.