CLI: when command usage is multi-line; amend and refactor cli/docs

* const `Usage` (refactor all sources) * combine `ais cp` help and documentation; fix all cross-refs * docs: add '--num-workers' Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Sep 16, 2024 · b6653a2 · b6653a2
1 parent 2414c68
commit b6653a2
Show file tree

Hide file tree

Showing 25 changed files with 359 additions and 337 deletions.
diff --git a/README.md b/README.md
@@ -186,8 +186,7 @@ Since AIS natively supports [remote backends](/docs/providers.md), you can also
   - [eXtended Actions (xactions)](https://github.com/NVIDIA/aistore/blob/main/xact/README.md)
   - [CLI: `ais job`](/docs/cli/job.md) and [`ais show job`](/docs/cli/show.md), including:
     - [prefetch remote datasets](/docs/cli/object.md#prefetch-objects)
-    - [copy bucket](/docs/cli/bucket.md#copy-bucket)
-    - [copy multiple objects](/docs/cli/bucket.md#copy-multiple-objects)
+    - [copy (list, range, and/or prefix) selected objects or entire (in-cluster or remote) buckets](/docs/cli/bucket.md#copy-list-range-andor-prefix-selected-objects-or-entire-in-cluster-or-remote-buckets)
     - [download remote BLOBs](/docs/cli/blob-downloader.md)
     - [promote NFS or SMB share](https://aistore.nvidia.com/blog/2022/03/17/promote)
 - Assorted Topics

diff --git a/cmd/cli/cli/arch_hdlr.go b/cmd/cli/cli/arch_hdlr.go
@@ -30,21 +30,48 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var (
-	archPutUsage = "archive a file, a directory, or multiple files and/or directories as\n" +
-		indent1 + "\t" + archExts + "-formatted object - aka \"shard\".\n" +
-		indent1 + "\tBoth APPEND (to an existing shard) and PUT (a new version of the shard) are supported.\n" +
-		indent1 + "\tExamples:\n" +
-		indent1 + "\t- 'local-file s3://q/shard-00123.tar.lz4 --append --archpath name-in-archive' - append file to a given shard,\n" +
-		indent1 + "\t   optionally, rename it (inside archive) as specified;\n" +
-		indent1 + "\t- 'local-file s3://q/shard-00123.tar.lz4 --append-or-put --archpath name-in-archive' - append file to a given shard if exists,\n" +
-		indent1 + "\t   otherwise, create a new shard (and name it shard-00123.tar.lz4, as specified);\n" +
-		indent1 + "\t- 'src-dir gs://w/shard-999.zip --append' - archive entire 'src-dir' directory; iff the destination .zip doesn't exist create a new one;\n" +
-		indent1 + "\t- '\"sys, docs\" ais://dst/CCC.tar --dry-run -y -r --archpath ggg/' - dry-run to recursively archive two directories.\n" +
-		indent1 + "\tTips:\n" +
-		indent1 + "\t- use '--dry-run' if in doubt;\n" +
-		indent1 + "\t- to archive objects from a ais:// or remote bucket, run 'ais archive bucket' (see --help for details)."
-)
+const archBucketUsage = "archive selected or matching objects from " + bucketObjectSrcArgument + " as\n" +
+	indent1 + archExts + "-formatted object (a.k.a. shard),\n" +
+	indent1 + "e.g.:\n" +
+	indent1 + "\t- 'archive bucket ais://src ais://dst/a.tar.lz4 --template \"shard-{001..997}\"'\n" +
+	indent1 + "\t- 'archive bucket \"ais://src/shard-{001..997}\" ais://dst/a.tar.lz4'\t- same as above (notice double quotes)\n" +
+	indent1 + "\t- 'archive bucket \"ais://src/shard-{998..999}\" ais://dst/a.tar.lz4 --append-or-put'\t- append (ie., archive) 2 more objects"
+
+const archPutUsage = "archive a file, a directory, or multiple files and/or directories as\n" +
+	indent1 + "\t" + archExts + "-formatted object - aka \"shard\".\n" +
+	indent1 + "\tBoth APPEND (to an existing shard) and PUT (a new version of the shard) are supported.\n" +
+	indent1 + "\tExamples:\n" +
+	indent1 + "\t- 'local-file s3://q/shard-00123.tar.lz4 --append --archpath name-in-archive' - append file to a given shard,\n" +
+	indent1 + "\t   optionally, rename it (inside archive) as specified;\n" +
+	indent1 + "\t- 'local-file s3://q/shard-00123.tar.lz4 --append-or-put --archpath name-in-archive' - append file to a given shard if exists,\n" +
+	indent1 + "\t   otherwise, create a new shard (and name it shard-00123.tar.lz4, as specified);\n" +
+	indent1 + "\t- 'src-dir gs://w/shard-999.zip --append' - archive entire 'src-dir' directory; iff the destination .zip doesn't exist create a new one;\n" +
+	indent1 + "\t- '\"sys, docs\" ais://dst/CCC.tar --dry-run -y -r --archpath ggg/' - dry-run to recursively archive two directories.\n" +
+	indent1 + "\tTips:\n" +
+	indent1 + "\t- use '--dry-run' if in doubt;\n" +
+	indent1 + "\t- to archive objects from a ais:// or remote bucket, run 'ais archive bucket' (see --help for details)."
+
+// (compare with  objGetUsage)
+const archGetUsage = "get a shard and extract its content; get an archived file;\n" +
+	indent4 + "\twrite the content locally with destination options including: filename, directory, STDOUT ('-'), or '/dev/null' (discard);\n" +
+	indent4 + "\tassorted options further include:\n" +
+	indent4 + "\t- '--prefix' to get multiple shards in one shot (empty prefix for the entire bucket);\n" +
+	indent4 + "\t- '--progress' and '--refresh' to watch progress bar;\n" +
+	indent4 + "\t- '-v' to produce verbose output when getting multiple objects.\n" +
+	indent1 + "'ais archive get' examples:\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar.lz4 /tmp/out - get and extract entire shard to /tmp/out/trunk/*\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar.lz4 --archpath file45.jpeg /tmp/out - extract one named file\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar.lz4/file45.jpeg /tmp/out - same as above (and note that '--archpath' is implied)\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar.lz4/file45 /tmp/out/file456.new - same as above, with destination explicitly (re)named\n" +
+	indent1 + "'ais archive get' multi-selection examples:\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar 111.tar --archregx=jpeg --archmode=suffix - return 111.tar with all *.jpeg files from a given shard\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar 222.tar --archregx=file45 --archmode=wdskey - return 222.tar with all file45.* files --/--\n" +
+	indent4 + "\t- ais://abc/trunk-0123.tar 333.tar --archregx=subdir/ --archmode=prefix - 333.tar with all subdir/* files --/--"
+
+const genShardsUsage = "generate random " + archExts + "-formatted objects (\"shards\"), e.g.:\n" +
+	indent4 + "\t- gen-shards 'ais://bucket1/shard-{001..999}.tar' - write 999 random shards (default sizes) to ais://bucket1\n" +
+	indent4 + "\t- gen-shards \"gs://bucket2/shard-{01..20..2}.tgz\" - 10 random gzipped tarfiles to Cloud bucket\n" +
+	indent4 + "\t(notice quotation marks in both cases)"
 
 var (
 	// flags
@@ -86,13 +113,8 @@ var (
 
 	// archive bucket
 	archBucketCmd = cli.Command{
-		Name: commandBucket,
-		Usage: "archive selected or matching objects from " + bucketObjectSrcArgument + " as\n" +
-			indent1 + archExts + "-formatted object (a.k.a. shard),\n" +
-			indent1 + "e.g.:\n" +
-			indent1 + "\t- 'archive bucket ais://src ais://dst/a.tar.lz4 --template \"shard-{001..997}\"'\n" +
-			indent1 + "\t- 'archive bucket \"ais://src/shard-{001..997}\" ais://dst/a.tar.lz4'\t- same as above (notice double quotes)\n" +
-			indent1 + "\t- 'archive bucket \"ais://src/shard-{998..999}\" ais://dst/a.tar.lz4 --append-or-put'\t- append (ie., archive) 2 more objects",
+		Name:         commandBucket,
+		Usage:        archBucketUsage,
 		ArgsUsage:    bucketObjectSrcArgument + " " + dstShardArgument,
 		Flags:        archCmdsFlags[commandBucket],
 		Action:       archMultiObjHandler,
@@ -111,23 +133,8 @@ var (
 
 	// archive get
 	archGetCmd = cli.Command{
-		Name: objectCmdGet.Name,
-		// NOTE: compare with  objectCmdGet.Usage
-		Usage: "get a shard and extract its content; get an archived file;\n" +
-			indent4 + "\twrite the content locally with destination options including: filename, directory, STDOUT ('-'), or '/dev/null' (discard);\n" +
-			indent4 + "\tassorted options further include:\n" +
-			indent4 + "\t- '--prefix' to get multiple shards in one shot (empty prefix for the entire bucket);\n" +
-			indent4 + "\t- '--progress' and '--refresh' to watch progress bar;\n" +
-			indent4 + "\t- '-v' to produce verbose output when getting multiple objects.\n" +
-			indent1 + "'ais archive get' examples:\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar.lz4 /tmp/out - get and extract entire shard to /tmp/out/trunk/*\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar.lz4 --archpath file45.jpeg /tmp/out - extract one named file\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar.lz4/file45.jpeg /tmp/out - same as above (and note that '--archpath' is implied)\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar.lz4/file45 /tmp/out/file456.new - same as above, with destination explicitly (re)named\n" +
-			indent1 + "'ais archive get' multi-selection examples:\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar 111.tar --archregx=jpeg --archmode=suffix - return 111.tar with all *.jpeg files from a given shard\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar 222.tar --archregx=file45 --archmode=wdskey - return 222.tar with all file45.* files --/--\n" +
-			indent4 + "\t- ais://abc/trunk-0123.tar 333.tar --archregx=subdir/ --archmode=prefix - 333.tar with all subdir/* files --/--",
+		Name:         objectCmdGet.Name,
+		Usage:        archGetUsage,
 		ArgsUsage:    getShardArgument,
 		Flags:        rmFlags(objectCmdGet.Flags, headObjPresentFlag, lengthFlag, offsetFlag),
 		Action:       getArchHandler,
@@ -146,11 +153,8 @@ var (
 
 	// gen shards
 	genShardsCmd = cli.Command{
-		Name: cmdGenShards,
-		Usage: "generate random " + archExts + "-formatted objects (\"shards\"), e.g.:\n" +
-			indent4 + "\t- gen-shards 'ais://bucket1/shard-{001..999}.tar' - write 999 random shards (default sizes) to ais://bucket1\n" +
-			indent4 + "\t- gen-shards \"gs://bucket2/shard-{01..20..2}.tgz\" - 10 random gzipped tarfiles to Cloud bucket\n" +
-			indent4 + "\t(notice quotation marks in both cases)",
+		Name:      cmdGenShards,
+		Usage:     genShardsUsage,
 		ArgsUsage: `"BUCKET/TEMPLATE.EXT"`,
 		Flags:     archCmdsFlags[cmdGenShards],
 		Action:    genShardsHandler,

diff --git a/cmd/cli/cli/bencodeway.go → cmd/cli/cli/bencodeway_hdlr.go b/cmd/cli/cli/bencodeway.go → cmd/cli/cli/bencodeway_hdlr.go
@@ -13,6 +13,15 @@ import (
 	"github.com/urfave/cli"
 )
 
+const mirrorUsage = "configure (or unconfigure) bucket as n-way mirror, and run the corresponding batch job, e.g.:\n" +
+	indent1 + "\t- 'ais start mirror ais://m --copies 3'\t- configure ais://m as a 3-way mirror;\n" +
+	indent1 + "\t- 'ais start mirror ais://m --copies 1'\t- configure ais://m for no redundancy (no extra copies).\n" +
+	indent1 + "(see also: 'ais start ec-encode')"
+
+const bencodeUsage = "erasure code entire bucket, e.g.:\n" +
+	indent1 + "\t- 'ais start ec-encode ais://m -d 8 -p 2'\t- erasure-code ais://m for (D=8, P=2).\n" +
+	indent1 + "(see also: 'ais start mirror')"
+
 var (
 	storageSvcCmdsFlags = map[string][]cli.Flag{
 		commandMirror: {
@@ -28,21 +37,16 @@ var (
 
 	storageSvcCmds = []cli.Command{
 		{
-			Name: commandMirror,
-			Usage: "configure (or unconfigure) bucket as n-way mirror, and run the corresponding batch job, e.g.:\n" +
-				indent1 + "\t- 'ais start mirror ais://m --copies 3'\t- configure ais://m as a 3-way mirror;\n" +
-				indent1 + "\t- 'ais start mirror ais://m --copies 1'\t- configure ais://m for no redundancy (no extra copies).\n" +
-				indent1 + "(see also: 'ais start ec-encode')",
+			Name:         commandMirror,
+			Usage:        mirrorUsage,
 			ArgsUsage:    bucketArgument,
 			Flags:        storageSvcCmdsFlags[commandMirror],
 			Action:       setCopiesHandler,
 			BashComplete: bucketCompletions(bcmplop{}),
 		},
 		{
-			Name: commandECEncode,
-			Usage: "erasure code entire bucket, e.g.:\n" +
-				indent1 + "\t- 'ais start ec-encode ais://m -d 8 -p 2'\t- erasure-code ais://m for (D=8, P=2).\n" +
-				indent1 + "(see also: 'ais start mirror')",
+			Name:         commandECEncode,
+			Usage:        bencodeUsage,
 			ArgsUsage:    bucketArgument,
 			Flags:        storageSvcCmdsFlags[commandECEncode],
 			Action:       ecEncodeHandler,

diff --git a/cmd/cli/cli/bucket_hdlr.go b/cmd/cli/cli/bucket_hdlr.go
@@ -68,6 +68,25 @@ var listAnyUsage = "list buckets, objects in buckets, and files in " + archExts
 	indent1 + "\t* ais ls s3 --summary --all \t- generate summary report for all s3 buckets; include remote objects and buckets that are _not present_\n" +
 	indent1 + "\t* ais ls s3 --summary --all --dont-add\t- same as above but without adding _non-present_ remote buckets to cluster's BMD"
 
+const setBpropsUsage = "update bucket properties; the command accepts both JSON-formatted input and plain Name=Value pairs, e.g.:\n" +
+	indent1 + "\t* ais bucket props set ais://nnn backend_bck=s3://mmm\n" +
+	indent1 + "\t* ais bucket props set ais://nnn backend_bck=none\n" +
+	indent1 + "\t* ais bucket props set gs://vvv versioning.validate_warm_get=false versioning.synchronize=true\n" +
+	indent1 + "\t* ais bucket props set gs://vvv mirror.enabled=true mirror.copies=4 checksum.type=md5\n" +
+	indent1 + "\t* ais bucket props set s3://mmm ec.enabled true ec.data_slices 6 ec.parity_slices 4 --force\n" +
+	indent1 + "\tReferences:\n" +
+	indent1 + "\t* for details and many more examples, see docs/cli/bucket.md\n" +
+	indent1 + "\t* to show bucket properties (names and current values), use 'ais bucket show'"
+
+const evictUsage = "evict one remote bucket, multiple remote buckets, or\n" +
+	indent1 + "selected objects in a given remote bucket or buckets, e.g.:\n" +
+	indent1 + "\t- 'evict gs://abc'\t- evict entire bucket (all gs://abc objects in aistore);\n" +
+	indent1 + "\t- 'evict gs:'\t- evict all GCP buckets from the cluster;\n" +
+	indent1 + "\t- 'evict gs://abc --template images/'\t- evict all objects from the virtual subdirectory \"images\";\n" +
+	indent1 + "\t- 'evict gs://abc/images/'\t- same as above;\n" +
+	indent1 + "\t- 'evict gs://abc --template \"shard-{0000..9999}.tar.lz4\"'\t- evict the matching range (prefix + brace expansion);\n" +
+	indent1 + "\t- 'evict \"gs://abc/shard-{0000..9999}.tar.lz4\"'\t- same as above (notice double quotes)"
+
 var (
 	// flags
 	bucketCmdsFlags = map[string][]cli.Flag{
@@ -188,15 +207,8 @@ var (
 		BashComplete: bucketCompletions(bcmplop{}),
 	}
 	bucketObjCmdEvict = cli.Command{
-		Name: commandEvict,
-		Usage: "evict one remote bucket, multiple remote buckets, or\n" +
-			indent1 + "selected objects in a given remote bucket or buckets, e.g.:\n" +
-			indent1 + "\t- 'evict gs://abc'\t- evict entire bucket (all gs://abc objects in aistore);\n" +
-			indent1 + "\t- 'evict gs:'\t- evict all GCP buckets from the cluster;\n" +
-			indent1 + "\t- 'evict gs://abc --template images/'\t- evict all objects from the virtual subdirectory \"images\";\n" +
-			indent1 + "\t- 'evict gs://abc/images/'\t- same as above;\n" +
-			indent1 + "\t- 'evict gs://abc --template \"shard-{0000..9999}.tar.lz4\"'\t- evict the matching range (prefix + brace expansion);\n" +
-			indent1 + "\t- 'evict \"gs://abc/shard-{0000..9999}.tar.lz4\"'\t- same as above (notice double quotes)",
+		Name:         commandEvict,
+		Usage:        evictUsage,
 		ArgsUsage:    bucketObjectOrTemplateMultiArg,
 		Flags:        bucketCmdsFlags[commandEvict],
 		Action:       evictHandler,
@@ -219,16 +231,8 @@ var (
 		BashComplete: manyBucketsCompletions([]cli.BashCompleteFunc{}, 0, 2),
 	}
 	bucketCmdSetProps = cli.Command{
-		Name: cmdSetBprops,
-		Usage: "update bucket properties; the command accepts both JSON-formatted input and plain Name=Value pairs, e.g.:\n" +
-			indent1 + "\t* ais bucket props set ais://nnn backend_bck=s3://mmm\n" +
-			indent1 + "\t* ais bucket props set ais://nnn backend_bck=none\n" +
-			indent1 + "\t* ais bucket props set gs://vvv versioning.validate_warm_get=false versioning.synchronize=true\n" +
-			indent1 + "\t* ais bucket props set gs://vvv mirror.enabled=true mirror.copies=4 checksum.type=md5\n" +
-			indent1 + "\t* ais bucket props set s3://mmm ec.enabled true ec.data_slices 6 ec.parity_slices 4 --force\n" +
-			indent1 + "\tReferences:\n" +
-			indent1 + "\t* for details and many more examples, see docs/cli/bucket.md\n" +
-			indent1 + "\t* to show bucket properties (names and current values), use 'ais bucket show'",
+		Name:      cmdSetBprops,
+		Usage:     setBpropsUsage,
 		ArgsUsage: bucketPropsArgument,
 		Flags:     bucketCmdsFlags[cmdSetBprops],
 		Action:    setPropsHandler,

diff --git a/cmd/cli/cli/cluster_hdlr.go b/cmd/cli/cli/cluster_hdlr.go
@@ -24,6 +24,17 @@ const (
 	roleTargetShort = "t"
 )
 
+// (compare with getLogUsage)
+const getCluLogsUsage = "download log archives from all clustered nodes (one TAR.GZ per node), e.g.:\n" +
+	indent4 + "\t - 'download-logs /tmp/www' - save log archives to /tmp/www directory\n" +
+	indent4 + "\t - 'download-logs --severity w' - errors and warnings to /tmp directory\n" +
+	indent4 + "\t   (see related: 'ais log show', 'ais log get')"
+
+const shutdownUsage = "shutdown a node, gracefully or immediately;\n" +
+	indent4 + "\tnote: upon shutdown the node won't be decommissioned - it'll remain in the cluster map\n" +
+	indent4 + "\tand can be manually restarted to rejoin the cluster at any later time;\n" +
+	indent4 + "\tsee also: 'ais advanced " + cmdRmSmap + "'"
+
 var (
 	clusterCmdsFlags = map[string][]cli.Flag{
 		cmdCluAttach: {},
@@ -130,11 +141,8 @@ var (
 				BashComplete: suggestProxies,
 			},
 			{
-				Name: cmdDownloadLogs,
-				Usage: "download log archives from all clustered nodes (one TAR.GZ per node), e.g.:\n" +
-					indent4 + "\t - 'download-logs /tmp/www' - save log archives to /tmp/www directory\n" +
-					indent4 + "\t - 'download-logs --severity w' - errors and warnings to system temporary directory\n" +
-					indent4 + "\t   (see related: 'ais log show', 'ais log get')",
+				Name:      cmdDownloadLogs,
+				Usage:     getCluLogsUsage,
 				ArgsUsage: "[OUT_DIR]",
 				Flags:     []cli.Flag{logSevFlag},
 				Action:    downloadAllLogs,
@@ -190,11 +198,8 @@ var (
 						BashComplete: suggestAllNodes,
 					},
 					{
-						Name: cmdShutdown,
-						Usage: "shutdown a node, gracefully or immediately;\n" +
-							indent4 + "\tnote: upon shutdown the node won't be decommissioned - it'll remain in the cluster map\n" +
-							indent4 + "\tand can be manually restarted to rejoin the cluster at any later time;\n" +
-							indent4 + "\tsee also: 'ais advanced " + cmdRmSmap + "'",
+						Name:         cmdShutdown,
+						Usage:        shutdownUsage,
 						ArgsUsage:    nodeIDArgument,
 						Flags:        clusterCmdsFlags[cmdShutdown+".node"],
 						Action:       nodeMaintShutDecommHandler,

diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go
@@ -5,7 +5,6 @@
 package cli
 
 import (
-	"strings"
 	"time"
 
 	"github.com/NVIDIA/aistore/api/apc"
@@ -217,10 +216,20 @@ const (
 	tabHelpDet = "press <TAB-TAB> to select, '--help' for details"
 )
 
-// `ArgsUsage`: argument placeholders in help messages
+// indentation
+const (
+	indent1 = "   "
+	indent2 = "      "       // repeat(indent1, 2)
+	indent4 = "            " // repeat(indent1, 4)
+)
+
 const (
-	indent1 = "   " // indent4 et al.
+	archFormats = ".tar, .tgz or .tar.gz, .zip, .tar.lz4" // namely, archive.FileExtensions
+	archExts    = "(" + archFormats + ")"
+)
 
+// `ArgsUsage`: argument placeholders in help messages
+const (
 	// Job IDs (download, dsort)
 	jobIDArgument                 = "JOB_ID"
 	optionalJobIDArgument         = "[JOB_ID]"
@@ -353,12 +362,6 @@ const (
 //
 
 var (
-	indent2 = strings.Repeat(indent1, 2)
-	indent4 = strings.Repeat(indent1, 4)
-
-	archFormats = ".tar, .tgz or .tar.gz, .zip, .tar.lz4" // namely, archive.FileExtensions
-	archExts    = "(" + archFormats + ")"
-
 	//
 	// scope 'all'
 	//