Added location flag to list command to specify location for non standard URLs

cmd/list.go

@@ -41,7 +41,8 @@ import (
 
 type rawListCmdArgs struct {
 	// obtained from argument
-	sourcePath string
+	src      string
+	location string
 
 	Properties      string
 	MachineReadable bool
@@ -105,17 +106,25 @@ func (raw rawListCmdArgs) cook() (cookedListCmdArgs, error) {
 	cooked = cookedListCmdArgs{}
 	// the expected argument in input is the container sas / or path of virtual directory in the container.
 	// verifying the location type
-	location := InferArgumentLocation(raw.sourcePath)
+	var err error
+	cooked.location, err = ValidateArgumentLocation(raw.src, raw.location)
+	if err != nil {
+		return cooked, err
+	}
 	// Only support listing for Azure locations
-	if location != location.Blob() && location != location.File() && location != location.BlobFS() {
-		return cooked, errors.New("invalid path passed for listing. given source is of type " + location.String() + " while expect is container / container path ")
+	switch cooked.location {
+	case common.ELocation.Blob():
+	case common.ELocation.File():
+	case common.ELocation.BlobFS():
+		break
+	default:
+		return cooked, fmt.Errorf("azcopy only supports Azure resources for listing i.e. Blob, File, BlobFS")
 	}
-	cooked.sourcePath = raw.sourcePath
+	cooked.sourcePath = raw.src
 	cooked.MachineReadable = raw.MachineReadable
 	cooked.RunningTally = raw.RunningTally
 	cooked.MegaUnits = raw.MegaUnits
-	cooked.location = location
-	err := cooked.trailingDot.Parse(raw.trailingDot)
+	err = cooked.trailingDot.Parse(raw.trailingDot)
 	if err != nil {
 		return cooked, err
 	}
@@ -153,10 +162,10 @@ func init() {
 
 			// If no argument is passed then it is not valid
 			// lsc expects the container path / virtual directory
-			if len(args) == 0 || len(args) > 2 {
+			if len(args) != 1 {
 				return errors.New("this command only requires container destination")
 			}
-			raw.sourcePath = args[0]
+			raw.src = args[0]
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
@@ -174,6 +183,7 @@ func init() {
 		},
 	}
 
+	listContainerCmd.PersistentFlags().StringVar(&raw.location, "location", "", "Optionally specifies the location. For Example: Blob, File, BlobFS")
 	listContainerCmd.PersistentFlags().BoolVar(&raw.MachineReadable, "machine-readable", false, "Lists file sizes in bytes.")
 	listContainerCmd.PersistentFlags().BoolVar(&raw.RunningTally, "running-tally", false, "Counts the total number of files and their sizes.")
 	listContainerCmd.PersistentFlags().BoolVar(&raw.MegaUnits, "mega-units", false, "Displays units in orders of 1000, not 1024.")
@@ -196,7 +206,7 @@ func (cooked cookedListCmdArgs) handleListContainerCommand() (err error) {
 		return err
 	}
 
-	if err := common.VerifyIsURLResolvable(raw.sourcePath); cooked.location.IsRemote() && err != nil {
+	if err := common.VerifyIsURLResolvable(raw.src); cooked.location.IsRemote() && err != nil {
 		return fmt.Errorf("failed to resolve target: %w", err)
 	}
 

cmd/remove.go

@@ -45,8 +45,8 @@ func init() {
 			// the resource to delete is set as the source
 			raw.src = args[0]
 
-			srcLocationType := InferArgumentLocation(raw.src)
 			if raw.fromTo == "" {
+				srcLocationType := InferArgumentLocation(raw.src)
 				switch srcLocationType {
 				case common.ELocation.Blob():
 					raw.fromTo = common.EFromTo.BlobTrash().String()
@@ -60,7 +60,7 @@ func init() {
 			} else if raw.fromTo != "" {
 				err := strings.Contains(raw.fromTo, "Trash")
 				if !err {
-					return fmt.Errorf("Invalid destination. Please enter a valid destination, i.e. BlobTrash, FileTrash, BlobFSTrash")
+					return fmt.Errorf("invalid destination. please enter a valid destination, i.e. BlobTrash, FileTrash, BlobFSTrash")
 				}
 			}
 			raw.setMandatoryDefaults()
@@ -117,8 +117,8 @@ func init() {
 	deleteCmd.PersistentFlags().StringVar(&raw.permanentDeleteOption, "permanent-delete", "none", "This is a preview feature that PERMANENTLY deletes soft-deleted snapshots/versions. Possible values include 'snapshots', 'versions', 'snapshotsandversions', 'none'.")
 	deleteCmd.PersistentFlags().StringVar(&raw.includeBefore, common.IncludeBeforeFlagName, "", "Include only those files modified before or on the given date/time. The value should be in ISO8601 format. If no timezone is specified, the value is assumed to be in the local timezone of the machine running AzCopy. E.g. '2020-08-19T15:04:00Z' for a UTC time, or '2020-08-19' for midnight (00:00) in the local timezone. As of AzCopy 10.7, this flag applies only to files, not folders, so folder properties won't be copied when using this flag with --preserve-smb-info or --preserve-smb-permissions.")
 	deleteCmd.PersistentFlags().StringVar(&raw.includeAfter, common.IncludeAfterFlagName, "", "Include only those files modified on or after the given date/time. The value should be in ISO8601 format. If no timezone is specified, the value is assumed to be in the local timezone of the machine running AzCopy. E.g. '2020-08-19T15:04:00Z' for a UTC time, or '2020-08-19' for midnight (00:00) in the local timezone. As of AzCopy 10.5, this flag applies only to files, not folders, so folder properties won't be copied when using this flag with --preserve-smb-info or --preserve-smb-permissions.")
-	deleteCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. " +
-		"Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation." +
+	deleteCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. "+
+		"Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation."+
 		"If the destination does not support trailing dot files (Windows or Blob Storage), AzCopy will fail if the trailing dot file is the root of the transfer and skip any trailing dot paths encountered during enumeration.")
 	// Public Documentation: https://docs.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys
 	// Clients making requests against Azure Blob storage have the option to provide an encryption key on a per-request basis.

cmd/validators.go

@@ -84,6 +84,37 @@ var fromToHelp = func() string {
 
 var fromToHelpText = fromToHelp
 
+const locationHelpFormat = "Specified to nudge AzCopy when resource detection may not work (e.g. emulator/azure stack); Valid Location are Source words (e.g. Blob, File) that specify the source resource type. All valid Locations are: %s"
+
+var locationHelp = func() string {
+	validLocations := ""
+
+	isSafeToOutput := func(loc common.Location) bool {
+		switch loc {
+		case common.ELocation.Benchmark(),
+			common.ELocation.None(),
+			common.ELocation.Unknown():
+			return false
+		default:
+			return true
+		}
+	}
+
+	enum.GetSymbols(reflect.TypeOf(common.ELocation), func(enumSymbolName string, enumSymbolValue interface{}) (stop bool) {
+		location := enumSymbolValue.(common.Location)
+
+		if isSafeToOutput(location) {
+			validLocations += location.String() + ", "
+		}
+
+		return false
+	})
+
+	return fmt.Sprintf(locationHelpFormat, strings.TrimSuffix(validLocations, ", "))
+}()
+
+var locationHelpText = locationHelp
+
 func inferFromTo(src, dst string) common.FromTo {
 	// Try to infer the 1st argument
 	srcLocation := InferArgumentLocation(src)
@@ -132,6 +163,28 @@ func inferFromTo(src, dst string) common.FromTo {
 
 var IPv4Regex = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) // simple regex
 
+func ValidateArgumentLocation(src string, userSpecifiedLocation string) (common.Location, error) {
+	if userSpecifiedLocation == "" {
+		inferredLocation := InferArgumentLocation(src)
+
+		// If user didn't explicitly specify Location, use what was inferred (if possible)
+		if inferredLocation == common.ELocation.Unknown() {
+			return common.ELocation.Unknown(), fmt.Errorf("the inferred location could not be identified, or is currently not supported")
+		}
+		return inferredLocation, nil
+	}
+
+	// User explicitly specified Location, therefore, we should respect what they specified.
+	var userLocation common.Location
+	err := userLocation.Parse(userSpecifiedLocation)
+	if err != nil {
+		return common.ELocation.Unknown(), fmt.Errorf("invalid --location value specified: %q. "+locationHelpText, userSpecifiedLocation)
+
+	}
+
+	return userLocation, nil
+}
+
 func InferArgumentLocation(arg string) common.Location {
 	if arg == pipeLocation {
 		return common.ELocation.Pipe()
@@ -164,6 +217,7 @@ func InferArgumentLocation(arg string) common.Location {
 			if common.IsGCPURL(*u) {
 				return common.ELocation.GCP()
 			}
+			return common.ELocation.Unknown()
 		}
 	}
 

cmd/validators_test.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestValidateArgumentLocation(t *testing.T) {
+	a := assert.New(t)
+
+	test := []struct {
+		src                   string
+		userSpecifiedLocation string
+
+		expectedLocation common.Location
+		expectedError    string
+	}{
+		// User does not specify location
+		{"https://test.blob.core.windows.net/container1", "", common.ELocation.Blob(), ""},
+		{"https://test.file.core.windows.net/container1", "", common.ELocation.File(), ""},
+		{"https://test.dfs.core.windows.net/container1", "", common.ELocation.BlobFS(), ""},
+		{"https://s3.amazonaws.com/bucket", "", common.ELocation.S3(), ""},
+		{"https://storage.cloud.google.com/bucket", "", common.ELocation.GCP(), ""},
+		{"https://privateendpoint.com/container1", "", common.ELocation.Unknown(), "the inferred location could not be identified, or is currently not supported"},
+		{"http://127.0.0.1:10000/devstoreaccount1/container1", "", common.ELocation.Unknown(), "the inferred location could not be identified, or is currently not supported"},
+
+		// User specifies location
+		{"https://privateendpoint.com/container1", "FILE", common.ELocation.File(), ""},
+		{"http://127.0.0.1:10000/devstoreaccount1/container1", "BloB", common.ELocation.Blob(), ""},
+		{"https://test.file.core.windows.net/container1", "blobfs", common.ELocation.BlobFS(), ""}, // Tests that the endpoint does not really matter
+		{"https://privateendpoint.com/container1", "random", common.ELocation.Unknown(), "invalid --location value specified"},
+	}
+
+	for _, v := range test {
+		loc, err := ValidateArgumentLocation(v.src, v.userSpecifiedLocation)
+		a.Equal(v.expectedLocation, loc)
+		a.Equal(err == nil, v.expectedError == "")
+		if err != nil {
+			a.Contains(err.Error(), v.expectedError)
+		}
+	}
+}

common/fe-ste-models.go

@@ -515,6 +515,14 @@ func (l Location) String() string {
 	return enum.StringInt(l, reflect.TypeOf(l))
 }
 
+func (l *Location) Parse(s string) error {
+	val, err := enum.ParseInt(reflect.TypeOf(l), s, true, true)
+	if err == nil {
+		*l = val.(Location)
+	}
+	return err
+}
+
 // AllStandardLocations returns all locations that are "normal" for testing purposes. Excludes the likes of Unknown, Benchmark and Pipe
 func (Location) AllStandardLocations() []Location {
 	return []Location{