local playground: environment vs STDIN; TAGS vs AIS_BACKEND_PROVIDERS

* (usability)

Signed-off-by: Alex Aizman <[email protected]>

Makefile

@@ -111,7 +111,7 @@ endif
 	@echo "done."
 
 cli: ## Build CLI binary. NOTE: 1) a separate go.mod, 2) static linkage with cgo disabled
-	@echo "Building ais (CLI) => $(BUILD_DEST)/ais [build tags:$(BUILD_TAGS)]"
+	@echo "Building ais (CLI) [build tags:$(BUILD_TAGS)]"
 ifdef CROSS_COMPILE_CLI
 	cd $(BUILD_DIR)/cli && \
 	$(CROSS_COMPILE_CLI) go build -o ./ais -tags="$(BUILD_TAGS)" $(BUILD_FLAGS) $(LDFLAGS) *.go && mv ./ais $(BUILD_DEST)/.

ais/target.go

@@ -6,6 +6,7 @@ package ais
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -158,11 +159,19 @@ func (t *target) _initBuiltin(tstats *stats.Trunner) error {
 			notlinked = append(notlinked, provider)
 		}
 	}
+	const c = "configured but missing in the build"
 	switch {
 	case len(notlinked) > 0:
-		return fmt.Errorf("%s backends: enabled %v, disabled %v, missing in the build %v", t, enabled, disabled, notlinked)
+		s := fmt.Sprintf("%s: %v backends are "+c, t, notlinked)
+		if len(notlinked) == 1 {
+			s = fmt.Sprintf("%s: %s backend is "+c, t, notlinked[0])
+		}
+		if len(enabled) == 0 && len(disabled) == 0 {
+			return errors.New(s)
+		}
+		return fmt.Errorf("%s (enabled %v, disabled %v)", s, enabled, disabled)
 	case len(disabled) > 0:
-		nlog.Warningf("%s backends: enabled %v, disabled %v", t, enabled, disabled)
+		nlog.Warningf("%s some backends are disabled via configuration: (enabled %v, disabled %v)", t, enabled, disabled)
 	default:
 		nlog.Infoln(t.String(), "backends:", enabled)
 	}

deploy/dev/docker/deploy_docker.sh

@@ -273,7 +273,7 @@ if [ $DRYRUN -ne 0 ]; then
     is_size $DRYOBJSIZE
 fi
 
-parse_backend_providers
+set_env_backends
 
 touch $LOCAL_AWS
 echo "Configured backend providers: '${AIS_BACKEND_PROVIDERS}'"

deploy/dev/k8s/utils/parse_cld.sh

@@ -3,7 +3,7 @@ export LOCAL_AWS="/tmp/credentials"
 touch $LOCAL_AWS
 export AIS_BACKEND_PROVIDERS="" # See deploy.sh for more information about empty AIS_BACKEND_PROVIDERS.
 source ../utils.sh
-parse_backend_providers
+set_env_backends
 
 if [[ "${AIS_BACKEND_PROVIDERS}" == *aws* ]]; then
     echo "Enter the location of your AWS configuration and credentials files:"

deploy/dev/local/aisnode_config.fspaths.sh

@@ -1,18 +1,9 @@
 # NOTE: system environment variables are listed in the `env` package,
 # see https://github.com/NVIDIA/aistore/blob/main/api/env/README.md
 
-backend_desc=()
-for backend in ${AIS_BACKEND_PROVIDERS}; do
-  case $backend in
-    aws)   backend_desc+=('"aws":   {}') ;;
-    azure) backend_desc+=('"azure": {}') ;;
-    gcp)   backend_desc+=('"gcp":   {}') ;;
-  esac
-done
-
 cat > $AIS_CONF_FILE <<EOL
 {
-	"backend": {$(IFS=$','; echo "${backend_desc[*]}")},
+	"backend": $(make_backend_conf),
 	"mirror": {
 		"copies":       2,
 		"burst_buffer": 128,

deploy/dev/local/aisnode_config.sh

@@ -1,18 +1,9 @@
 # NOTE: system environment variables are listed in the `env` package,
 # see https://github.com/NVIDIA/aistore/blob/main/api/env/README.md
 
-backend_desc=()
-for backend in ${AIS_BACKEND_PROVIDERS}; do
-  case $backend in
-    aws)   backend_desc+=('"aws":   {}') ;;
-    azure) backend_desc+=('"azure": {}') ;;
-    gcp)   backend_desc+=('"gcp":   {}') ;;
-  esac
-done
-
 cat > $AIS_CONF_FILE <<EOL
 {
-	"backend": {$(IFS=$','; echo "${backend_desc[*]}")},
+	"backend": $(make_backend_conf),
 	"mirror": {
 		"copies":       2,
 		"burst_buffer": 128,

deploy/dev/local/deploy.sh

@@ -157,6 +157,8 @@ fi
 START=0
 END=$((TARGET_CNT + PROXY_CNT - 1))
 
+### begin reading STDIN ---------------------------------------------------------------------
+
 echo "Number of local mountpaths (enter 0 for preconfigured filesystems):"
 read test_fspath_cnt
 is_number ${test_fspath_cnt}
@@ -167,9 +169,11 @@ TEST_LOOPBACK_COUNT=0
 # If not specified, AIS_BACKEND_PROVIDERS will remain empty (or `0`) and
 # aisnode build will include neither AWS ("aws") nor GCP ("gcp").
 
-parse_backend_providers
+set_env_backends
+
+create_loopbacks_or_skip
 
-create_loopbacks
+### end reading STDIN ------------------------------------------------------------------------
 
 ## NOTE: to enable StatsD instead of Prometheus, use build tag `statsd` in the make command, as follows:
 ## TAGS=statsd make ...

deploy/dev/utils.sh

@@ -30,35 +30,80 @@ run_cmd() {
 #  { set +x; } 2>/dev/null
 }
 
-# NOTE: `AIS_BACKEND_PROVIDERS` and all other system environment variables
-# are listed in the `env` package:
+# NOTE 1:
+# AIS_BACKEND_PROVIDERS and all other system environment variables are listed in the `env` package:
 # https://github.com/NVIDIA/aistore/blob/main/api/env/README.md
 
-parse_backend_providers() {
+# NOTE 2:
+# defined AIS_BACKEND_PROVIDERS (empty or non-empty) always takes precedence over STDIN
+
+set_env_backends() {
+  if [[ ! -z $TAGS ]]; then
+    ## env var TAGS may contain all build tags, including backends
+    arr=( aws gcp azure )
+    for b in "${arr[@]}"; do
+      re="\\b$b\\b"
+      if [[ $TAGS =~ $re && ! $AIS_BACKEND_PROVIDERS =~ $re ]]; then
+        AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} $b"
+      fi
+    done
+  fi
+
+  if [[  -v AIS_BACKEND_PROVIDERS ]]; then
+    ## env takes precedence over STDIN
+    local orig=$AIS_BACKEND_PROVIDERS
+    _set_env_backends
+    AIS_BACKEND_PROVIDERS=$orig
+  else
+    _set_env_backends
+  fi
+}
+
+_set_env_backends() {
   AIS_BACKEND_PROVIDERS=""
-  echo "Select backend providers:"
+  echo "Select backend providers (press Enter at any point to stop adding backends):"
   echo "Amazon S3: (y/n) ?"
   read -r cld_aws
+  if [[ "$cld_aws" == "" ]] ; then
+    return
+  fi
   is_boolean "${cld_aws}"
-  echo "Google Cloud Storage: (y/n) ?"
-  read -r cld_gcp
-  is_boolean "${cld_gcp}"
-  echo "Azure: (y/n) ?"
-  read -r cld_azure
-  is_boolean "${cld_azure}"
-
   if  [[ "${cld_aws}" == "y" ]] ; then
     AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} aws"
   fi
+  echo "Google Cloud Storage: (y/n) ?"
+  read -r cld_gcp
+  if [[ "$cld_gcp" == "" ]] ; then
+    return
+  fi
+  is_boolean "${cld_gcp}"
   if  [[ "${cld_gcp}" == "y" ]] ; then
     AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} gcp"
   fi
+  echo "Azure: (y/n) ?"
+  read -r cld_azure
+  if [[ "$cld_azure" == "" ]] ; then
+    return
+  fi
+  is_boolean "${cld_azure}"
   if  [[ "${cld_azure}" == "y" ]] ; then
     AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} azure"
   fi
 }
 
-create_loopbacks() {
+make_backend_conf() {
+  backend_conf=()
+  for backend in ${AIS_BACKEND_PROVIDERS}; do
+    case $backend in
+      aws)   backend_conf+=('"aws":   {}') ;;
+      azure) backend_conf+=('"azure": {}') ;;
+      gcp)   backend_conf+=('"gcp":   {}') ;;
+    esac
+  done
+  echo {$(IFS=$','; echo "${backend_conf[*]}")}
+}
+
+create_loopbacks_or_skip() {
   echo "Loopback device size, e.g. 10G, 100M (press Enter to skip): "
   read -r loopback_size
   if [[ "$loopback_size" == "" || "$loopback_size" == "0" ]] ; then