build: demote ht:// backend; revise local-playground scripts

* add build tag `ht`; link `ht://` conditionally
* related commit: 50db672

Signed-off-by: Alex Aizman <[email protected]>

.github/workflows/lint.yml

@@ -35,7 +35,7 @@ jobs:
           export GOPATH="$(go env GOPATH)"
           make lint
           TAGS=statsd make lint
-          TAGS="statsd nethttp debug" make lint
+          TAGS="statsd nethttp ht debug" make lint
           TAGS="aws gcp azure" make lint
           make fmt-check
           make spell-check

.gitlab-ci.yml

@@ -122,8 +122,8 @@ build:linux:
   script:
     - MODE="" make node # Build node without backends in production mode.
     - MODE="debug" make node # Build node without backends in debug mode.
-    - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="" make node # Build with all backends (production mode).
-    - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="debug" make node # Build with all backends (debug mode).
+    - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="" make node # Build with all Cloud backends (production mode).
+    - AIS_BACKEND_PROVIDERS="aws azure gcp ht" MODE="debug" make node # Build with all backends (debug mode).
     - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile.
     - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default).
     - make authn
@@ -144,7 +144,7 @@ build:k8s:
   before_script:
     - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_TOKEN $CI_REGISTRY
   script:
-    - buildah images 
+    - buildah images
     - buildah build -t $FQ_IMAGE_NAME -f deploy/dev/k8s/Dockerfile --build-arg MODE="debug" --build-arg providers="gcp"
     - buildah images
     - buildah push $FQ_IMAGE_NAME
@@ -166,6 +166,7 @@ lint:linux:
   <<: *default_only_def
   script:
     - make lint
+    - TAGS="statsd ht" make lint
     - make fmt-check
     - make spell-check
 
@@ -311,7 +312,7 @@ test:long:
     NUM_PROXY: 6
     BUCKET: "ais://ais-ci"
   script:
-    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
+    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --ht
     - make test-long
 
 test:long:aws:
@@ -320,7 +321,7 @@ test:long:aws:
     NUM_PROXY: 6
     BUCKET: "aws://ais-cicd"
   script:
-    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws
+    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws --ht
     - make test-long
 
 test:long:gcp:
@@ -329,7 +330,7 @@ test:long:gcp:
     NUM_PROXY: 6
     BUCKET: "gs://ais-ci"
   script:
-    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
+    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --ht
     - make test-long
 
 test:long:azure:
@@ -338,7 +339,7 @@ test:long:azure:
     NUM_PROXY: 6
     BUCKET: "az://ais-ci"
   script:
-    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure
+    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure --ht
     - make test-long
 
 test:long:aisloader:

Makefile

@@ -315,8 +315,6 @@ help:
 		"make kill clean" "Stop locally deployed cluster and cleanup all cluster-related data and bucket metadata (but not cluster map)" \
 		"make kill deploy <<< $$'7\n2\n4\ny\ny\nn\n0\n'"  "Shutdown and then (non-interactively) generate local configs and deploy a cluster consisting of 7 targets (4 mountpaths each) and 2 proxies; build 'aisnode' executable with the support for GCP and AWS backends" \
 		"make restart <<< $$'7\n2\n4\ny\ny\nn\nn\n0\n'"  "Restart a cluster of 7 targets (4 mountpaths each) and 2 proxies; utilize previously generated (pre-shutdown) local configurations" \
-		"RUN_ARGS=-override_backends MODE=debug make kill deploy <<< $$'4\n1\n4\nn\nn\nn\nn\n0\n'"  "Redeploy (4 targets + 1 proxy) cluster; build executable for debug without any backend-supporting libraries; use RUN_ARGS to pass additional command-line option ('-override_backends=true') to each running node"\
-		"RUN_ARGS='-override_backends -standby' MODE=debug make kill deploy <<< $$'4\n1\n4\nn\nn\nn\nn\n0\n'"  "Same as above, but additionally run all 4 targets in a standby mode"\
 		"make kill clean cli deploy <<< $$'7\n2\n4\ny\ny\nn\n1G\n'"  "Shutdown, cleanup, build CLI, and redeploy from scratch; create 4 loopback devices (size = 1G, one loopback per mountpath)" \
 		"GORACE='log_path=/tmp/race' make deploy" "Deploy cluster with race detector, write reports to /tmp/race.<PID>" \
 		"MODE=debug make deploy" "Deploy cluster with 'aisnode' (AIS target and proxy) executable built with debug symbols and debug asserts enabled" \

ais/backend/common.go

@@ -199,9 +199,6 @@ func calcPageSize(pageSize, maxPageSize int64) int64 {
 	return min(pageSize, maxPageSize)
 }
 
-//nolint:deadcode,unused // used by dummy backends
-func newErrInitBackend(provider string) error { return &cmn.ErrInitBackend{Provider: provider} }
-
 func allocPutParams(res core.GetReaderResult, owt cmn.OWT) *core.PutParams {
 	params := core.AllocPutParams()
 	{

ais/backend/http.go → ais/backend/ht.go

@@ -1,3 +1,5 @@
+//go:build ht
+
 // Package backend contains implementation of various backend providers.
 /*
  * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
@@ -32,14 +34,14 @@ type (
 // interface guard
 var _ core.Backend = (*htbp)(nil)
 
-func NewHTTP(t core.TargetPut, config *cmn.Config, tstats stats.Tracker) core.Backend {
+func NewHT(t core.TargetPut, config *cmn.Config, tstats stats.Tracker) (core.Backend, error) {
 	bp := &htbp{
 		t:    t,
-		base: base{provider: apc.HTTP},
+		base: base{provider: apc.HT},
 	}
 	bp.cliH, bp.cliTLS = cmn.NewDefaultClients(config.Client.TimeoutLong.D())
 	bp.init(t.Snode(), tstats)
-	return bp
+	return bp, nil
 }
 
 func (htbp *htbp) client(u string) *http.Client {
@@ -79,7 +81,7 @@ func (htbp *htbp) HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.S
 	}
 
 	bckProps = make(cos.StrKVs)
-	bckProps[apc.HdrBackendProvider] = apc.HTTP
+	bckProps[apc.HdrBackendProvider] = apc.HT
 	return
 }
 
@@ -128,7 +130,7 @@ func (htbp *htbp) HeadObj(ctx context.Context, lom *core.LOM, _ *http.Request) (
 		return nil, resp.StatusCode, fmt.Errorf("error occurred: %v", resp.StatusCode)
 	}
 	oa = &cmn.ObjAttrs{}
-	oa.SetCustomKey(cmn.SourceObjMD, apc.HTTP)
+	oa.SetCustomKey(cmn.SourceObjMD, apc.HT)
 	if resp.ContentLength >= 0 {
 		oa.Size = resp.ContentLength
 	}
@@ -196,7 +198,7 @@ func (htbp *htbp) GetObjReader(ctx context.Context, lom *core.LOM, offset, lengt
 		nlog.Infof("[HTTP CLOUD][GET] success, size: %d", resp.ContentLength)
 	}
 
-	lom.SetCustomKey(cmn.SourceObjMD, apc.HTTP)
+	lom.SetCustomKey(cmn.SourceObjMD, apc.HT)
 	lom.SetCustomKey(cmn.OrigURLObjMD, origURL)
 	if v, ok := h.EncodeVersion(resp.Header.Get(cos.HdrETag)); ok {
 		lom.SetCustomKey(cmn.ETag, v)

ais/backend/mock_aws.go

@@ -19,7 +19,7 @@ import (
 )
 
 func NewAWS(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
-	return nil, newErrInitBackend(apc.AWS)
+	return nil, &cmn.ErrInitBackend{Provider: apc.AWS}
 }
 
 func StartMpt(*core.LOM, *http.Request, url.Values) (string, int, error) {

ais/backend/mock_azure.go

@@ -8,10 +8,11 @@ package backend
 
 import (
 	"github.com/NVIDIA/aistore/api/apc"
+	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/core"
 	"github.com/NVIDIA/aistore/stats"
 )
 
 func NewAzure(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
-	return nil, newErrInitBackend(apc.Azure)
+	return nil, &cmn.ErrInitBackend{Provider: apc.Azure}
 }

ais/backend/mock_gcp.go

@@ -8,10 +8,11 @@ package backend
 
 import (
 	"github.com/NVIDIA/aistore/api/apc"
+	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/core"
 	"github.com/NVIDIA/aistore/stats"
 )
 
 func NewGCP(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) {
-	return nil, newErrInitBackend(apc.GCP)
+	return nil, &cmn.ErrInitBackend{Provider: apc.GCP}
 }

ais/backend/mock_ht.go

@@ -0,0 +1,18 @@
+//go:build !ht
+
+// Package backend contains implementation of various backend providers.
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ */
+package backend
+
+import (
+	"github.com/NVIDIA/aistore/api/apc"
+	"github.com/NVIDIA/aistore/cmn"
+	"github.com/NVIDIA/aistore/core"
+	"github.com/NVIDIA/aistore/stats"
+)
+
+func NewHT(core.TargetPut, *cmn.Config, stats.Tracker) (core.Backend, error) {
+	return nil, &cmn.ErrInitBackend{Provider: apc.HT}
+}

ais/bucketmeta.go

@@ -527,7 +527,7 @@ func mergeRemoteBckProps(props *cmn.Bprops, header http.Header) *cmn.Bprops {
 		props.Extra.AWS.CloudRegion = header.Get(apc.HdrS3Region)
 		props.Extra.AWS.Endpoint = header.Get(apc.HdrS3Endpoint)
 		props.Extra.AWS.Profile = header.Get(apc.HdrS3Profile)
-	case apc.HTTP:
+	case apc.HT:
 		props.Extra.HTTP.OrigURLBck = header.Get(apc.HdrOrigURLBck)
 	}
 

ais/proxy.go

@@ -1387,7 +1387,7 @@ func (p *proxy) _bckpost(w http.ResponseWriter, r *http.Request, msg *apc.ActMsg
 			eq = true
 			nlog.Warningf("multi-object operation %q within the same bucket %q", msg.Action, bck)
 		}
-		if bckTo.IsHTTP() {
+		if bckTo.IsHT() {
 			p.writeErrf(w, r, "cannot %s to HTTP bucket %q", msg.Action, bckTo)
 			return
 		}
@@ -1621,7 +1621,7 @@ func (p *proxy) listObjects(w http.ResponseWriter, r *http.Request, bck *meta.Bc
 	case lsmsg.Props == apc.GetPropsNameSize:
 		lsmsg.SetFlag(apc.LsNameSize)
 	}
-	if bck.IsHTTP() || lsmsg.IsFlagSet(apc.LsArchDir) {
+	if bck.IsHT() || lsmsg.IsFlagSet(apc.LsArchDir) {
 		lsmsg.SetFlag(apc.LsObjCached)
 	}
 
@@ -2113,7 +2113,7 @@ func (p *proxy) listBuckets(w http.ResponseWriter, r *http.Request, qbck *cmn.Qu
 		bmd     = p.owner.bmd.get()
 		present bool
 	)
-	if qbck.IsAIS() || qbck.IsHTTP() {
+	if qbck.IsAIS() || qbck.IsHT() {
 		bcks := bmd.Select(qbck)
 		p.writeJSON(w, r, bcks, "list-buckets")
 		return
@@ -3129,11 +3129,11 @@ func (p *proxy) htHandler(w http.ResponseWriter, r *http.Request) {
 		nlog.Infof("[HTTP CLOUD] RevProxy handler for: %s -> %s", baseURL, r.URL.Path)
 	}
 	if r.Method == http.MethodGet || r.Method == http.MethodHead {
-		// bck.IsHTTP()
+		// bck.IsHT()
 		hbo := cmn.NewHTTPObj(r.URL)
 		q := r.URL.Query()
 		q.Set(apc.QparamOrigURL, r.URL.String())
-		q.Set(apc.QparamProvider, apc.HTTP)
+		q.Set(apc.QparamProvider, apc.HT)
 		r.URL.Path = apc.URLPathObjects.Join(hbo.Bck.Name, hbo.ObjName)
 		r.URL.RawQuery = q.Encode()
 		if r.Method == http.MethodGet {
@@ -3143,7 +3143,7 @@ func (p *proxy) htHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		return
 	}
-	p.writeErrf(w, r, "%q provider doesn't support %q", apc.HTTP, r.Method)
+	p.writeErrf(w, r, "%q provider doesn't support %q", apc.HT, r.Method)
 }
 
 //
@@ -3348,12 +3348,12 @@ func (p *proxy) headRemoteBck(bck *cmn.Bck, q url.Values) (header http.Header, s
 	if tsi, err = smap.GetRandTarget(); err != nil {
 		return
 	}
-	if bck.IsCloud() {
+	if bck.IsBuiltTagged() {
 		config := cmn.GCO.Get()
 		if config.Backend.Get(bck.Provider) == nil {
 			err = &cmn.ErrMissingBackend{Provider: bck.Provider}
 			statusCode = http.StatusNotFound
-			err = cmn.NewErrFailedTo(p, "lookup Cloud bucket", bck, err, statusCode)
+			err = cmn.NewErrFailedTo(p, "lookup bucket", bck, err, statusCode)
 			return
 		}
 	}

ais/prxauth.go

@@ -266,7 +266,7 @@ func (p *proxy) access(hdr http.Header, bck *meta.Bck, ace apc.AccessAttrs) (err
 		tk, err = p.validateToken(hdr)
 		if err != nil {
 			// NOTE: making exception to allow 3rd party clients read remote ht://bucket
-			if err == tok.ErrNoToken && bck != nil && bck.IsHTTP() {
+			if err == tok.ErrNoToken && bck != nil && bck.IsHT() {
 				err = nil
 			}
 			return err

ais/prxbck.go

@@ -163,7 +163,7 @@ func (bctx *bctx) accessSupported() error {
 		goto rerr
 	}
 	// HTTP buckets are not writeable
-	if bctx.bck.IsHTTP() && bctx._perm(apc.AcePUT) {
+	if bctx.bck.IsHT() && bctx._perm(apc.AcePUT) {
 		op = "write to HTTP bucket"
 		goto rerr
 	}
@@ -292,7 +292,7 @@ func (bctx *bctx) _try() (bck *meta.Bck, ecode int, err error) {
 	}
 
 	// orig-url for the ht:// bucket
-	if bck.IsHTTP() {
+	if bck.IsHT() {
 		if bctx.origURLBck != "" {
 			remoteHdr.Set(apc.HdrOrigURLBck, bctx.origURLBck)
 		} else {
@@ -364,7 +364,7 @@ func (bctx *bctx) lookup(bck *meta.Bck) (hdr http.Header, code int, err error) {
 		q       = url.Values{}
 		retried bool
 	)
-	if bck.IsHTTP() {
+	if bck.IsHT() {
 		origURL := bctx.getOrigURL()
 		q.Set(apc.QparamOrigURL, origURL)
 	}

ais/prxtxn.go

@@ -208,7 +208,7 @@ func (p *proxy) createBucket(msg *apc.ActMsg, bck *meta.Bck, remoteHdr http.Head
 		if bck.IsCloud() {
 			return cmn.NewErrNotImpl("create", bck.Provider+"(cloud) bucket")
 		}
-		if bck.IsHTTP() {
+		if bck.IsHT() {
 			return cmn.NewErrNotImpl("create", "bucket for HTTP provider")
 		}
 		// can do remote ais though

ais/target.go

@@ -106,8 +106,7 @@ func (*target) interruptedRestarted() (interrupted, restarted bool) {
 func (t *target) initBackends(tstats *stats.Trunner) {
 	config := cmn.GCO.Get()
 	aisbp := backend.NewAIS(t, tstats)
-	t.backend[apc.AIS] = aisbp                               // always present
-	t.backend[apc.HTTP] = backend.NewHTTP(t, config, tstats) // ditto
+	t.backend[apc.AIS] = aisbp // always present
 
 	if aisConf := config.Backend.Get(apc.AIS); aisConf != nil {
 		if err := aisbp.Apply(aisConf, "init", &config.ClusterConfig); err != nil {
@@ -117,15 +116,14 @@ func (t *target) initBackends(tstats *stats.Trunner) {
 		}
 	}
 
-	if err := t._initBuiltin(tstats); err != nil {
+	if err := t._initBuiltTagged(tstats); err != nil {
 		cos.ExitLog(err)
 	}
 }
 
-// init built-in (via build tags) backends
 // - remote (e.g. cloud) backends  w/ empty stubs unless populated via build tags
 // - enabled/disabled via config.Backend
-func (t *target) _initBuiltin(tstats *stats.Trunner) error {
+func (t *target) _initBuiltTagged(tstats *stats.Trunner) error {
 	var (
 		enabled, disabled, notlinked []string
 		config                       = cmn.GCO.Get()
@@ -142,7 +140,9 @@ func (t *target) _initBuiltin(tstats *stats.Trunner) error {
 			add, err = backend.NewGCP(t, tstats)
 		case apc.Azure:
 			add, err = backend.NewAzure(t, tstats)
-		case apc.AIS, apc.HTTP:
+		case apc.HT:
+			add, err = backend.NewHT(t, config, tstats)
+		case apc.AIS:
 			continue
 		default:
 			return fmt.Errorf(cmn.FmtErrUnknown, t, "backend provider", provider)
@@ -761,7 +761,7 @@ func (t *target) getObject(w http.ResponseWriter, r *http.Request, dpq *dpq, bck
 	}
 
 	// apc.QparamOrigURL
-	if bck.IsHTTP() {
+	if bck.IsHT() {
 		originalURL := dpq.origURL
 		goi.ctx = context.WithValue(goi.ctx, cos.CtxOriginalURL, originalURL)
 	}

ais/test/bucket_test.go

@@ -45,7 +45,7 @@ func TestHTTPProviderBucket(t *testing.T) {
 	var (
 		bck = cmn.Bck{
 			Name:     t.Name() + "Bucket",
-			Provider: apc.HTTP,
+			Provider: apc.HT,
 		}
 		proxyURL   = tools.RandomProxyURL(t)
 		baseParams = tools.BaseAPIParams(proxyURL)

ais/test/object_test.go

@@ -186,6 +186,7 @@ func TestRemoteBucketObject(t *testing.T) {
 }
 
 func TestHttpProviderObjectGet(t *testing.T) {
+	tools.CheckSkip(t, &tools.SkipTestArgs{Long: true}) // NOTE: ht:// is now conditionally linked, requires 'ht' build tag
 	var (
 		proxyURL   = tools.RandomProxyURL()
 		baseParams = tools.BaseAPIParams(proxyURL)