From aa9f4288658ed07532916a1a64b8ac2fcb2d37ae Mon Sep 17 00:00:00 2001 From: Alex Aizman Date: Mon, 5 Aug 2024 12:50:57 -0400 Subject: [PATCH] build: demote `ht://` backend; revise local-playground scripts * add build tag `ht`; link `ht://` conditionally * related commit: 50db672cb34f90 Signed-off-by: Alex Aizman --- .github/workflows/lint.yml | 2 +- .gitlab-ci.yml | 15 ++++++----- Makefile | 2 -- ais/backend/common.go | 3 --- ais/backend/{http.go => ht.go} | 14 +++++----- ais/backend/mock_aws.go | 2 +- ais/backend/mock_azure.go | 3 ++- ais/backend/mock_gcp.go | 3 ++- ais/backend/mock_ht.go | 18 +++++++++++++ ais/bucketmeta.go | 2 +- ais/proxy.go | 16 +++++------ ais/prxauth.go | 2 +- ais/prxbck.go | 6 ++--- ais/prxtxn.go | 2 +- ais/target.go | 14 +++++----- ais/test/bucket_test.go | 2 +- ais/test/object_test.go | 1 + ais/test/rproxy_test.go | 4 ++- ais/tgtbck.go | 6 ++--- ais/tgtimpl.go | 3 --- api/apc/provider.go | 8 +++--- api/daemon.go | 8 +----- cmd/cli/cli/completions.go | 2 +- cmd/cli/cli/get.go | 4 +-- cmd/cli/cli/ls.go | 4 +-- cmd/cli/cli/parse_uri.go | 2 +- cmd/cli/cli/utils.go | 6 ++--- cmd/cli/cli/utils_internal_test.go | 12 ++++----- cmd/cli/go.mod | 2 +- cmd/cli/go.sum | 4 +-- cmn/api.go | 2 +- cmn/bck.go | 10 ++++--- cmn/config.go | 21 +++------------ cmn/cos/url.go | 2 +- cmn/err.go | 11 ++++---- core/meta/bck.go | 2 +- deploy/dev/local/deploy.sh | 4 +-- deploy/dev/utils.sh | 43 +++++++++++++++++++++--------- docs/command_line.md | 16 ++++++----- docs/development.md | 29 +++++++++++++++++--- ext/dload/job.go | 2 +- scripts/clean_deploy.sh | 39 +++++++++++++-------------- 42 files changed, 199 insertions(+), 154 deletions(-) rename ais/backend/{http.go => ht.go} (95%) create mode 100644 ais/backend/mock_ht.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 21d4593789..2942d6a6e4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: export GOPATH="$(go env GOPATH)" make lint TAGS=statsd make lint - TAGS="statsd nethttp debug" make lint + TAGS="statsd nethttp ht debug" make lint TAGS="aws gcp azure" make lint make fmt-check make spell-check diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00a5dfe29d..bdba34c79f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,8 +122,8 @@ build:linux: script: - MODE="" make node # Build node without backends in production mode. - MODE="debug" make node # Build node without backends in debug mode. - - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="" make node # Build with all backends (production mode). - - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="debug" make node # Build with all backends (debug mode). + - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="" make node # Build with all Cloud backends (production mode). + - AIS_BACKEND_PROVIDERS="aws azure gcp ht" MODE="debug" make node # Build with all backends (debug mode). - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile. - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default). - make authn @@ -144,7 +144,7 @@ build:k8s: before_script: - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_TOKEN $CI_REGISTRY script: - - buildah images + - buildah images - buildah build -t $FQ_IMAGE_NAME -f deploy/dev/k8s/Dockerfile --build-arg MODE="debug" --build-arg providers="gcp" - buildah images - buildah push $FQ_IMAGE_NAME @@ -166,6 +166,7 @@ lint:linux: <<: *default_only_def script: - make lint + - TAGS="statsd ht" make lint - make fmt-check - make spell-check @@ -311,7 +312,7 @@ test:long: NUM_PROXY: 6 BUCKET: "ais://ais-ci" script: - - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --ht - make test-long test:long:aws: @@ -320,7 +321,7 @@ test:long:aws: NUM_PROXY: 6 BUCKET: "aws://ais-cicd" script: - - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws --ht - make test-long test:long:gcp: @@ -329,7 +330,7 @@ test:long:gcp: NUM_PROXY: 6 BUCKET: "gs://ais-ci" script: - - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --ht - make test-long test:long:azure: @@ -338,7 +339,7 @@ test:long:azure: NUM_PROXY: 6 BUCKET: "az://ais-ci" script: - - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure --ht - make test-long test:long:aisloader: diff --git a/Makefile b/Makefile index 9ddc0780d5..fca0b38311 100644 --- a/Makefile +++ b/Makefile @@ -315,8 +315,6 @@ help: "make kill clean" "Stop locally deployed cluster and cleanup all cluster-related data and bucket metadata (but not cluster map)" \ "make kill deploy <<< $$'7\n2\n4\ny\ny\nn\n0\n'" "Shutdown and then (non-interactively) generate local configs and deploy a cluster consisting of 7 targets (4 mountpaths each) and 2 proxies; build 'aisnode' executable with the support for GCP and AWS backends" \ "make restart <<< $$'7\n2\n4\ny\ny\nn\nn\n0\n'" "Restart a cluster of 7 targets (4 mountpaths each) and 2 proxies; utilize previously generated (pre-shutdown) local configurations" \ - "RUN_ARGS=-override_backends MODE=debug make kill deploy <<< $$'4\n1\n4\nn\nn\nn\nn\n0\n'" "Redeploy (4 targets + 1 proxy) cluster; build executable for debug without any backend-supporting libraries; use RUN_ARGS to pass additional command-line option ('-override_backends=true') to each running node"\ - "RUN_ARGS='-override_backends -standby' MODE=debug make kill deploy <<< $$'4\n1\n4\nn\nn\nn\nn\n0\n'" "Same as above, but additionally run all 4 targets in a standby mode"\ "make kill clean cli deploy <<< $$'7\n2\n4\ny\ny\nn\n1G\n'" "Shutdown, cleanup, build CLI, and redeploy from scratch; create 4 loopback devices (size = 1G, one loopback per mountpath)" \ "GORACE='log_path=/tmp/race' make deploy" "Deploy cluster with race detector, write reports to /tmp/race." \ "MODE=debug make deploy" "Deploy cluster with 'aisnode' (AIS target and proxy) executable built with debug symbols and debug asserts enabled" \ diff --git a/ais/backend/common.go b/ais/backend/common.go index a3aa420da7..a170f3f1f9 100644 --- a/ais/backend/common.go +++ b/ais/backend/common.go @@ -199,9 +199,6 @@ func calcPageSize(pageSize, maxPageSize int64) int64 { return min(pageSize, maxPageSize) } -//nolint:deadcode,unused // used by dummy backends -func newErrInitBackend(provider string) error { return &cmn.ErrInitBackend{Provider: provider} } - func allocPutParams(res core.GetReaderResult, owt cmn.OWT) *core.PutParams { params := core.AllocPutParams() { diff --git a/ais/backend/http.go b/ais/backend/ht.go similarity index 95% rename from ais/backend/http.go rename to ais/backend/ht.go index 2d3e8fb6a1..7af43fcb26 100644 --- a/ais/backend/http.go +++ b/ais/backend/ht.go @@ -1,3 +1,5 @@ +//go:build ht + // Package backend contains implementation of various backend providers. /* * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. @@ -32,14 +34,14 @@ type ( // interface guard var _ core.Backend = (*htbp)(nil) -func NewHTTP(t core.TargetPut, config *cmn.Config, tstats stats.Tracker) core.Backend { +func NewHT(t core.TargetPut, config *cmn.Config, tstats stats.Tracker) (core.Backend, error) { bp := &htbp{ t: t, - base: base{provider: apc.HTTP}, + base: base{provider: apc.HT}, } bp.cliH, bp.cliTLS = cmn.NewDefaultClients(config.Client.TimeoutLong.D()) bp.init(t.Snode(), tstats) - return bp + return bp, nil } func (htbp *htbp) client(u string) *http.Client { @@ -79,7 +81,7 @@ func (htbp *htbp) HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.S } bckProps = make(cos.StrKVs) - bckProps[apc.HdrBackendProvider] = apc.HTTP + bckProps[apc.HdrBackendProvider] = apc.HT return } @@ -128,7 +130,7 @@ func (htbp *htbp) HeadObj(ctx context.Context, lom *core.LOM, _ *http.Request) ( return nil, resp.StatusCode, fmt.Errorf("error occurred: %v", resp.StatusCode) } oa = &cmn.ObjAttrs{} - oa.SetCustomKey(cmn.SourceObjMD, apc.HTTP) + oa.SetCustomKey(cmn.SourceObjMD, apc.HT) if resp.ContentLength >= 0 { oa.Size = resp.ContentLength } @@ -196,7 +198,7 @@ func (htbp *htbp) GetObjReader(ctx context.Context, lom *core.LOM, offset, lengt nlog.Infof("[HTTP CLOUD][GET] success, size: %d", resp.ContentLength) } - lom.SetCustomKey(cmn.SourceObjMD, apc.HTTP) + lom.SetCustomKey(cmn.SourceObjMD, apc.HT) lom.SetCustomKey(cmn.OrigURLObjMD, origURL) if v, ok := h.EncodeVersion(resp.Header.Get(cos.HdrETag)); ok { lom.SetCustomKey(cmn.ETag, v) diff --git a/ais/backend/mock_aws.go b/ais/backend/mock_aws.go index d62d9be366..fed9384e14 100644 --- a/ais/backend/mock_aws.go +++ b/ais/backend/mock_aws.go @@ -19,7 +19,7 @@ import ( ) func NewAWS(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) { - return nil, newErrInitBackend(apc.AWS) + return nil, &cmn.ErrInitBackend{Provider: apc.AWS} } func StartMpt(*core.LOM, *http.Request, url.Values) (string, int, error) { diff --git a/ais/backend/mock_azure.go b/ais/backend/mock_azure.go index 2d462d46fa..4addb0494e 100644 --- a/ais/backend/mock_azure.go +++ b/ais/backend/mock_azure.go @@ -8,10 +8,11 @@ package backend import ( "github.com/NVIDIA/aistore/api/apc" + "github.com/NVIDIA/aistore/cmn" "github.com/NVIDIA/aistore/core" "github.com/NVIDIA/aistore/stats" ) func NewAzure(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) { - return nil, newErrInitBackend(apc.Azure) + return nil, &cmn.ErrInitBackend{Provider: apc.Azure} } diff --git a/ais/backend/mock_gcp.go b/ais/backend/mock_gcp.go index 7648f8ac5c..66673ac732 100644 --- a/ais/backend/mock_gcp.go +++ b/ais/backend/mock_gcp.go @@ -8,10 +8,11 @@ package backend import ( "github.com/NVIDIA/aistore/api/apc" + "github.com/NVIDIA/aistore/cmn" "github.com/NVIDIA/aistore/core" "github.com/NVIDIA/aistore/stats" ) func NewGCP(_ core.TargetPut, _ stats.Tracker) (core.Backend, error) { - return nil, newErrInitBackend(apc.GCP) + return nil, &cmn.ErrInitBackend{Provider: apc.GCP} } diff --git a/ais/backend/mock_ht.go b/ais/backend/mock_ht.go new file mode 100644 index 0000000000..18edc42c84 --- /dev/null +++ b/ais/backend/mock_ht.go @@ -0,0 +1,18 @@ +//go:build !ht + +// Package backend contains implementation of various backend providers. +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + */ +package backend + +import ( + "github.com/NVIDIA/aistore/api/apc" + "github.com/NVIDIA/aistore/cmn" + "github.com/NVIDIA/aistore/core" + "github.com/NVIDIA/aistore/stats" +) + +func NewHT(core.TargetPut, *cmn.Config, stats.Tracker) (core.Backend, error) { + return nil, &cmn.ErrInitBackend{Provider: apc.HT} +} diff --git a/ais/bucketmeta.go b/ais/bucketmeta.go index 9eb1918f5f..f1e69a08ca 100644 --- a/ais/bucketmeta.go +++ b/ais/bucketmeta.go @@ -527,7 +527,7 @@ func mergeRemoteBckProps(props *cmn.Bprops, header http.Header) *cmn.Bprops { props.Extra.AWS.CloudRegion = header.Get(apc.HdrS3Region) props.Extra.AWS.Endpoint = header.Get(apc.HdrS3Endpoint) props.Extra.AWS.Profile = header.Get(apc.HdrS3Profile) - case apc.HTTP: + case apc.HT: props.Extra.HTTP.OrigURLBck = header.Get(apc.HdrOrigURLBck) } diff --git a/ais/proxy.go b/ais/proxy.go index ec2525142d..f844eaef21 100644 --- a/ais/proxy.go +++ b/ais/proxy.go @@ -1387,7 +1387,7 @@ func (p *proxy) _bckpost(w http.ResponseWriter, r *http.Request, msg *apc.ActMsg eq = true nlog.Warningf("multi-object operation %q within the same bucket %q", msg.Action, bck) } - if bckTo.IsHTTP() { + if bckTo.IsHT() { p.writeErrf(w, r, "cannot %s to HTTP bucket %q", msg.Action, bckTo) return } @@ -1621,7 +1621,7 @@ func (p *proxy) listObjects(w http.ResponseWriter, r *http.Request, bck *meta.Bc case lsmsg.Props == apc.GetPropsNameSize: lsmsg.SetFlag(apc.LsNameSize) } - if bck.IsHTTP() || lsmsg.IsFlagSet(apc.LsArchDir) { + if bck.IsHT() || lsmsg.IsFlagSet(apc.LsArchDir) { lsmsg.SetFlag(apc.LsObjCached) } @@ -2113,7 +2113,7 @@ func (p *proxy) listBuckets(w http.ResponseWriter, r *http.Request, qbck *cmn.Qu bmd = p.owner.bmd.get() present bool ) - if qbck.IsAIS() || qbck.IsHTTP() { + if qbck.IsAIS() || qbck.IsHT() { bcks := bmd.Select(qbck) p.writeJSON(w, r, bcks, "list-buckets") return @@ -3129,11 +3129,11 @@ func (p *proxy) htHandler(w http.ResponseWriter, r *http.Request) { nlog.Infof("[HTTP CLOUD] RevProxy handler for: %s -> %s", baseURL, r.URL.Path) } if r.Method == http.MethodGet || r.Method == http.MethodHead { - // bck.IsHTTP() + // bck.IsHT() hbo := cmn.NewHTTPObj(r.URL) q := r.URL.Query() q.Set(apc.QparamOrigURL, r.URL.String()) - q.Set(apc.QparamProvider, apc.HTTP) + q.Set(apc.QparamProvider, apc.HT) r.URL.Path = apc.URLPathObjects.Join(hbo.Bck.Name, hbo.ObjName) r.URL.RawQuery = q.Encode() if r.Method == http.MethodGet { @@ -3143,7 +3143,7 @@ func (p *proxy) htHandler(w http.ResponseWriter, r *http.Request) { } return } - p.writeErrf(w, r, "%q provider doesn't support %q", apc.HTTP, r.Method) + p.writeErrf(w, r, "%q provider doesn't support %q", apc.HT, r.Method) } // @@ -3348,12 +3348,12 @@ func (p *proxy) headRemoteBck(bck *cmn.Bck, q url.Values) (header http.Header, s if tsi, err = smap.GetRandTarget(); err != nil { return } - if bck.IsCloud() { + if bck.IsBuiltTagged() { config := cmn.GCO.Get() if config.Backend.Get(bck.Provider) == nil { err = &cmn.ErrMissingBackend{Provider: bck.Provider} statusCode = http.StatusNotFound - err = cmn.NewErrFailedTo(p, "lookup Cloud bucket", bck, err, statusCode) + err = cmn.NewErrFailedTo(p, "lookup bucket", bck, err, statusCode) return } } diff --git a/ais/prxauth.go b/ais/prxauth.go index 176094b84a..63cc432416 100644 --- a/ais/prxauth.go +++ b/ais/prxauth.go @@ -266,7 +266,7 @@ func (p *proxy) access(hdr http.Header, bck *meta.Bck, ace apc.AccessAttrs) (err tk, err = p.validateToken(hdr) if err != nil { // NOTE: making exception to allow 3rd party clients read remote ht://bucket - if err == tok.ErrNoToken && bck != nil && bck.IsHTTP() { + if err == tok.ErrNoToken && bck != nil && bck.IsHT() { err = nil } return err diff --git a/ais/prxbck.go b/ais/prxbck.go index 5f1926235e..6ebbaff5a6 100644 --- a/ais/prxbck.go +++ b/ais/prxbck.go @@ -163,7 +163,7 @@ func (bctx *bctx) accessSupported() error { goto rerr } // HTTP buckets are not writeable - if bctx.bck.IsHTTP() && bctx._perm(apc.AcePUT) { + if bctx.bck.IsHT() && bctx._perm(apc.AcePUT) { op = "write to HTTP bucket" goto rerr } @@ -292,7 +292,7 @@ func (bctx *bctx) _try() (bck *meta.Bck, ecode int, err error) { } // orig-url for the ht:// bucket - if bck.IsHTTP() { + if bck.IsHT() { if bctx.origURLBck != "" { remoteHdr.Set(apc.HdrOrigURLBck, bctx.origURLBck) } else { @@ -364,7 +364,7 @@ func (bctx *bctx) lookup(bck *meta.Bck) (hdr http.Header, code int, err error) { q = url.Values{} retried bool ) - if bck.IsHTTP() { + if bck.IsHT() { origURL := bctx.getOrigURL() q.Set(apc.QparamOrigURL, origURL) } diff --git a/ais/prxtxn.go b/ais/prxtxn.go index 04d97b2a33..cc32ed5bd3 100644 --- a/ais/prxtxn.go +++ b/ais/prxtxn.go @@ -208,7 +208,7 @@ func (p *proxy) createBucket(msg *apc.ActMsg, bck *meta.Bck, remoteHdr http.Head if bck.IsCloud() { return cmn.NewErrNotImpl("create", bck.Provider+"(cloud) bucket") } - if bck.IsHTTP() { + if bck.IsHT() { return cmn.NewErrNotImpl("create", "bucket for HTTP provider") } // can do remote ais though diff --git a/ais/target.go b/ais/target.go index e3e0e3605c..3abbe2323d 100644 --- a/ais/target.go +++ b/ais/target.go @@ -106,8 +106,7 @@ func (*target) interruptedRestarted() (interrupted, restarted bool) { func (t *target) initBackends(tstats *stats.Trunner) { config := cmn.GCO.Get() aisbp := backend.NewAIS(t, tstats) - t.backend[apc.AIS] = aisbp // always present - t.backend[apc.HTTP] = backend.NewHTTP(t, config, tstats) // ditto + t.backend[apc.AIS] = aisbp // always present if aisConf := config.Backend.Get(apc.AIS); aisConf != nil { if err := aisbp.Apply(aisConf, "init", &config.ClusterConfig); err != nil { @@ -117,15 +116,14 @@ func (t *target) initBackends(tstats *stats.Trunner) { } } - if err := t._initBuiltin(tstats); err != nil { + if err := t._initBuiltTagged(tstats); err != nil { cos.ExitLog(err) } } -// init built-in (via build tags) backends // - remote (e.g. cloud) backends w/ empty stubs unless populated via build tags // - enabled/disabled via config.Backend -func (t *target) _initBuiltin(tstats *stats.Trunner) error { +func (t *target) _initBuiltTagged(tstats *stats.Trunner) error { var ( enabled, disabled, notlinked []string config = cmn.GCO.Get() @@ -142,7 +140,9 @@ func (t *target) _initBuiltin(tstats *stats.Trunner) error { add, err = backend.NewGCP(t, tstats) case apc.Azure: add, err = backend.NewAzure(t, tstats) - case apc.AIS, apc.HTTP: + case apc.HT: + add, err = backend.NewHT(t, config, tstats) + case apc.AIS: continue default: return fmt.Errorf(cmn.FmtErrUnknown, t, "backend provider", provider) @@ -761,7 +761,7 @@ func (t *target) getObject(w http.ResponseWriter, r *http.Request, dpq *dpq, bck } // apc.QparamOrigURL - if bck.IsHTTP() { + if bck.IsHT() { originalURL := dpq.origURL goi.ctx = context.WithValue(goi.ctx, cos.CtxOriginalURL, originalURL) } diff --git a/ais/test/bucket_test.go b/ais/test/bucket_test.go index e2ee4992a0..24f027f8ef 100644 --- a/ais/test/bucket_test.go +++ b/ais/test/bucket_test.go @@ -45,7 +45,7 @@ func TestHTTPProviderBucket(t *testing.T) { var ( bck = cmn.Bck{ Name: t.Name() + "Bucket", - Provider: apc.HTTP, + Provider: apc.HT, } proxyURL = tools.RandomProxyURL(t) baseParams = tools.BaseAPIParams(proxyURL) diff --git a/ais/test/object_test.go b/ais/test/object_test.go index 809e013d2e..86a55461cd 100644 --- a/ais/test/object_test.go +++ b/ais/test/object_test.go @@ -186,6 +186,7 @@ func TestRemoteBucketObject(t *testing.T) { } func TestHttpProviderObjectGet(t *testing.T) { + tools.CheckSkip(t, &tools.SkipTestArgs{Long: true}) // NOTE: ht:// is now conditionally linked, requires 'ht' build tag var ( proxyURL = tools.RandomProxyURL() baseParams = tools.BaseAPIParams(proxyURL) diff --git a/ais/test/rproxy_test.go b/ais/test/rproxy_test.go index 83dce5fd2e..45c4734b04 100644 --- a/ais/test/rproxy_test.go +++ b/ais/test/rproxy_test.go @@ -84,6 +84,7 @@ func extractSpeed(out []byte) int64 { } func TestRProxyGCS(t *testing.T) { + tools.CheckSkip(t, &tools.SkipTestArgs{Long: true}) // NOTE: ht:// is now conditionally linked, requires 'ht' build tag var ( resURL = genObjURL(false, true) proxyURL = tools.GetPrimaryURL() @@ -98,7 +99,7 @@ func TestRProxyGCS(t *testing.T) { } initMountpaths(t, proxyURL) - bck := cmn.Bck{Provider: apc.HTTP} + bck := cmn.Bck{Provider: apc.HT} queryBck := cmn.QueryBcks(bck) bckList, err := api.ListBuckets(baseParams, queryBck, apc.FltExists) tassert.CheckFatal(t, err) @@ -193,6 +194,7 @@ retry: } func TestRProxyInvalidURL(t *testing.T) { + tools.CheckSkip(t, &tools.SkipTestArgs{Long: true}) // NOTE: ht:// is now conditionally linked, requires 'ht' build tag var ( proxyURL = tools.GetPrimaryURL() baseParams = tools.BaseAPIParams(proxyURL) diff --git a/ais/tgtbck.go b/ais/tgtbck.go index 06e7593e53..d707fc3df3 100644 --- a/ais/tgtbck.go +++ b/ais/tgtbck.go @@ -175,7 +175,7 @@ func (t *target) listBuckets(w http.ResponseWriter, r *http.Request, qbck *cmn.Q code int ) if qbck.Provider != "" { - if qbck.IsAIS() || qbck.IsHTTP() { // built-in providers + if qbck.IsAIS() || qbck.IsHT() { // built-in providers bcks = bmd.Select(qbck) } else { bcks, code, err = t.blist(qbck, config) @@ -191,7 +191,7 @@ func (t *target) listBuckets(w http.ResponseWriter, r *http.Request, qbck *cmn.Q for provider := range apc.Providers { var buckets cmn.Bcks qbck.Provider = provider - if qbck.IsAIS() || qbck.IsHTTP() { + if qbck.IsAIS() || qbck.IsHT() { buckets = bmd.Select(qbck) } else { buckets, code, err = t.blist(qbck, config) @@ -491,7 +491,7 @@ func (t *target) httpbckhead(w http.ResponseWriter, r *http.Request, apireq *api debug.Assert(!apireq.bck.IsAIS()) - if apireq.bck.IsHTTP() { + if apireq.bck.IsHT() { originalURL := apireq.query.Get(apc.QparamOrigURL) ctx = context.WithValue(ctx, cos.CtxOriginalURL, originalURL) if !inBMD && originalURL == "" { diff --git a/ais/tgtimpl.go b/ais/tgtimpl.go index 1c7fe90b97..4db955a663 100644 --- a/ais/tgtimpl.go +++ b/ais/tgtimpl.go @@ -41,9 +41,6 @@ func (t *target) Backend(bck *meta.Bck) core.Backend { if bck.IsRemoteAIS() { return t.backend[apc.AIS] } - if bck.IsHTTP() { - return t.backend[apc.HTTP] - } provider := bck.Provider if bck.Props != nil { provider = bck.RemoteBck().Provider diff --git a/api/apc/provider.go b/api/apc/provider.go index a99df4a561..526349734f 100644 --- a/api/apc/provider.go +++ b/api/apc/provider.go @@ -14,7 +14,7 @@ const ( AWS = "aws" Azure = "azure" GCP = "gcp" - HTTP = "ht" + HT = "ht" AllProviders = "ais, aws (s3://), gcp (gs://), azure (az://), ht://" // NOTE: must include all @@ -34,7 +34,7 @@ const ( const RemAIS = "remais" // to differentiate ais vs ais; also, default (remote ais cluster) alias -var Providers = cos.NewStrSet(AIS, GCP, AWS, Azure, HTTP) +var Providers = cos.NewStrSet(AIS, GCP, AWS, Azure, HT) func IsProvider(p string) bool { return Providers.Contains(p) } @@ -44,7 +44,7 @@ func IsCloudProvider(p string) bool { // NOTE: not to confuse w/ bck.IsRemote() which also includes remote AIS func IsRemoteProvider(p string) bool { - return IsCloudProvider(p) || p == HTTP + return IsCloudProvider(p) || p == HT } func ToScheme(p string) string { @@ -88,7 +88,7 @@ func DisplayProvider(p string) string { return "Azure" case GCP, GSScheme: return "GCP" - case HTTP: + case HT: return "HTTP(S)" default: return p diff --git a/api/daemon.go b/api/daemon.go index abaafee761..db40515feb 100644 --- a/api/daemon.go +++ b/api/daemon.go @@ -119,13 +119,7 @@ func GetDaemonConfig(bp BaseParams, node *meta.Snode) (config *cmn.Config, err e } _, err = reqParams.DoReqAny(&config) FreeRp(reqParams) - if err != nil { - return nil, err - } - // FIXME: transform backend structures on the client side - // as a side effect, config.Backend validation populates non-JSON structs that client can utilize; - _ = config.Backend.Validate() - return config, nil + return config, err } // names _and_ kinds, i.e. (name, kind) pairs diff --git a/cmd/cli/cli/completions.go b/cmd/cli/cli/completions.go index e1fe974a3e..bec8e257d9 100644 --- a/cmd/cli/cli/completions.go +++ b/cmd/cli/cli/completions.go @@ -521,7 +521,7 @@ func bpropsFilterExtra(c *cli.Context, tag string) bool { switch c.Args().Get(0) { case apc.S3Scheme, apc.AWS: return strings.HasPrefix(tag, "extra.aws") - case apc.HTTP: + case apc.HT: return strings.HasPrefix(tag, "extra.http") } return false diff --git a/cmd/cli/cli/get.go b/cmd/cli/cli/get.go index c26b398686..26118c9c5a 100644 --- a/cmd/cli/cli/get.go +++ b/cmd/cli/cli/get.go @@ -77,7 +77,7 @@ func getHandler(c *cli.Context) error { if err != nil { return err } - if !bck.IsHTTP() { + if !bck.IsHT() { if bck.Props, err = headBucket(bck, false /* don't add */); err != nil { return err } @@ -631,7 +631,7 @@ func (a *qparamArch) getQuery(c *cli.Context, bck *cmn.Bck) (q url.Values) { q = make(url.Values, 4) } } - if bck.IsHTTP() { + if bck.IsHT() { f() uri := c.Args().Get(0) q.Set(apc.QparamOrigURL, uri) diff --git a/cmd/cli/cli/ls.go b/cmd/cli/cli/ls.go index f6010388e7..f7b3b9daa1 100644 --- a/cmd/cli/cli/ls.go +++ b/cmd/cli/cli/ls.go @@ -91,7 +91,7 @@ func listBckTableNoSummary(c *cli.Context, qbck cmn.QueryBcks, bcks cmn.Bcks, fl if info.IsBckPresent { footer.nbp++ } - if bck.IsHTTP() { + if bck.IsHT() { if bmd == nil { bmd, err = api.GetBMD(apiBP) if err != nil { @@ -186,7 +186,7 @@ func listBckTableWithSummary(c *cli.Context, qbck cmn.QueryBcks, bcks cmn.Bcks, footer.size += info.TotalSize.OnDisk footer.pct += int(info.UsedPct) } - if bck.IsHTTP() { + if bck.IsHT() { bck.Name += " (URL: " + props.Extra.HTTP.OrigURLBck + ")" } data = append(data, teb.ListBucketsHelper{XactID: xid, Bck: bck, Props: props, Info: info}) diff --git a/cmd/cli/cli/parse_uri.go b/cmd/cli/cli/parse_uri.go index 4acfbe39b7..49362cebde 100644 --- a/cmd/cli/cli/parse_uri.go +++ b/cmd/cli/cli/parse_uri.go @@ -128,7 +128,7 @@ func parseDest(c *cli.Context, uri string) (bck cmn.Bck, pathSuffix string, err bck, pathSuffix, err = parseBckObjURI(c, uri, true /*optional objName*/) if err != nil { return - } else if bck.IsHTTP() { + } else if bck.IsHT() { err = errors.New("http bucket is not supported as destination") return } diff --git a/cmd/cli/cli/utils.go b/cmd/cli/cli/utils.go index 66898e4ea0..9ac01f6303 100644 --- a/cmd/cli/cli/utils.go +++ b/cmd/cli/cli/utils.go @@ -133,7 +133,7 @@ func reorderTailArgs(left string, middle []string, right ...string) string { return strings.TrimSuffix(sb.String(), " ") } -func isWebURL(url string) bool { return cos.IsHTTP(url) || cos.IsHTTPS(url) } +func isWebURL(url string) bool { return cos.IsHT(url) || cos.IsHTTPS(url) } func jsonMarshalIndent(v any) ([]byte, error) { return jsoniter.MarshalIndent(v, "", " ") } @@ -595,7 +595,7 @@ func bckPropList(props *cmn.Bprops, verbose bool) (propList nvpairList) { {"lru", props.LRU.String()}, {"versioning", props.Versioning.String()}, } - if props.Provider == apc.HTTP { + if props.Provider == apc.HT { origURL := props.Extra.HTTP.OrigURLBck if origURL != "" { propList = append(propList, nvpair{Name: "original-url", Value: origURL}) @@ -718,7 +718,7 @@ func parseURLtoBck(strURL string) (bck cmn.Bck) { if !cos.IsLastB(strURL, '/') { strURL += "/" } - bck.Provider = apc.HTTP + bck.Provider = apc.HT bck.Name = cmn.OrigURLBck2Name(strURL) return } diff --git a/cmd/cli/cli/utils_internal_test.go b/cmd/cli/cli/utils_internal_test.go index 8946b8034d..b41283fdb8 100644 --- a/cmd/cli/cli/utils_internal_test.go +++ b/cmd/cli/cli/utils_internal_test.go @@ -305,8 +305,8 @@ func TestParseQueryBckURI(t *testing.T) { {uri: "ais://bucket", bck: cmn.QueryBcks{Provider: apc.AIS, Name: "bucket"}}, {uri: "ais://#ns/bucket", bck: cmn.QueryBcks{Provider: apc.AIS, Name: "bucket", Ns: cmn.Ns{Name: "ns"}}}, {uri: "ais://@uuid#ns/bucket", bck: cmn.QueryBcks{Provider: apc.AIS, Name: "bucket", Ns: cmn.Ns{Name: "ns", UUID: "uuid"}}}, - {uri: "http://web.url/dataset", bck: cmn.QueryBcks{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, - {uri: "https://web.url/dataset", bck: cmn.QueryBcks{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, + {uri: "http://web.url/dataset", bck: cmn.QueryBcks{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, + {uri: "https://web.url/dataset", bck: cmn.QueryBcks{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, } for _, test := range positiveTests { bck, err := parseQueryBckURI(&cli.Context{}, test.uri) @@ -341,8 +341,8 @@ func TestParseBckURI(t *testing.T) { {uri: "ais://bucket", bck: cmn.Bck{Provider: apc.AIS, Name: "bucket"}}, {uri: "ais://#ns/bucket", bck: cmn.Bck{Provider: apc.AIS, Name: "bucket", Ns: cmn.Ns{Name: "ns"}}}, {uri: "ais://@uuid#ns/bucket", bck: cmn.Bck{Provider: apc.AIS, Name: "bucket", Ns: cmn.Ns{Name: "ns", UUID: "uuid"}}}, - {uri: "http://web.url/dataset", bck: cmn.Bck{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, - {uri: "https://web.url/dataset", bck: cmn.Bck{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, + {uri: "http://web.url/dataset", bck: cmn.Bck{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, + {uri: "https://web.url/dataset", bck: cmn.Bck{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}}, } for _, test := range positiveTests { bck, err := parseBckURI(&cli.Context{}, test.uri, true /*require provider*/) @@ -411,12 +411,12 @@ func TestParseBckObjectURI(t *testing.T) { }, { uri: "http://web.url/dataset/object_name", - bck: cmn.Bck{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}, + bck: cmn.Bck{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}, objName: "object_name", }, { uri: "https://web.url/dataset/object_name", - bck: cmn.Bck{Provider: apc.HTTP, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}, + bck: cmn.Bck{Provider: apc.HT, Name: "ZWUyYWFiOGEzYjEwMTJkNw"}, objName: "object_name", }, } diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index d3f14c1f78..f24e5e8c86 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli go 1.22.3 require ( - github.com/NVIDIA/aistore v1.3.24-0.20240803001017-7a15bb331ebe + github.com/NVIDIA/aistore v1.3.24-0.20240804185917-fb2c0726306b github.com/fatih/color v1.17.0 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo/v2 v2.19.0 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index 5e4a6ba304..78c7101b46 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/NVIDIA/aistore v1.3.24-0.20240803001017-7a15bb331ebe h1:buGZhZ+75V1YzVBx/MDpP7ZXOX4NpoqZzDDwqbDiNAs= -github.com/NVIDIA/aistore v1.3.24-0.20240803001017-7a15bb331ebe/go.mod h1:A4wCIW7GooZSzDxTxh4pS092Ve9gCiXh1EvtjlVB8ew= +github.com/NVIDIA/aistore v1.3.24-0.20240804185917-fb2c0726306b h1:3WXudE1CSePptfWp7xe6wrHfp9o2iA/OApTYqWiIGIE= +github.com/NVIDIA/aistore v1.3.24-0.20240804185917-fb2c0726306b/go.mod h1:A4wCIW7GooZSzDxTxh4pS092Ve9gCiXh1EvtjlVB8ew= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= diff --git a/cmn/api.go b/cmn/api.go index 79d41bef68..910a6db6d0 100644 --- a/cmn/api.go +++ b/cmn/api.go @@ -279,7 +279,7 @@ func NewBpropsToSet(nvs cos.StrKVs) (props *BpropsToSet, err error) { func (c *ExtraProps) ValidateAsProps(arg ...any) error { provider, ok := arg[0].(string) debug.Assert(ok) - if provider == apc.HTTP && c.HTTP.OrigURLBck == "" { + if provider == apc.HT && c.HTTP.OrigURLBck == "" { return errors.New("original bucket URL must be set for a bucket with HTTP provider") } return nil diff --git a/cmn/bck.go b/cmn/bck.go index f77c51947b..3c02e8f3ae 100644 --- a/cmn/bck.go +++ b/cmn/bck.go @@ -372,7 +372,7 @@ func (b *Bck) IsAIS() bool { } func (b *Bck) IsRemoteAIS() bool { return b.Provider == apc.AIS && b.Ns.IsRemote() } -func (b *Bck) IsHTTP() bool { return b.Provider == apc.HTTP } +func (b *Bck) IsHT() bool { return b.Provider == apc.HT } func (b *Bck) IsRemote() bool { return apc.IsRemoteProvider(b.Provider) || b.IsRemoteAIS() || b.Backend() != nil @@ -382,6 +382,10 @@ func (b *Bck) IsRemote() bool { // NOTE: for more Is* accessors (e.g. IsRemoteS3), see also: core/meta/bck.go // +func (b *Bck) IsBuiltTagged() bool { + return b.IsCloud() || b.Provider == apc.HT +} + func (b *Bck) IsCloud() bool { if apc.IsCloudProvider(b.Provider) { return true @@ -472,7 +476,7 @@ func (qbck QueryBcks) String() string { } func (qbck *QueryBcks) IsAIS() bool { b := (*Bck)(qbck); return b.IsAIS() } -func (qbck *QueryBcks) IsHTTP() bool { b := (*Bck)(qbck); return b.IsHTTP() } +func (qbck *QueryBcks) IsHT() bool { b := (*Bck)(qbck); return b.IsHT() } func (qbck *QueryBcks) IsRemoteAIS() bool { b := (*Bck)(qbck); return b.IsRemoteAIS() } func (qbck *QueryBcks) IsCloud() bool { return apc.IsCloudProvider(qbck.Provider) } @@ -574,7 +578,7 @@ func (bcks Bcks) Equal(other Bcks) bool { func NewHTTPObj(u *url.URL) *HTTPBckObj { hbo := &HTTPBckObj{ Bck: Bck{ - Provider: apc.HTTP, + Provider: apc.HT, Ns: NsGlobal, }, } diff --git a/cmn/config.go b/cmn/config.go index f77b0a8b75..136bff6fc9 100644 --- a/cmn/config.go +++ b/cmn/config.go @@ -165,11 +165,8 @@ type ( } BackendConf struct { - // Provider implementation-dependent. We are using custom marshaling - // which populates this field. - Conf map[string]any `json:"-"` - // 3rd party Cloud(s) -- set during validation - Providers map[string]Ns `json:"-"` + Conf map[string]any `json:"-"` // backend implementation-dependent (custom marshaling to populate this field) + Providers map[string]Ns `json:"-"` // conditional (build tag) providers set during validation (BackendConf.Validate) } BackendConfAIS map[string][]string // cluster alias -> [urls...] @@ -888,7 +885,7 @@ func (c *BackendConf) Validate() (err error) { func (c *BackendConf) setProvider(provider string) { var ns Ns switch provider { - case apc.AWS, apc.Azure, apc.GCP: + case apc.AWS, apc.Azure, apc.GCP, apc.HT: ns = NsGlobal default: debug.Assert(false, "unknown backend provider "+provider) @@ -910,18 +907,6 @@ func (c *BackendConf) Set(provider string, newConf any) { c.Conf[provider] = newConf } -func (c *BackendConf) EqualClouds(o *BackendConf) bool { - if len(o.Conf) != len(c.Conf) { - return false - } - for k := range o.Conf { - if _, ok := c.Conf[k]; !ok { - return false - } - } - return true -} - func (c *BackendConf) EqualRemAIS(o *BackendConf, sname string) bool { var oldRemotes, newRemotes BackendConfAIS oais, oko := o.Conf[apc.AIS] diff --git a/cmn/cos/url.go b/cmn/cos/url.go index 459055aa49..f63cf61a87 100644 --- a/cmn/cos/url.go +++ b/cmn/cos/url.go @@ -24,7 +24,7 @@ const ( ) func IsHTTPS(url string) bool { return strings.HasPrefix(url, "https://") } -func IsHTTP(url string) bool { return strings.HasPrefix(url, "http://") } +func IsHT(url string) bool { return strings.HasPrefix(url, "http://") } func ParseURL(s string) (u *url.URL, valid bool) { if s == "" { diff --git a/cmn/err.go b/cmn/err.go index 7486196c7c..cf21b15b43 100644 --- a/cmn/err.go +++ b/cmn/err.go @@ -706,17 +706,18 @@ func AsErrAborted(err error) (errAborted *ErrAborted) { // ErrInitBackend & ErrMissingBackend func (e *ErrInitBackend) Error() string { - return fmt.Sprintf( - "cannot initialize %q backend (present in the cluster configuration): missing %s-supporting libraries in the build", - e.Provider, e.Provider, - ) + p := apc.DisplayProvider(e.Provider) + s := "cannot initialize " + p + " backend " + s += "(present in the cluster configuration): " + s += "missing " + p + "-supporting libraries in the build" + return s } func (e *ErrMissingBackend) Error() string { if e.Msg != "" { return e.Msg } - return fmt.Sprintf("%q backend is missing in the cluster configuration", e.Provider) + return apc.DisplayProvider(e.Provider) + " backend is missing in the cluster configuration" } // ErrETL diff --git a/core/meta/bck.go b/core/meta/bck.go index f2b381cc56..03bfdd9744 100644 --- a/core/meta/bck.go +++ b/core/meta/bck.go @@ -48,7 +48,7 @@ func (b *Bck) Bucket() *cmn.Bck { return (*cmn.Bck)(b) } func (b *Bck) IsAIS() bool { return (*cmn.Bck)(b).IsAIS() } func (b *Bck) HasProvider() bool { return (*cmn.Bck)(b).HasProvider() } -func (b *Bck) IsHTTP() bool { return (*cmn.Bck)(b).IsHTTP() } +func (b *Bck) IsHT() bool { return (*cmn.Bck)(b).IsHT() } func (b *Bck) IsCloud() bool { return (*cmn.Bck)(b).IsCloud() } func (b *Bck) IsRemote() bool { return (*cmn.Bck)(b).IsRemote() } func (b *Bck) IsRemoteAIS() bool { return (*cmn.Bck)(b).IsRemoteAIS() } diff --git a/deploy/dev/local/deploy.sh b/deploy/dev/local/deploy.sh index db87e07dd6..337a8ee558 100755 --- a/deploy/dev/local/deploy.sh +++ b/deploy/dev/local/deploy.sh @@ -166,8 +166,8 @@ TEST_FSPATH_COUNT=${test_fspath_cnt} TEST_LOOPBACK_COUNT=0 -# If not specified, AIS_BACKEND_PROVIDERS will remain empty (or `0`) and -# aisnode build will include neither AWS ("aws") nor GCP ("gcp"). +# If not specified, AIS_BACKEND_PROVIDERS will remain empty and +# aisnode build won't include any conditionally-linked backends (AWS, GCP, et al.) set_env_backends diff --git a/deploy/dev/utils.sh b/deploy/dev/utils.sh index 7bb3142eea..109143a96a 100644 --- a/deploy/dev/utils.sh +++ b/deploy/dev/utils.sh @@ -30,29 +30,47 @@ run_cmd() { # { set +x; } 2>/dev/null } -# NOTE 1: -# AIS_BACKEND_PROVIDERS and all other system environment variables are listed in the `env` package: -# https://github.com/NVIDIA/aistore/blob/main/api/env/README.md - -# NOTE 2: -# defined AIS_BACKEND_PROVIDERS (empty or non-empty) always takes precedence over STDIN +# NOTE: +# 1. AIS_BACKEND_PROVIDERS and all other system environment variables are listed in the `env` package: +# https://github.com/NVIDIA/aistore/blob/main/api/env/README.md +# 2. environment AIS_BACKEND_PROVIDERS (empty or non-empty) +# always takes precedence over STDIN +# 3. when adding/deleting backends, update the 3 (three) functions that follow below: set_env_backends() { + known_backends=( aws gcp azure ht ) if [[ ! -z $TAGS ]]; then - ## env var TAGS may contain all build tags, including backends - arr=( aws gcp azure ) - for b in "${arr[@]}"; do + ## environment var TAGS may contain any/all build tags, including backends + for b in "${known_backends[@]}"; do re="\\b$b\\b" - if [[ $TAGS =~ $re && ! $AIS_BACKEND_PROVIDERS =~ $re ]]; then - AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} $b" + if [[ $TAGS =~ $re ]]; then + if [[ ! $AIS_BACKEND_PROVIDERS =~ $re ]]; then + AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} $b" + fi + ## dedup + TAGS=${TAGS//$b/} fi done fi if [[ -v AIS_BACKEND_PROVIDERS ]]; then - ## env takes precedence over STDIN + ## environment takes precedence over STDIN local orig=$AIS_BACKEND_PROVIDERS + + ## validate + for b in ${AIS_BACKEND_PROVIDERS}; do + case $b in + aws) ;; + azure) ;; + gcp) ;; + ht) ;; + *) echo "fatal: unknown backend '$b' in 'AIS_BACKEND_PROVIDERS=${AIS_BACKEND_PROVIDERS}'"; exit 1;; + esac + done + ## consume (and discard) all reads _set_env_backends + + ## restore from env AIS_BACKEND_PROVIDERS=$orig else _set_env_backends @@ -98,6 +116,7 @@ make_backend_conf() { aws) backend_conf+=('"aws": {}') ;; azure) backend_conf+=('"azure": {}') ;; gcp) backend_conf+=('"gcp": {}') ;; + ht) backend_conf+=('"ht": {}') ;; esac done echo {$(IFS=$','; echo "${backend_conf[*]}")} diff --git a/docs/command_line.md b/docs/command_line.md index c4cc98f5e3..4bcba20a70 100644 --- a/docs/command_line.md +++ b/docs/command_line.md @@ -22,6 +22,8 @@ $ aisnode -config=/etc/ais/config.json -local_config=/etc/ais/local_config.json The common executable, typically called `aisnode`, supports the following command-line arguments: ```console + -allow_shared_no_disks + NOTE: deprecated, will be removed in future releases -config string config filename: local file that stores the global cluster configuration -config_custom string @@ -31,21 +33,21 @@ The common executable, typically called `aisnode`, supports the following comman -h show usage and exit -local_config string config filename: local file that stores daemon's local configuration + -loopback + use loopback devices (local playground, target-only) -ntargets int number of storage targets expected to be joining at startup (optional, primary-only) - -override_backends - configure remote backends at deployment time (potentially, override previously stored configuration) -role string _role_ of this aisnode: 'proxy' OR 'target' -skip_startup whether primary, when starting up, should skip waiting for target joins (used only in tests) -standby - when starting up, do not try to join cluster - standby and wait for admin request (target-only) + when starting up, do not try to auto-join cluster - stand by and wait for admin request (target-only) + -start_with_lost_mountpath + force starting up with a lost or missing mountpath (target-only) -transient - false: store customized (via config_custom) configuration - true: runtime only (non-persistent) - -v value - log level for V logs + false: store customized (via '-config_custom') configuration + true: keep '-config_custom' settings in memory only (non-persistent) ``` For usage and the most recently updated set of command-line options, run `aisnode` with empty command-line: diff --git a/docs/development.md b/docs/development.md index aeca64b6ec..09db2caee5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -84,11 +84,34 @@ $ make help ### Clean deploy ```console -./clean_deploy.sh [--target-cnt TARGET_CNT] [--proxy-cnt PROXY_CNT] [--mountpath-cnt MOUNTPATH_CNT] [--https] [--deployment local|remote|all] [--remote-alias REMOTE_ALIAS] [--PROVIDER ...] [--debug PKG=LOG_LEVEL[,PKG=LOG_LEVEL]] [--loopback SIZE] [--cleanup] [--dir] -[--override_backends] [--standby] [--transient] [--debug] +$ ./scripts/clean_deploy.sh -h +NAME: + clean_deploy.sh - locally deploy AIS cluster(s) + +USAGE: + ./clean_deploy.sh [options...] + +OPTIONS: + --target-cnt Number of target nodes in the cluster (default: 5) + --proxy-cnt Number of proxies/gateways (default: 5) + --mountpath-cnt Number of mountpaths (default: 5) + --cleanup Cleanup data and metadata from the previous deployments + --deployment Choose which AIS cluster(s) to deploy, one of: 'local', 'remote', 'all' (default: 'local') + --remote-alias Alias to assign to the remote cluster (default: 'remais') + --aws Build with AWS S3 backend + --gcp Build with Google Cloud Storage backend + --azure Build with Azure Blob Storage backend + --ht Build with ht:// backend (experimental) + --loopback Loopback device size, e.g. 10G, 100M (default: 0). Zero size means emulated mountpaths (with no loopback devices). + --dir The root directory of the aistore repository + --https Use HTTPS (note: X509 certificates may be required) + --standby When starting up, do not join cluster - wait instead for admin request (advanced usage, target-only) + --transient Do not store config changes, keep all the updates in memory + -h, --help Show this help text ``` -Deploys a new instance of an AIS cluster after killing any current instances. +Deploys a new instance of an AIS cluster after shutting down currently running cluster(s), if any. + To make it even more convenient, consider setting up an alias: ```bash diff --git a/ext/dload/job.go b/ext/dload/job.go index ed6fdadf60..5c3e5ba02e 100644 --- a/ext/dload/job.go +++ b/ext/dload/job.go @@ -332,7 +332,7 @@ func (j *rangeDlJob) getNextObjs() error { func newBackendDlJob(id string, bck *meta.Bck, payload *BackendBody, xdl *Xact) (bj *backendDlJob, err error) { if !bck.IsRemote() { return nil, errors.New("bucket download requires a remote bucket") - } else if bck.IsHTTP() { + } else if bck.IsHT() { return nil, errors.New("bucket download does not support HTTP buckets") } bj = &backendDlJob{} diff --git a/scripts/clean_deploy.sh b/scripts/clean_deploy.sh index 9b1caf7dee..3de0f342d6 100755 --- a/scripts/clean_deploy.sh +++ b/scripts/clean_deploy.sh @@ -23,15 +23,14 @@ function retry { root_dir="$(cd "$(dirname "$0")/../"; pwd -P)" ## NOTE: this assumes `clean_deploy.sh` itself is one level below # Default values -aws_provider="n" -azure_provider="n" -gcp_provider="n" +AIS_BACKEND_PROVIDERS="" + loopback=0 target_cnt=5 proxy_cnt=5 mountpath_cnt=5 deployment="local" -remote_alias="rmtais" +remote_alias="remais" cleanup="false" usage="NAME: @@ -46,14 +45,14 @@ OPTIONS: --mountpath-cnt Number of mountpaths (default: 5) --cleanup Cleanup data and metadata from the previous deployments --deployment Choose which AIS cluster(s) to deploy, one of: 'local', 'remote', 'all' (default: 'local') - --remote-alias Alias to assign to the remote cluster (default: 'rmtais') - --aws Support AWS S3 backend (i.e., build \`aisnode\` executable with AWS S3 SDK) - --gcp Support Google Cloud Platform (i.e., build \`aisnode\` with libraries to access GCP) - --azure Support Azure Cloud (experimental) - --loopback Loopback device size, e.g. 10G, 100M (default: 0). Zero size means: no loopbacks. + --remote-alias Alias to assign to the remote cluster (default: 'remais') + --aws Build with AWS S3 backend + --gcp Build with Google Cloud Storage backend + --azure Build with Azure Blob Storage backend + --ht Build with ht:// backend (experimental) + --loopback Loopback device size, e.g. 10G, 100M (default: 0). Zero size means emulated mountpaths (with no loopback devices). --dir The root directory of the aistore repository - --https Use HTTPS - --override_backends Configure remote backends at deployment time (override previously stored backend configuration) + --https Use HTTPS (note: X509 certificates may be required) --standby When starting up, do not join cluster - wait instead for admin request (advanced usage, target-only) --transient Do not store config changes, keep all the updates in memory -h, --help Show this help text @@ -78,11 +77,13 @@ while (( "$#" )); do case "${1}" in -h|--help) echo -n "${usage}"; exit;; - --aws) aws_provider="y"; shift;; - --azure) azure_provider="y"; shift;; - --gcp) gcp_provider="y"; shift;; + --aws) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} aws"; shift;; + --azure) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} azure"; shift;; + --gcp) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} gcp"; shift;; + --ht) AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS} ht"; shift;; + --loopback) loopback=$2; - + # if loopback is empty stop and notify validate_arg $1 $2 @@ -136,8 +137,6 @@ while (( "$#" )); do --cleanup) cleanup="true"; shift;; --transient) RUN_ARGS="$RUN_ARGS -transient"; shift;; --standby) RUN_ARGS="$RUN_ARGS -standby"; shift;; - --override_backends) RUN_ARGS="$RUN_ARGS -override_backends"; shift;; - --override-backends) RUN_ARGS="$RUN_ARGS -override_backends"; shift;; --https) export AIS_USE_HTTPS="true" export AIS_SKIP_VERIFY_CRT="true" @@ -159,8 +158,8 @@ if [[ ${cleanup} == "true" ]]; then fi if [[ ${deployment} == "local" || ${deployment} == "all" ]]; then - echo -e "${target_cnt}\n${proxy_cnt}\n${mountpath_cnt}\n${aws_provider}\n${gcp_provider}\n${azure_provider}\n${loopback}\n" |\ - make deploy "RUN_ARGS=${RUN_ARGS}" + echo -e "${target_cnt}\n${proxy_cnt}\n${mountpath_cnt}\nn\nn\nn\n${loopback}\n" |\ + AIS_BACKEND_PROVIDERS="${AIS_BACKEND_PROVIDERS}" make deploy "RUN_ARGS=${RUN_ARGS}" fi make -j8 authn aisloader cli 1>/dev/null # Build binaries in parallel @@ -169,7 +168,7 @@ if [[ ${deployment} == "remote" || ${deployment} == "all" ]]; then if [[ ${deployment} == "all" ]]; then echo -e "\n*** Remote cluster ***" fi - echo -e "1\n1\n3\n${aws_provider}\n${gcp_provider}\n${azure_provider}\n${loopback}\n" | DEPLOY_AS_NEXT_TIER="true" AIS_AUTHN_ENABLED=false make deploy + echo -e "1\n1\n3\n" | DEPLOY_AS_NEXT_TIER="true" AIS_AUTHN_ENABLED=false make deploy # Do not try attach remote cluster if the main cluster did not start. if [[ ${deployment} == "all" ]]; then