Merge remote-tracking branch 'up/master' into bump-grpc

labkode · Sep 3, 2024 · 83c599a · 83c599a
2 parents 220202c + 8d1b4ab
commit 83c599a
Show file tree

Hide file tree

Showing 54 changed files with 629 additions and 6,496 deletions.
diff --git a/changelog/unreleased/fix-ocm-open-driver.md b/changelog/unreleased/fix-ocm-open-driver.md
@@ -0,0 +1,8 @@
+Bugfix: ocm: fixed domain not having a protocol scheme
+
+This PR fixes a bug in the OCM open driver that causes it to be unable to probe
+OCM services at the remote server due to the domain having an unsupported
+protocol scheme. in this case domain doesn't have a scheme and the changes in
+this PR add a scheme to the domain before doing the probe.
+
+https://github.com/cs3org/reva/pull/4790
diff --git a/changelog/unreleased/locks-uploads.md b/changelog/unreleased/locks-uploads.md
@@ -0,0 +1,6 @@
+Enhancement: Pass lock holder metadata on uploads
+
+We now pass relevant metadata (lock id and lock holder) downstream
+on uploads, and handle the case of conflicts due to lock mismatch.
+
+https://github.com/cs3org/reva/pull/4514
diff --git a/changelog/unreleased/ocm-access.md b/changelog/unreleased/ocm-access.md
@@ -0,0 +1,8 @@
+Enhancement: ocm: support bearer token access
+
+This PR adds support for accessing remote OCM 1.1 shares via bearer token,
+as opposed to having the shared secret in the URL only.
+In addition, the OCM client package is now part of the OCMD server package,
+and the Discover methods have been all consolidated in one place.
+
+https://github.com/cs3org/reva/pull/4670
diff --git a/docker/Dockerfile.revad-ceph b/docker/Dockerfile.revad-ceph
@@ -18,13 +18,7 @@
 
 FROM quay.io/ceph/ceph:v18
 
-RUN mkdir -p /etc/selinux/config
-
-# this is a workaround as the Ceph docker image is still based on CentOS 8 Stream, which is EOL
-RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-
-RUN dnf update --exclude=ceph-iscsi,chrony -y && dnf install -y \
+RUN dnf update --exclude=ceph-iscsi -y  && dnf install -y \
   git \
   gcc \
   make \

diff --git a/examples/cernbox/cernbox.toml b/examples/cernbox/cernbox.toml
@@ -199,7 +199,8 @@ webapp_template = "{{ vars.external_reva_endpoint }}/external/sciencemesh/{{.Tok
 file = "{{ vars.ocmshares_json_file }}"
 
 [grpc.services.ocmproviderauthorizer]
-driver = "json"
+driver = "open"    # pure OCM, all remote shares are accepted
+#driver = "json"   # to enable sciencemesh
 
 [grpc.services.ocmproviderauthorizer.drivers.json]
 # this is used by the docker-based test deployment, not in production
@@ -279,8 +280,10 @@ sender_mail = "sciencemesh@{{ vars.provider_domain }}"
 smtp_server = "smtp.{{ vars.provider_domain }}"
 smtp_port = 25
 
-[http.services.ocmprovider]
+[http.services.wellknown]
 address = ":443"
+
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "Reva for CERNBox"
 endpoint = "{{ vars.external_reva_endpoint }}"
@@ -392,8 +395,6 @@ insecure = true
 [http.services.prometheus]
 address = ":443"
 
-[http.services.sysinfo]
-
 #[http.services.ui]
 #address = ":443"
 

diff --git a/examples/ocm/server-1.toml b/examples/ocm/server-1.toml
@@ -166,7 +166,7 @@ driver = "ocmreceived"
 address = "0.0.0.0:8080"
 expose_recipient_display_name = true
 
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "reva@cern"
 endpoint = "http://localhost:{{ http.services.ocm.address.port }}"

diff --git a/examples/ocm/server-2.toml b/examples/ocm/server-2.toml
@@ -166,7 +166,7 @@ driver = "ocmreceived"
 address = "0.0.0.0:80"
 expose_recipient_display_name = true
 
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "reva@cesnet"
 endpoint = "http://localhost:{{ http.services.ocm.address.port }}"

diff --git a/examples/sciencemesh/sciencemesh.toml b/examples/sciencemesh/sciencemesh.toml
@@ -253,8 +253,10 @@ sender_mail = "sciencemesh@{{ vars.provider_domain }}"
 smtp_server = "smtp.{{ vars.provider_domain }}"
 smtp_port = 25
 
-[http.services.ocmprovider]
+[http.services.wellknown]
 address = ":443"
+
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "Reva for ownCloud/Nextcloud"
 endpoint = "{{ vars.external_reva_endpoint }}"
@@ -284,7 +286,5 @@ metrics_data_driver_type = "json"
 metrics_data_location = "/etc/revad/metrics.json"
 metrics_record_interval = 5000
 
-[http.services.sysinfo]
-
 [http.middlewares.cors]
 [http.middlewares.log]
diff --git a/examples/standalone/standalone.toml b/examples/standalone/standalone.toml
@@ -17,6 +17,6 @@
 [http.services.dataprovider]
 [http.services.prometheus]
 [http.services.ocm]
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 [http.services.ocdav]
 [http.services.ocs]
diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml
@@ -57,7 +57,7 @@ app_url = "https://your-collabora-server.org:9980"
 [http.services.datagateway]
 [http.services.prometheus]
 [http.services.ocm]
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 [http.services.ocdav]
 [http.services.ocs]
 

diff --git a/examples/two-server-setup/gateway-1.toml b/examples/two-server-setup/gateway-1.toml
@@ -84,7 +84,7 @@ address = "0.0.0.0:19001"
 [http.services.datagateway]
 [http.services.prometheus]
 [http.services.ocm]
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 provider = "Reva-Server-1"
 endpoint = "http://localhost:19001"
 enable_webapp = true

diff --git a/examples/two-server-setup/gateway-2.toml b/examples/two-server-setup/gateway-2.toml
@@ -84,7 +84,7 @@ address = "0.0.0.0:29001"
 [http.services.datagateway]
 [http.services.prometheus]
 [http.services.ocm]
-[http.services.ocmprovider]
+[http.services.wellknown.ocmprovider]
 provider = "Reva-Server-2"
 endpoint = "http://localhost:29001"
 enable_webapp = true

diff --git a/go.mod b/go.mod
@@ -72,6 +72,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
+	github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

diff --git a/go.sum b/go.sum
@@ -848,6 +848,8 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
 github.com/ceph/go-ceph v0.26.0 h1:LZoATo25ZH5aeL5t85BwIbrNLKCDfcDM+e0qV0cmwHY=
 github.com/ceph/go-ceph v0.26.0/go.mod h1:ISxb295GszZwtLPkeWi+L2uLYBVsqbsh0M104jZMOX4=
+github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38 h1:+81ss4Vut1khzEhl7ximWF/V+EadspY47V4JrQkwlI4=
+github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38/go.mod h1:ZiIzbg4sDO2MwYlspcnauUR2dfwZHUzxker+HP9k+20=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go
@@ -18,6 +18,8 @@
 
 package ocmcore
 
+// This package implements the core OCM API for receiving external shares from remote EFSS systems.
+
 import (
 	"context"
 	"fmt"
@@ -102,7 +104,7 @@ func (s *service) UnprotectedEndpoints() []string {
 	return []string{"/cs3.ocm.core.v1beta1.OcmCoreAPI/CreateOCMCoreShare"}
 }
 
-// CreateOCMCoreShare is called when an OCM request comes into this reva instance from.
+// CreateOCMCoreShare is called when a remote OCM request comes into this reva instance.
 func (s *service) CreateOCMCoreShare(ctx context.Context, req *ocmcore.CreateOCMCoreShareRequest) (*ocmcore.CreateOCMCoreShareResponse, error) {
 	if req.ShareType != ocm.ShareType_SHARE_TYPE_USER {
 		return nil, errtypes.NotSupported("share type not supported")

diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go
@@ -26,9 +26,9 @@ import (
 	invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1"
 	ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1"
 	rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	"github.com/cs3org/reva/internal/http/services/opencloudmesh/ocmd"
 	"github.com/cs3org/reva/pkg/appctx"
 	"github.com/cs3org/reva/pkg/errtypes"
-	"github.com/cs3org/reva/pkg/ocm/client"
 	"github.com/cs3org/reva/pkg/ocm/invite"
 	"github.com/cs3org/reva/pkg/ocm/invite/repository/registry"
 	"github.com/cs3org/reva/pkg/plugin"
@@ -66,7 +66,7 @@ type config struct {
 type service struct {
 	conf      *config
 	repo      invite.Repository
-	ocmClient *client.OCMClient
+	ocmClient *ocmd.OCMClient
 }
 
 func (c *config) ApplyDefaults() {
@@ -110,12 +110,9 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) {
 	}
 
 	service := &service{
-		conf: &c,
-		repo: repo,
-		ocmClient: client.New(&client.Config{
-			Timeout:  time.Duration(c.OCMClientTimeout) * time.Second,
-			Insecure: c.OCMClientInsecure,
-		}),
+		conf:      &c,
+		repo:      repo,
+		ocmClient: ocmd.NewClient(time.Duration(c.OCMClientTimeout)*time.Second, c.OCMClientInsecure),
 	}
 	return service, nil
 }
@@ -166,7 +163,7 @@ func (s *service) ForwardInvite(ctx context.Context, req *invitepb.ForwardInvite
 		return nil, err
 	}
 
-	remoteUser, err := s.ocmClient.InviteAccepted(ctx, ocmEndpoint, &client.InviteAcceptedRequest{
+	remoteUser, err := s.ocmClient.InviteAccepted(ctx, ocmEndpoint, &ocmd.InviteAcceptedRequest{
 		Token:             req.InviteToken.GetToken(),
 		RecipientProvider: s.conf.ProviderDomain,
 		UserID:            user.GetId().GetOpaqueId(),
@@ -175,19 +172,19 @@ func (s *service) ForwardInvite(ctx context.Context, req *invitepb.ForwardInvite
 	})
 	if err != nil {
 		switch {
-		case errors.Is(err, client.ErrTokenInvalid):
+		case errors.Is(err, ocmd.ErrTokenInvalid):
 			return &invitepb.ForwardInviteResponse{
 				Status: status.NewInvalid(ctx, "token not valid"),
 			}, nil
-		case errors.Is(err, client.ErrTokenNotFound):
+		case errors.Is(err, ocmd.ErrTokenNotFound):
 			return &invitepb.ForwardInviteResponse{
 				Status: status.NewNotFound(ctx, "token not found"),
 			}, nil
-		case errors.Is(err, client.ErrUserAlreadyAccepted):
+		case errors.Is(err, ocmd.ErrUserAlreadyAccepted):
 			return &invitepb.ForwardInviteResponse{
 				Status: status.NewAlreadyExists(ctx, err, err.Error()),
 			}, nil
-		case errors.Is(err, client.ErrServiceNotTrusted):
+		case errors.Is(err, ocmd.ErrServiceNotTrusted):
 			return &invitepb.ForwardInviteResponse{
 				Status: status.NewPermissionDenied(ctx, err, err.Error()),
 			}, nil

diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go
@@ -18,6 +18,9 @@
 
 package ocmshareprovider
 
+// This package implements the OCM client API: it allows shares created on this Reva instance
+// to be sent to a remote EFSS system via OCM.
+
 import (
 	"context"
 	"fmt"
@@ -38,7 +41,6 @@ import (
 
 	"github.com/cs3org/reva/pkg/appctx"
 	"github.com/cs3org/reva/pkg/errtypes"
-	"github.com/cs3org/reva/pkg/ocm/client"
 	"github.com/cs3org/reva/pkg/ocm/share"
 	"github.com/cs3org/reva/pkg/ocm/share/repository/registry"
 	"github.com/cs3org/reva/pkg/plugin"
@@ -70,13 +72,13 @@ type config struct {
 	GatewaySVC     string                            `mapstructure:"gatewaysvc"                                    validate:"required"`
 	ProviderDomain string                            `docs:"The same domain registered in the provider authorizer" mapstructure:"provider_domain" validate:"required"`
 	WebDAVEndpoint string                            `mapstructure:"webdav_endpoint"                               validate:"required"`
-	WebappTemplate string                            `mapstructure:"webapp_template"`
+	WebappTemplate string                            `mapstructure:"webapp_template"                               validate:"required"`
 }
 
 type service struct {
 	conf       *config
 	repo       share.Repository
-	client     *client.OCMClient
+	client     *ocmd.OCMClient
 	gateway    gateway.GatewayAPIClient
 	webappTmpl *template.Template
 	walker     walker.Walker
@@ -89,9 +91,6 @@ func (c *config) ApplyDefaults() {
 	if c.ClientTimeout == 0 {
 		c.ClientTimeout = 10
 	}
-	if c.WebappTemplate == "" {
-		c.WebappTemplate = "https://cernbox.cern.ch/external/sciencemesh/{{.Token}}{relative-path-to-shared-resource}"
-	}
 
 	c.GatewaySVC = sharedconf.GetGatewaySVC(c.GatewaySVC)
 }
@@ -119,11 +118,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) {
 		return nil, err
 	}
 
-	client := client.New(&client.Config{
-		Timeout:  time.Duration(c.ClientTimeout) * time.Second,
-		Insecure: c.ClientInsecure,
-	})
-
 	gateway, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySVC))
 	if err != nil {
 		return nil, err
@@ -135,10 +129,11 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) {
 	}
 	walker := walker.NewWalker(gateway)
 
+	ocmcl := ocmd.NewClient(time.Duration(c.ClientTimeout)*time.Second, c.ClientInsecure)
 	service := &service{
 		conf:       &c,
 		repo:       repo,
-		client:     client,
+		client:     ocmcl,
 		gateway:    gateway,
 		webappTmpl: tpl,
 		walker:     walker,
@@ -178,13 +173,14 @@ func getResourceType(info *providerpb.ResourceInfo) string {
 	return "unknown"
 }
 
-func (s *service) webdavURL(ctx context.Context, share *ocm.Share) string {
-	// the url is in the form of https://cernbox.cern.ch/remote.php/dav/ocm/token
-	p, _ := url.JoinPath(s.conf.WebDAVEndpoint, "/remote.php/dav/ocm", share.Token)
+func (s *service) webdavURL(share *ocm.Share) string {
+	// the url is expected to be in the form https://ourserver/remote.php/dav/ocm/{ShareId}, see c.WebdavRoot in ocmprovider.go
+	// TODO(lopresti) take the root from http.services.wellknown.ocmprovider's config
+	p, _ := url.JoinPath(s.conf.WebDAVEndpoint, "/remote.php/dav/ocm", share.Id.OpaqueId)
 	return p
 }
 
-func (s *service) getWebdavProtocol(ctx context.Context, share *ocm.Share, m *ocm.AccessMethod_WebdavOptions) *ocmd.WebDAV {
+func (s *service) getWebdavProtocol(share *ocm.Share, m *ocm.AccessMethod_WebdavOptions) *ocmd.WebDAV {
 	var perms []string
 	if m.WebdavOptions.Permissions.InitiateFileDownload {
 		perms = append(perms, "read")
@@ -195,7 +191,7 @@ func (s *service) getWebdavProtocol(ctx context.Context, share *ocm.Share, m *oc
 
 	return &ocmd.WebDAV{
 		Permissions:  perms,
-		URL:          s.webdavURL(ctx, share),
+		URL:          s.webdavURL(share),
 		SharedSecret: share.Token,
 	}
 }
@@ -233,7 +229,7 @@ func (s *service) getDataTransferProtocol(ctx context.Context, share *ocm.Share)
 		panic(err)
 	}
 	return &ocmd.Datatx{
-		SourceURI: s.webdavURL(ctx, share),
+		SourceURI: s.webdavURL(share),
 		Size:      size,
 	}
 }
@@ -248,7 +244,7 @@ func (s *service) getProtocols(ctx context.Context, share *ocm.Share) ocmd.Proto
 	for _, m := range share.AccessMethods {
 		switch t := m.Term.(type) {
 		case *ocm.AccessMethod_WebdavOptions:
-			p = append(p, s.getWebdavProtocol(ctx, share, t))
+			p = append(p, s.getWebdavProtocol(share, t))
 		case *ocm.AccessMethod_WebappOptions:
 			p = append(p, s.getWebappProtocol(share))
 		case *ocm.AccessMethod_TransferOptions:
@@ -323,7 +319,7 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq
 		}, nil
 	}
 
-	newShareReq := &client.NewShareRequest{
+	newShareReq := &ocmd.NewShareRequest{
 		ShareWith:  formatOCMUser(req.Grantee.GetUserId()),
 		Name:       ocmshare.Name,
 		ProviderID: ocmshare.Id.OpaqueId,
@@ -348,11 +344,11 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq
 	newShareRes, err := s.client.NewShare(ctx, ocmEndpoint, newShareReq)
 	if err != nil {
 		switch {
-		case errors.Is(err, client.ErrInvalidParameters):
+		case errors.Is(err, ocmd.ErrInvalidParameters):
 			return &ocm.CreateOCMShareResponse{
 				Status: status.NewInvalidArg(ctx, err.Error()),
 			}, nil
-		case errors.Is(err, client.ErrServiceNotTrusted):
+		case errors.Is(err, ocmd.ErrServiceNotTrusted):
 			return &ocm.CreateOCMShareResponse{
 				Status: status.NewInvalidArg(ctx, err.Error()),
 			}, nil

diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go
@@ -284,10 +284,10 @@ func (s *service) SetLock(ctx context.Context, req *provider.SetLockRequest) (*p
 		var st *rpc.Status
 		switch err.(type) {
 		case errtypes.IsNotFound:
-			st = status.NewNotFound(ctx, "path not found when setting lock")
+			st = status.NewNotFound(ctx, "resource not found when setting lock")
 		case errtypes.PermissionDenied:
 			st = status.NewPermissionDenied(ctx, err, "permission denied")
-		case errtypes.BadRequest:
+		case errtypes.Conflict:
 			st = status.NewFailedPrecondition(ctx, err, "reference already locked")
 		default:
 			st = status.NewInternal(ctx, err, "error setting lock: "+req.Ref.String())