From 58de124842717c7fd7cdd7caeb24ead28d9b7114 Mon Sep 17 00:00:00 2001 From: Andre Duffeck Date: Mon, 21 Mar 2022 10:19:01 +0100 Subject: [PATCH] Implement a publicshare manager using the metadata storage backend (#2644) * Move Indexer and Storage interfaces to their libraries for reusability * Fix naming of nested field indexes * Add initial version of the cs3-based publicshare manager * Cleanup unused parameter * Return empty result instead of error when listed directory doesn't exist * Implement ListPublicShares * Add support for adding signature to the returned public shares * Implement RevokePublicShare * Extract authenticate into the main package for reusability * Implement GetPublicShareByToken * More API cleanup * Add config and make NewDefault work * Implement UpdatePublicShare * Fix rebase artifact * Improve tests * Make sure to always initialize the manager * Fix tests * Return NotFoundErr if there is no match * Linter fixes * Add changelog * Make sure to address the proper index * Make sure to initialize the manager atomically * Fix setting the display name from the metadata * Improve style according to review --- .../unreleased/cs3-publicshare-manager.md | 6 + .../publicshareprovider.go | 5 +- pkg/cbox/publicshare/sql/sql.go | 4 +- pkg/publicshare/manager/cs3/cs3.go | 411 ++++++++++++++ pkg/publicshare/manager/cs3/cs3_suite_test.go | 31 ++ pkg/publicshare/manager/cs3/cs3_test.go | 504 ++++++++++++++++++ pkg/publicshare/manager/json/json.go | 30 +- pkg/publicshare/manager/loader/loader.go | 1 + pkg/publicshare/manager/memory/memory.go | 4 +- pkg/publicshare/publicshare.go | 28 +- pkg/share/manager/cs3/cs3.go | 22 +- pkg/share/manager/cs3/cs3_test.go | 11 +- pkg/storage/utils/indexer/index/non_unique.go | 4 + pkg/storage/utils/indexer/indexer.go | 34 +- pkg/storage/utils/indexer/indexer_test.go | 4 +- pkg/storage/utils/indexer/map.go | 9 +- .../utils/indexer}/mocks/Indexer.go | 0 pkg/storage/utils/metadata/disk.go | 4 + .../utils/metadata}/mocks/Storage.go | 0 pkg/storage/utils/metadata/storage.go | 2 + 20 files changed, 1037 insertions(+), 77 deletions(-) create mode 100644 changelog/unreleased/cs3-publicshare-manager.md create mode 100644 pkg/publicshare/manager/cs3/cs3.go create mode 100644 pkg/publicshare/manager/cs3/cs3_suite_test.go create mode 100644 pkg/publicshare/manager/cs3/cs3_test.go rename pkg/{share/manager/cs3 => storage/utils/indexer}/mocks/Indexer.go (100%) rename pkg/{share/manager/cs3 => storage/utils/metadata}/mocks/Storage.go (100%) diff --git a/changelog/unreleased/cs3-publicshare-manager.md b/changelog/unreleased/cs3-publicshare-manager.md new file mode 100644 index 0000000000..e4666edb9c --- /dev/null +++ b/changelog/unreleased/cs3-publicshare-manager.md @@ -0,0 +1,6 @@ +Enhancement: add new public share manager + +We added a new public share manager which uses the new metadata storage backend for +persisting the public share information. + +https://github.com/cs3org/reva/pull/2644 \ No newline at end of file diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index ffa59209c8..6aca7026be 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -23,7 +23,6 @@ import ( "regexp" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" @@ -226,7 +225,7 @@ func (s *service) ListPublicShares(ctx context.Context, req *link.ListPublicShar log.Info().Str("publicshareprovider", "list").Msg("list public share") user, _ := ctxpkg.ContextGetUser(ctx) - shares, err := s.sm.ListPublicShares(ctx, user, req.Filters, &provider.ResourceInfo{}, req.GetSign()) + shares, err := s.sm.ListPublicShares(ctx, user, req.Filters, req.GetSign()) if err != nil { log.Err(err).Msg("error listing shares") return &link.ListPublicSharesResponse{ @@ -250,7 +249,7 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS log.Error().Msg("error getting user from context") } - updateR, err := s.sm.UpdatePublicShare(ctx, u, req, nil) + updateR, err := s.sm.UpdatePublicShare(ctx, u, req) if err != nil { log.Err(err).Msgf("error updating public shares: %v", err) } diff --git a/pkg/cbox/publicshare/sql/sql.go b/pkg/cbox/publicshare/sql/sql.go index 415efcab15..ca05075dae 100644 --- a/pkg/cbox/publicshare/sql/sql.go +++ b/pkg/cbox/publicshare/sql/sql.go @@ -194,7 +194,7 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr }, nil } -func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest, g *link.Grant) (*link.PublicShare, error) { +func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) { query := "update oc_share set " paramsMap := map[string]interface{}{} params := []interface{}{} @@ -308,7 +308,7 @@ func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.Pu return s, nil } -func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) { +func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) { uid := conversions.FormatUserID(u.Id) query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner=? or uid_initiator=?) AND (share_type=?)" var filterQuery string diff --git a/pkg/publicshare/manager/cs3/cs3.go b/pkg/publicshare/manager/cs3/cs3.go new file mode 100644 index 0000000000..3c0331f284 --- /dev/null +++ b/pkg/publicshare/manager/cs3/cs3.go @@ -0,0 +1,411 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cs3 + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "path" + "sync" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "golang.org/x/crypto/bcrypt" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/publicshare" + "github.com/cs3org/reva/v2/pkg/publicshare/manager/registry" + "github.com/cs3org/reva/v2/pkg/storage/utils/indexer" + "github.com/cs3org/reva/v2/pkg/storage/utils/indexer/option" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" + "github.com/cs3org/reva/v2/pkg/utils" +) + +func init() { + registry.Register("cs3", NewDefault) +} + +// Manager implements a publicshare manager using a cs3 storage backend +type Manager struct { + sync.RWMutex + + storage metadata.Storage + indexer indexer.Indexer + passwordHashCost int + + initialized bool +} + +// PublicShareWithPassword represents a public share including its hashes password +type PublicShareWithPassword struct { + PublicShare *link.PublicShare `json:"public_share"` + HashedPassword string `json:"password"` +} + +type config struct { + GatewayAddr string `mapstructure:"gateway_addr"` + ProviderAddr string `mapstructure:"provider_addr"` + ServiceUserID string `mapstructure:"service_user_id"` + ServiceUserIdp string `mapstructure:"service_user_idp"` + MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` +} + +// NewDefault returns a new manager instance with default dependencies +func NewDefault(m map[string]interface{}) (publicshare.Manager, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error creating a new manager") + return nil, err + } + + s, err := metadata.NewCS3Storage(c.GatewayAddr, c.ProviderAddr, c.ServiceUserID, c.ServiceUserIdp, c.MachineAuthAPIKey) + if err != nil { + return nil, err + } + indexer := indexer.CreateIndexer(s) + + return New(s, indexer, bcrypt.DefaultCost) +} + +// New returns a new manager instance +func New(storage metadata.Storage, indexer indexer.Indexer, passwordHashCost int) (publicshare.Manager, error) { + return &Manager{ + storage: storage, + indexer: indexer, + passwordHashCost: passwordHashCost, + initialized: false, + }, nil +} + +func (m *Manager) initialize() error { + if m.initialized { + return nil + } + + m.Lock() + defer m.Unlock() + + if m.initialized { // check if initialization happened while grabbing the lock + return nil + } + + err := m.storage.Init(context.Background(), "public-share-manager-metadata") + if err != nil { + return err + } + if err := m.storage.MakeDirIfNotExist(context.Background(), "publicshares"); err != nil { + return err + } + err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByField("Id.OpaqueId"), "Token", "publicshares", "unique", nil, true) + if err != nil { + return err + } + err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByFunc{ + Name: "Owner", + Func: indexOwnerFunc, + }, "Token", "publicshares", "non_unique", nil, true) + if err != nil { + return err + } + m.initialized = true + return nil +} + +// CreatePublicShare creates a new public share +func (m *Manager) CreatePublicShare(ctx context.Context, u *user.User, ri *provider.ResourceInfo, g *link.Grant) (*link.PublicShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + id := &link.PublicShareId{ + OpaqueId: utils.RandString(15), + } + + tkn := utils.RandString(15) + now := time.Now().UnixNano() + + displayName := tkn + if ri.ArbitraryMetadata != nil { + metadataName, ok := ri.ArbitraryMetadata.Metadata["name"] + if ok { + displayName = metadataName + } + } + + var passwordProtected bool + password := g.Password + if password != "" { + h, err := bcrypt.GenerateFromPassword([]byte(password), m.passwordHashCost) + if err != nil { + return nil, errors.Wrap(err, "could not hash share password") + } + password = string(h) + passwordProtected = true + } + + createdAt := &typespb.Timestamp{ + Seconds: uint64(now / 1000000000), + Nanos: uint32(now % 1000000000), + } + + s := &PublicShareWithPassword{ + PublicShare: &link.PublicShare{ + Id: id, + Owner: ri.GetOwner(), + Creator: u.Id, + ResourceId: ri.Id, + Token: tkn, + Permissions: g.Permissions, + Ctime: createdAt, + Mtime: createdAt, + PasswordProtected: passwordProtected, + Expiration: g.Expiration, + DisplayName: displayName, + }, + HashedPassword: password, + } + + err := m.persist(ctx, s) + if err != nil { + return nil, err + } + + return s.PublicShare, nil +} + +// UpdatePublicShare updates an existing public share +func (m *Manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + ps, err := m.getWithPassword(ctx, req.Ref) + if err != nil { + return nil, err + } + + switch req.Update.Type { + case link.UpdatePublicShareRequest_Update_TYPE_DISPLAYNAME: + ps.PublicShare.DisplayName = req.Update.DisplayName + case link.UpdatePublicShareRequest_Update_TYPE_PERMISSIONS: + ps.PublicShare.Permissions = req.Update.Grant.Permissions + case link.UpdatePublicShareRequest_Update_TYPE_EXPIRATION: + ps.PublicShare.Expiration = req.Update.Grant.Expiration + case link.UpdatePublicShareRequest_Update_TYPE_PASSWORD: + h, err := bcrypt.GenerateFromPassword([]byte(req.Update.Grant.Password), m.passwordHashCost) + if err != nil { + return nil, errors.Wrap(err, "could not hash share password") + } + ps.HashedPassword = string(h) + ps.PublicShare.PasswordProtected = true + default: + return nil, errtypes.BadRequest("no valid update type given") + } + + err = m.persist(ctx, ps) + if err != nil { + return nil, err + } + + return ps.PublicShare, nil +} + +// GetPublicShare returns an existing public share +func (m *Manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference, sign bool) (*link.PublicShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + ps, err := m.getWithPassword(ctx, ref) + if err != nil { + return nil, err + } + + if ps.PublicShare.PasswordProtected && sign { + err = publicshare.AddSignature(ps.PublicShare, ps.HashedPassword) + if err != nil { + return nil, err + } + } + + return ps.PublicShare, nil +} + +func (m *Manager) getWithPassword(ctx context.Context, ref *link.PublicShareReference) (*PublicShareWithPassword, error) { + switch { + case ref.GetToken() != "": + return m.getByToken(ctx, ref.GetToken()) + case ref.GetId().GetOpaqueId() != "": + return m.getByID(ctx, ref.GetId().GetOpaqueId()) + default: + return nil, errtypes.BadRequest("neither id nor token given") + } +} + +func (m *Manager) getByID(ctx context.Context, id string) (*PublicShareWithPassword, error) { + tokens, err := m.indexer.FindBy(&link.PublicShare{}, "Id.OpaqueId", id) + if err != nil { + return nil, err + } + if len(tokens) == 0 { + return nil, errtypes.NotFound("publicshare with the given id not found") + } + return m.getByToken(ctx, tokens[0]) +} + +func (m *Manager) getByToken(ctx context.Context, token string) (*PublicShareWithPassword, error) { + fn := path.Join("publicshares", token) + data, err := m.storage.SimpleDownload(ctx, fn) + if err != nil { + return nil, err + } + + ps := &PublicShareWithPassword{} + err = json.Unmarshal(data, ps) + if err != nil { + return nil, err + } + return ps, nil +} + +// ListPublicShares lists existing public shares matching the given filters +func (m *Manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + tokens, err := m.indexer.FindBy(&link.PublicShare{}, "Owner", userIDToIndex(u.Id)) + if err != nil { + return nil, err + } + + result := []*link.PublicShare{} + for _, token := range tokens { + ps, err := m.getByToken(ctx, token) + if err != nil { + return nil, err + } + + if publicshare.MatchesFilters(ps.PublicShare, filters) && !publicshare.IsExpired(ps.PublicShare) { + result = append(result, ps.PublicShare) + } + + if ps.PublicShare.PasswordProtected && sign { + err = publicshare.AddSignature(ps.PublicShare, ps.HashedPassword) + if err != nil { + return nil, err + } + } + } + + return result, nil +} + +// RevokePublicShare revokes an existing public share +func (m *Manager) RevokePublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference) error { + if err := m.initialize(); err != nil { + return err + } + + ps, err := m.GetPublicShare(ctx, u, ref, false) + if err != nil { + return err + } + + err = m.storage.Delete(ctx, path.Join("publicshares", ps.Token)) + if err != nil { + if _, ok := err.(errtypes.NotFound); !ok { + return err + } + } + + return m.indexer.Delete(ps) +} + +// GetPublicShareByToken gets an existing public share in an unauthenticated context using either a password or a signature +func (m *Manager) GetPublicShareByToken(ctx context.Context, token string, auth *link.PublicShareAuthentication, sign bool) (*link.PublicShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + ps, err := m.getByToken(ctx, token) + if err != nil { + return nil, err + } + + if publicshare.IsExpired(ps.PublicShare) { + return nil, errtypes.NotFound("public share has expired") + } + + if ps.PublicShare.PasswordProtected { + if !publicshare.Authenticate(ps.PublicShare, ps.HashedPassword, auth) { + return nil, errtypes.InvalidCredentials("access denied") + } + } + + return ps.PublicShare, nil +} + +func indexOwnerFunc(v interface{}) (string, error) { + ps, ok := v.(*link.PublicShare) + if !ok { + return "", fmt.Errorf("given entity is not a public share") + } + return userIDToIndex(ps.Owner), nil +} + +func userIDToIndex(id *userpb.UserId) string { + return url.QueryEscape(id.Idp + ":" + id.OpaqueId) +} + +func (m *Manager) persist(ctx context.Context, ps *PublicShareWithPassword) error { + data, err := json.Marshal(ps) + if err != nil { + return err + } + + fn := path.Join("publicshares", ps.PublicShare.Token) + err = m.storage.SimpleUpload(ctx, fn, data) + if err != nil { + return err + } + + _, err = m.indexer.Add(ps.PublicShare) + if err != nil { + if _, ok := err.(errtypes.IsAlreadyExists); !ok { + return err + } + err = m.indexer.Delete(ps.PublicShare) + if err != nil { + return err + } + _, err = m.indexer.Add(ps.PublicShare) + return err + } + + return nil +} diff --git a/pkg/publicshare/manager/cs3/cs3_suite_test.go b/pkg/publicshare/manager/cs3/cs3_suite_test.go new file mode 100644 index 0000000000..8601fcf454 --- /dev/null +++ b/pkg/publicshare/manager/cs3/cs3_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cs3_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCs3(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cs3 Suite") +} diff --git a/pkg/publicshare/manager/cs3/cs3_test.go b/pkg/publicshare/manager/cs3/cs3_test.go new file mode 100644 index 0000000000..244b5578bb --- /dev/null +++ b/pkg/publicshare/manager/cs3/cs3_test.go @@ -0,0 +1,504 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cs3_test + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "path" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "golang.org/x/crypto/bcrypt" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/publicshare" + "github.com/cs3org/reva/v2/pkg/publicshare/manager/cs3" + indexerpkg "github.com/cs3org/reva/v2/pkg/storage/utils/indexer" + indexermocks "github.com/cs3org/reva/v2/pkg/storage/utils/indexer/mocks" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" + storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" + "github.com/cs3org/reva/v2/pkg/utils" +) + +var _ = Describe("Cs3", func() { + var ( + m publicshare.Manager + user *userpb.User + ctx context.Context + + indexer indexerpkg.Indexer + storage *storagemocks.Storage + + ri *provider.ResourceInfo + grant *link.Grant + share *link.PublicShare + + tmpdir string + ) + + BeforeEach(func() { + var err error + tmpdir, err = ioutil.TempDir("", "cs3-publicshare-test") + Expect(err).ToNot(HaveOccurred()) + + ds, err := metadata.NewDiskStorage(tmpdir) + Expect(err).ToNot(HaveOccurred()) + indexer = indexerpkg.CreateIndexer(ds) + + storage = &storagemocks.Storage{} + storage.On("Init", mock.Anything, mock.Anything).Return(nil) + storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) + storage.On("SimpleUpload", mock.Anything, mock.MatchedBy(func(in string) bool { + return strings.HasPrefix(in, "publicshares/") + }), mock.Anything).Return(nil) + user = &userpb.User{ + Id: &userpb.UserId{ + Idp: "localhost:1111", + OpaqueId: "1", + }, + } + ctx = ctxpkg.ContextSetUser(context.Background(), user) + + share = &link.PublicShare{ + Id: &link.PublicShareId{OpaqueId: "1"}, + Token: "abcd", + } + + ri = &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Path: "/share1", + Id: &provider.ResourceId{OpaqueId: "sharedId"}, + Owner: user.Id, + PermissionSet: &provider.ResourcePermissions{ + Stat: true, + }, + Size: 10, + } + grant = &link.Grant{ + Permissions: &link.PublicSharePermissions{ + Permissions: &provider.ResourcePermissions{AddGrant: true}, + }, + } + }) + + JustBeforeEach(func() { + var err error + m, err = cs3.New(storage, indexer, bcrypt.DefaultCost) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + if tmpdir != "" { + os.RemoveAll(tmpdir) + } + }) + + Describe("New", func() { + It("returns a new instance", func() { + m, err := cs3.New(storage, indexer, bcrypt.DefaultCost) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(BeNil()) + }) + }) + + Describe("CreatePublicShare", func() { + It("creates a new share and adds it to the index", func() { + link, err := m.CreatePublicShare(ctx, user, ri, grant) + Expect(err).ToNot(HaveOccurred()) + Expect(link).ToNot(BeNil()) + Expect(link.Token).ToNot(Equal("")) + Expect(link.PasswordProtected).To(BeFalse()) + }) + + It("sets 'PasswordProtected' and stores the password hash if a password is set", func() { + grant.Password = "secret123" + + link, err := m.CreatePublicShare(ctx, user, ri, grant) + Expect(err).ToNot(HaveOccurred()) + Expect(link).ToNot(BeNil()) + Expect(link.Token).ToNot(Equal("")) + Expect(link.PasswordProtected).To(BeTrue()) + storage.AssertCalled(GinkgoT(), "SimpleUpload", mock.Anything, mock.Anything, mock.MatchedBy(func(in []byte) bool { + ps := cs3.PublicShareWithPassword{} + err = json.Unmarshal(in, &ps) + Expect(err).ToNot(HaveOccurred()) + return bcrypt.CompareHashAndPassword([]byte(ps.HashedPassword), []byte("secret123")) == nil + })) + }) + + It("picks up the displayname from the metadata", func() { + ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + "name": "metadata name", + }, + } + + link, err := m.CreatePublicShare(ctx, user, ri, grant) + Expect(err).ToNot(HaveOccurred()) + Expect(link).ToNot(BeNil()) + Expect(link.DisplayName).To(Equal("metadata name")) + }) + }) + + Context("with an existing share", func() { + var ( + existingShare *link.PublicShare + hashedPassword string + ) + + JustBeforeEach(func() { + grant.Password = "foo" + var err error + existingShare, err = m.CreatePublicShare(ctx, user, ri, grant) + Expect(err).ToNot(HaveOccurred()) + + h, err := bcrypt.GenerateFromPassword([]byte(grant.Password), bcrypt.DefaultCost) + Expect(err).ToNot(HaveOccurred()) + hashedPassword = string(h) + shareJSON, err := json.Marshal(cs3.PublicShareWithPassword{PublicShare: existingShare, HashedPassword: hashedPassword}) + Expect(err).ToNot(HaveOccurred()) + storage.On("SimpleDownload", mock.Anything, mock.MatchedBy(func(in string) bool { + return strings.HasPrefix(in, "publicshares/") + })).Return(shareJSON, nil) + }) + + Describe("ListPublicShares", func() { + It("lists existing shares", func() { + shares, err := m.ListPublicShares(ctx, user, []*link.ListPublicSharesRequest_Filter{}, false) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].Signature).To(BeNil()) + }) + + It("adds a signature", func() { + shares, err := m.ListPublicShares(ctx, user, []*link.ListPublicSharesRequest_Filter{}, true) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].Signature).ToNot(BeNil()) + }) + + It("filters by id", func() { + shares, err := m.ListPublicShares(ctx, user, []*link.ListPublicSharesRequest_Filter{ + publicshare.ResourceIDFilter(&provider.ResourceId{OpaqueId: "UnknownId"}), + }, false) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(0)) + }) + + It("filters by storage", func() { + shares, err := m.ListPublicShares(ctx, user, []*link.ListPublicSharesRequest_Filter{ + publicshare.StorageIDFilter("unknownstorage"), + }, false) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(0)) + }) + + Context("when the share has expired", func() { + BeforeEach(func() { + t := time.Date(2022, time.January, 1, 12, 0, 0, 0, time.UTC) + grant.Expiration = &typespb.Timestamp{ + Seconds: uint64(t.Unix()), + } + }) + + It("does not consider the share", func() { + shares, err := m.ListPublicShares(ctx, user, []*link.ListPublicSharesRequest_Filter{}, false) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(0)) + }) + }) + }) + + Describe("GetPublicShare", func() { + It("gets the public share by token", func() { + returnedShare, err := m.GetPublicShare(ctx, user, &link.PublicShareReference{ + Spec: &link.PublicShareReference_Token{ + Token: share.Token, + }, + }, false) + Expect(err).ToNot(HaveOccurred()) + Expect(returnedShare).ToNot(BeNil()) + Expect(returnedShare.Id.OpaqueId).To(Equal(existingShare.Id.OpaqueId)) + Expect(returnedShare.Token).To(Equal(existingShare.Token)) + }) + + It("gets the public share by id", func() { + returnedShare, err := m.GetPublicShare(ctx, user, &link.PublicShareReference{ + Spec: &link.PublicShareReference_Id{ + Id: &link.PublicShareId{ + OpaqueId: existingShare.Id.OpaqueId, + }, + }, + }, false) + Expect(err).ToNot(HaveOccurred()) + Expect(returnedShare).ToNot(BeNil()) + Expect(returnedShare.Signature).To(BeNil()) + }) + + It("adds a signature", func() { + returnedShare, err := m.GetPublicShare(ctx, user, &link.PublicShareReference{ + Spec: &link.PublicShareReference_Id{ + Id: &link.PublicShareId{ + OpaqueId: existingShare.Id.OpaqueId, + }, + }, + }, true) + Expect(err).ToNot(HaveOccurred()) + Expect(returnedShare).ToNot(BeNil()) + Expect(returnedShare.Signature).ToNot(BeNil()) + }) + }) + + Describe("RevokePublicShare", func() { + var ( + mockIndexer *indexermocks.Indexer + ) + BeforeEach(func() { + mockIndexer = &indexermocks.Indexer{} + mockIndexer.On("AddIndex", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockIndexer.On("Add", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + mockIndexer.On("Delete", mock.Anything, mock.Anything).Return(nil, nil) + mockIndexer.On("FindBy", mock.Anything, mock.Anything, mock.Anything).Return([]string{existingShare.Token}, nil) + + indexer = mockIndexer + }) + + It("deletes the share by token", func() { + storage.On("Delete", mock.Anything, mock.Anything).Return(nil) + ref := &link.PublicShareReference{ + Spec: &link.PublicShareReference_Token{ + Token: existingShare.Token, + }, + } + err := m.RevokePublicShare(ctx, user, ref) + Expect(err).ToNot(HaveOccurred()) + storage.AssertCalled(GinkgoT(), "Delete", mock.Anything, path.Join("publicshares", existingShare.Token)) + }) + + It("deletes the share by id", func() { + storage.On("Delete", mock.Anything, mock.Anything).Return(nil) + ref := &link.PublicShareReference{ + Spec: &link.PublicShareReference_Id{ + Id: existingShare.Id, + }, + } + err := m.RevokePublicShare(ctx, user, ref) + Expect(err).ToNot(HaveOccurred()) + storage.AssertCalled(GinkgoT(), "Delete", mock.Anything, path.Join("publicshares", existingShare.Token)) + }) + + It("still removes the share from the index when the share itself couldn't be found", func() { + storage.On("Delete", mock.Anything, mock.Anything).Return(errtypes.NotFound("")) + ref := &link.PublicShareReference{ + Spec: &link.PublicShareReference_Token{ + Token: existingShare.Token, + }, + } + err := m.RevokePublicShare(ctx, user, ref) + Expect(err).ToNot(HaveOccurred()) + + mockIndexer.AssertCalled(GinkgoT(), "Delete", mock.Anything, mock.Anything) + }) + + It("does not removes the share from the index when the share itself couldn't be found", func() { + storage.On("Delete", mock.Anything, mock.Anything).Return(errtypes.InternalError("")) + ref := &link.PublicShareReference{ + Spec: &link.PublicShareReference_Token{ + Token: existingShare.Token, + }, + } + err := m.RevokePublicShare(ctx, user, ref) + Expect(err).To(HaveOccurred()) + + mockIndexer.AssertNotCalled(GinkgoT(), "Delete", mock.Anything, mock.Anything) + }) + }) + + Describe("GetPublicShareByToken", func() { + It("doesn't get the share using a wrong password", func() { + auth := &link.PublicShareAuthentication{ + Spec: &link.PublicShareAuthentication_Password{ + Password: "wroooong", + }, + } + ps, err := m.GetPublicShareByToken(ctx, existingShare.Token, auth, false) + Expect(err).To(HaveOccurred()) + Expect(ps).To(BeNil()) + }) + + It("gets the share using a password", func() { + auth := &link.PublicShareAuthentication{ + Spec: &link.PublicShareAuthentication_Password{ + Password: grant.Password, + }, + } + ps, err := m.GetPublicShareByToken(ctx, existingShare.Token, auth, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + }) + + It("gets the share using a signature", func() { + err := publicshare.AddSignature(existingShare, hashedPassword) + Expect(err).ToNot(HaveOccurred()) + auth := &link.PublicShareAuthentication{ + Spec: &link.PublicShareAuthentication_Signature{ + Signature: existingShare.Signature, + }, + } + ps, err := m.GetPublicShareByToken(ctx, existingShare.Token, auth, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + + }) + + Context("when the share has expired", func() { + BeforeEach(func() { + t := time.Date(2022, time.January, 1, 12, 0, 0, 0, time.UTC) + grant.Expiration = &typespb.Timestamp{ + Seconds: uint64(t.Unix()), + } + }) + It("it doesn't consider expired shares", func() { + auth := &link.PublicShareAuthentication{ + Spec: &link.PublicShareAuthentication_Password{ + Password: grant.Password, + }, + } + ps, err := m.GetPublicShareByToken(ctx, existingShare.Token, auth, false) + Expect(err).To(HaveOccurred()) + Expect(ps).To(BeNil()) + }) + }) + }) + + Describe("UpdatePublicShare", func() { + var ( + ref *link.PublicShareReference + ) + + JustBeforeEach(func() { + ref = &link.PublicShareReference{ + Spec: &link.PublicShareReference_Token{ + Token: existingShare.Token, + }, + } + }) + + It("fails when an invalid reference is given", func() { + _, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: &link.PublicShareReference{Spec: &link.PublicShareReference_Id{Id: &link.PublicShareId{OpaqueId: "doesnotexist"}}}, + }) + Expect(err).To(HaveOccurred()) + }) + + It("fails when no valid update request is given", func() { + _, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: ref, + Update: &link.UpdatePublicShareRequest_Update{}, + }) + Expect(err).To(HaveOccurred()) + }) + + It("updates the display name", func() { + ps, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: ref, + Update: &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_DISPLAYNAME, + DisplayName: "new displayname", + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + Expect(ps.DisplayName).To(Equal("new displayname")) + storage.AssertCalled(GinkgoT(), "SimpleUpload", mock.Anything, path.Join("publicshares", ps.Token), mock.MatchedBy(func(data []byte) bool { + s := cs3.PublicShareWithPassword{} + err := json.Unmarshal(data, &s) + Expect(err).ToNot(HaveOccurred()) + return s.PublicShare.DisplayName == "new displayname" + })) + }) + + It("updates the password", func() { + ps, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: ref, + Update: &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_PASSWORD, + Grant: &link.Grant{Password: "NewPass"}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + storage.AssertCalled(GinkgoT(), "SimpleUpload", mock.Anything, path.Join("publicshares", ps.Token), mock.MatchedBy(func(data []byte) bool { + s := cs3.PublicShareWithPassword{} + err := json.Unmarshal(data, &s) + Expect(err).ToNot(HaveOccurred()) + return s.HashedPassword != "" + })) + }) + + It("updates the permissions", func() { + ps, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: ref, + Update: &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_PERMISSIONS, + Grant: &link.Grant{Permissions: &link.PublicSharePermissions{Permissions: &provider.ResourcePermissions{Delete: true}}}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + storage.AssertCalled(GinkgoT(), "SimpleUpload", mock.Anything, path.Join("publicshares", ps.Token), mock.MatchedBy(func(data []byte) bool { + s := cs3.PublicShareWithPassword{} + err := json.Unmarshal(data, &s) + Expect(err).ToNot(HaveOccurred()) + return s.PublicShare.Permissions.Permissions.Delete + })) + }) + + It("updates the expiration", func() { + ts := utils.TSNow() + ps, err := m.UpdatePublicShare(ctx, user, &link.UpdatePublicShareRequest{ + Ref: ref, + Update: &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_EXPIRATION, + Grant: &link.Grant{Expiration: utils.TSNow()}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(ps).ToNot(BeNil()) + storage.AssertCalled(GinkgoT(), "SimpleUpload", mock.Anything, path.Join("publicshares", ps.Token), mock.MatchedBy(func(data []byte) bool { + s := cs3.PublicShareWithPassword{} + err := json.Unmarshal(data, &s) + Expect(err).ToNot(HaveOccurred()) + return s.PublicShare.Expiration != nil && s.PublicShare.Expiration.Seconds == ts.Seconds + })) + }) + }) + }) +}) diff --git a/pkg/publicshare/manager/json/json.go b/pkg/publicshare/manager/json/json.go index 3bd325bb3e..aa7ad191a0 100644 --- a/pkg/publicshare/manager/json/json.go +++ b/pkg/publicshare/manager/json/json.go @@ -219,7 +219,7 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr } // UpdatePublicShare updates the public share -func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest, g *link.Grant) (*link.PublicShare, error) { +func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) { log := appctx.GetLogger(ctx) share, err := m.GetPublicShare(ctx, u, req.Ref, false) if err != nil { @@ -353,7 +353,7 @@ func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.Pu } // ListPublicShares retrieves all the shares on the manager that are valid. -func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) { +func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) { var shares []*link.PublicShare m.mutex.Lock() @@ -521,7 +521,7 @@ func (m *manager) GetPublicShareByToken(ctx context.Context, token string, auth } if local.PasswordProtected { - if authenticate(&local, passDB, auth) { + if publicshare.Authenticate(&local, passDB, auth) { if sign { err := publicshare.AddSignature(&local, passDB) if err != nil { @@ -565,30 +565,6 @@ func (m *manager) writeDb(db map[string]interface{}) error { return nil } -func authenticate(share *link.PublicShare, pw string, auth *link.PublicShareAuthentication) bool { - switch { - case auth.GetPassword() != "": - if err := bcrypt.CompareHashAndPassword([]byte(pw), []byte(auth.GetPassword())); err == nil { - return true - } - case auth.GetSignature() != nil: - sig := auth.GetSignature() - now := time.Now() - expiration := time.Unix(int64(sig.GetSignatureExpiration().GetSeconds()), int64(sig.GetSignatureExpiration().GetNanos())) - if now.After(expiration) { - return false - } - s, err := publicshare.CreateSignature(share.Token, pw, expiration) - if err != nil { - // TODO(labkode): pass ctx to log error - // Now we are blind - return false - } - return sig.GetSignature() == s - } - return false -} - type publicShare struct { link.PublicShare Password string `json:"password"` diff --git a/pkg/publicshare/manager/loader/loader.go b/pkg/publicshare/manager/loader/loader.go index b9698096c3..d5c9db6c10 100644 --- a/pkg/publicshare/manager/loader/loader.go +++ b/pkg/publicshare/manager/loader/loader.go @@ -20,6 +20,7 @@ package loader import ( // Load core share manager drivers. + _ "github.com/cs3org/reva/v2/pkg/publicshare/manager/cs3" _ "github.com/cs3org/reva/v2/pkg/publicshare/manager/json" _ "github.com/cs3org/reva/v2/pkg/publicshare/manager/memory" // Add your own here diff --git a/pkg/publicshare/manager/memory/memory.go b/pkg/publicshare/manager/memory/memory.go index 7fd19ad01d..417a530cbc 100644 --- a/pkg/publicshare/manager/memory/memory.go +++ b/pkg/publicshare/manager/memory/memory.go @@ -104,7 +104,7 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr } // UpdatePublicShare updates the expiration date, permissions and Mtime -func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest, g *link.Grant) (*link.PublicShare, error) { +func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) { log := appctx.GetLogger(ctx) share, err := m.GetPublicShare(ctx, u, req.Ref, false) if err != nil { @@ -167,7 +167,7 @@ func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.Pu return } -func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) { +func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) { // TODO(refs) filter out expired shares shares := []*link.PublicShare{} m.shares.Range(func(k, v interface{}) bool { diff --git a/pkg/publicshare/publicshare.go b/pkg/publicshare/publicshare.go index 601a75b0a9..d85507048a 100644 --- a/pkg/publicshare/publicshare.go +++ b/pkg/publicshare/publicshare.go @@ -31,6 +31,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/v2/pkg/utils" + "golang.org/x/crypto/bcrypt" ) const ( @@ -42,9 +43,9 @@ const ( // Manager manipulates public shares. type Manager interface { CreatePublicShare(ctx context.Context, u *user.User, md *provider.ResourceInfo, g *link.Grant) (*link.PublicShare, error) - UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest, g *link.Grant) (*link.PublicShare, error) + UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) GetPublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference, sign bool) (*link.PublicShare, error) - ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) + ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) RevokePublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference) error GetPublicShareByToken(ctx context.Context, token string, auth *link.PublicShareAuthentication, sign bool) (*link.PublicShare, error) } @@ -163,3 +164,26 @@ func IsExpired(s *link.PublicShare) bool { expiration := time.Unix(int64(s.Expiration.GetSeconds()), int64(s.Expiration.GetNanos())) return s.Expiration != nil && expiration.Before(time.Now()) } + +// Authenticate checks the signature or password authentication for a public share +func Authenticate(share *link.PublicShare, pw string, auth *link.PublicShareAuthentication) bool { + switch { + case auth.GetPassword() != "": + if err := bcrypt.CompareHashAndPassword([]byte(pw), []byte(auth.GetPassword())); err == nil { + return true + } + case auth.GetSignature() != nil: + sig := auth.GetSignature() + now := time.Now() + expiration := time.Unix(int64(sig.GetSignatureExpiration().GetSeconds()), int64(sig.GetSignatureExpiration().GetNanos())) + if now.After(expiration) { + return false + } + s, err := CreateSignature(share.Token, pw, expiration) + if err != nil { + return false + } + return sig.GetSignature() == s + } + return false +} diff --git a/pkg/share/manager/cs3/cs3.go b/pkg/share/manager/cs3/cs3.go index 9a541d09c2..e831407123 100644 --- a/pkg/share/manager/cs3/cs3.go +++ b/pkg/share/manager/cs3/cs3.go @@ -44,28 +44,12 @@ import ( "google.golang.org/genproto/protobuf/field_mask" ) -//go:generate mockery -name Storage -//go:generate mockery -name Indexer - -// Storage is the interface to the metadata storage backend -type Storage interface { - metadata.Storage -} - -// Indexer is the interface to the indexer being used for indexing shares -type Indexer interface { - AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error - Add(t interface{}) ([]indexer.IdxAddResult, error) - FindBy(t interface{}, field string, val string) ([]string, error) - Delete(t interface{}) error -} - // Manager implements a share manager using a cs3 storage backend type Manager struct { sync.RWMutex - storage Storage - indexer Indexer + storage metadata.Storage + indexer indexer.Indexer initialized bool } @@ -106,7 +90,7 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { } // New returns a new manager instance -func New(s Storage, indexer Indexer) (*Manager, error) { +func New(s metadata.Storage, indexer indexer.Indexer) (*Manager, error) { return &Manager{ storage: s, indexer: indexer, diff --git a/pkg/share/manager/cs3/cs3_test.go b/pkg/share/manager/cs3/cs3_test.go index 214dbe833a..f81e7322f5 100644 --- a/pkg/share/manager/cs3/cs3_test.go +++ b/pkg/share/manager/cs3/cs3_test.go @@ -31,8 +31,9 @@ import ( ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share/manager/cs3" - "github.com/cs3org/reva/v2/pkg/share/manager/cs3/mocks" indexerpkg "github.com/cs3org/reva/v2/pkg/storage/utils/indexer" + indexermocks "github.com/cs3org/reva/v2/pkg/storage/utils/indexer/mocks" + storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" "github.com/stretchr/testify/mock" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -42,8 +43,8 @@ import ( var _ = Describe("Manager", func() { var ( - storage *mocks.Storage - indexer *mocks.Indexer + storage *storagemocks.Storage + indexer *indexermocks.Indexer user *userpb.User grantee *userpb.User share *collaboration.Share @@ -57,11 +58,11 @@ var _ = Describe("Manager", func() { ) BeforeEach(func() { - storage = &mocks.Storage{} + storage = &storagemocks.Storage{} storage.On("Init", mock.Anything, mock.Anything).Return(nil) storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) storage.On("SimpleUpload", mock.Anything, mock.Anything, mock.Anything).Return(nil) - indexer = &mocks.Indexer{} + indexer = &indexermocks.Indexer{} indexer.On("AddIndex", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) indexer.On("Add", mock.Anything).Return([]indexerpkg.IdxAddResult{}, nil) indexer.On("Delete", mock.Anything).Return(nil) diff --git a/pkg/storage/utils/indexer/index/non_unique.go b/pkg/storage/utils/indexer/index/non_unique.go index f4b24007ac..f40e992a68 100644 --- a/pkg/storage/utils/indexer/index/non_unique.go +++ b/pkg/storage/utils/indexer/index/non_unique.go @@ -92,6 +92,10 @@ func (idx *NonUnique) Lookup(v string) ([]string, error) { matches = append(matches, path.Base(p)) } + if len(matches) == 0 { + return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v} + } + return matches, nil } diff --git a/pkg/storage/utils/indexer/indexer.go b/pkg/storage/utils/indexer/indexer.go index 8b48b12960..8afd7bef4f 100644 --- a/pkg/storage/utils/indexer/indexer.go +++ b/pkg/storage/utils/indexer/indexer.go @@ -36,8 +36,18 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/sync" ) +//go:generate mockery -name Indexer + // Indexer is a facade to configure and query over multiple indices. -type Indexer struct { +type Indexer interface { + AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error + Add(t interface{}) ([]IdxAddResult, error) + FindBy(t interface{}, field string, val string) ([]string, error) + Delete(t interface{}) error +} + +// StorageIndexer is the indexer implementation using metadata storage +type StorageIndexer struct { storage metadata.Storage indices typeMap mu sync.NamedRWMutex @@ -49,8 +59,8 @@ type IdxAddResult struct { } // CreateIndexer creates a new Indexer. -func CreateIndexer(storage metadata.Storage) *Indexer { - return &Indexer{ +func CreateIndexer(storage metadata.Storage) Indexer { + return &StorageIndexer{ storage: storage, indices: typeMap{}, mu: sync.NewNamedRWMutex(), @@ -58,7 +68,7 @@ func CreateIndexer(storage metadata.Storage) *Indexer { } // Reset takes care of deleting all indices from storage and from the internal map of indices -func (i *Indexer) Reset() error { +func (i *StorageIndexer) Reset() error { for j := range i.indices { for _, indices := range i.indices[j].IndicesByField { for _, idx := range indices { @@ -75,7 +85,7 @@ func (i *Indexer) Reset() error { } // AddIndex adds a new index to the indexer receiver. -func (i *Indexer) AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error { +func (i *StorageIndexer) AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error { var idx index.Index var f func(metadata.Storage, ...option.Option) index.Index @@ -102,7 +112,7 @@ func (i *Indexer) AddIndex(t interface{}, indexBy option.IndexBy, pkName, entity } // Add a new entry to the indexer -func (i *Indexer) Add(t interface{}) ([]IdxAddResult, error) { +func (i *StorageIndexer) Add(t interface{}) ([]IdxAddResult, error) { typeName := getTypeFQN(t) i.mu.Lock(typeName) @@ -136,7 +146,7 @@ func (i *Indexer) Add(t interface{}) ([]IdxAddResult, error) { } // FindBy finds a value on an index by field and value. -func (i *Indexer) FindBy(t interface{}, findBy, val string) ([]string, error) { +func (i *StorageIndexer) FindBy(t interface{}, findBy, val string) ([]string, error) { typeName := getTypeFQN(t) i.mu.RLock(typeName) @@ -171,7 +181,7 @@ func (i *Indexer) FindBy(t interface{}, findBy, val string) ([]string, error) { } // Delete deletes all indexed fields of a given type t on the Indexer. -func (i *Indexer) Delete(t interface{}) error { +func (i *StorageIndexer) Delete(t interface{}) error { typeName := getTypeFQN(t) i.mu.Lock(typeName) @@ -199,7 +209,7 @@ func (i *Indexer) Delete(t interface{}) error { } // FindByPartial allows for glob search across all indexes. -func (i *Indexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) { +func (i *StorageIndexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) { typeName := getTypeFQN(t) i.mu.RLock(typeName) @@ -234,7 +244,7 @@ func (i *Indexer) FindByPartial(t interface{}, field string, pattern string) ([] } // Update updates all indexes on a value to a value . -func (i *Indexer) Update(from, to interface{}) error { +func (i *StorageIndexer) Update(from, to interface{}) error { typeNameFrom := getTypeFQN(from) i.mu.Lock(typeNameFrom) @@ -285,7 +295,7 @@ func (i *Indexer) Update(from, to interface{}) error { } // Query parses an OData query into something our indexer.Index understands and resolves it. -func (i *Indexer) Query(ctx context.Context, t interface{}, q string) ([]string, error) { +func (i *StorageIndexer) Query(ctx context.Context, t interface{}, q string) ([]string, error) { query, err := godata.ParseFilterString(ctx, q) if err != nil { return nil, err @@ -308,7 +318,7 @@ func (i *Indexer) Query(ctx context.Context, t interface{}, q string) ([]string, // conventions and be in PascalCase. For a better overview on this contemplate reading the reflection package under the // indexer directory. Traversal of the tree happens in a pre-order fashion. // TODO implement logic for `and` operators. -func (i *Indexer) resolveTree(t interface{}, tree *queryTree, partials *[]string) error { +func (i *StorageIndexer) resolveTree(t interface{}, tree *queryTree, partials *[]string) error { if partials == nil { return errors.New("return value cannot be nil: partials") } diff --git a/pkg/storage/utils/indexer/indexer_test.go b/pkg/storage/utils/indexer/indexer_test.go index f2e01c2de2..ed96b0b18a 100644 --- a/pkg/storage/utils/indexer/indexer_test.go +++ b/pkg/storage/utils/indexer/indexer_test.go @@ -283,11 +283,11 @@ func TestQueryDiskImpl(t *testing.T) { _ = os.RemoveAll(dataDir) } -func createDiskIndexer(dataDir string) *Indexer { +func createDiskIndexer(dataDir string) *StorageIndexer { storage, err := metadata.NewDiskStorage(dataDir) if err != nil { return nil } - return CreateIndexer(storage) + return CreateIndexer(storage).(*StorageIndexer) } diff --git a/pkg/storage/utils/indexer/map.go b/pkg/storage/utils/indexer/map.go index f15e1165d4..19de60bfcb 100644 --- a/pkg/storage/utils/indexer/map.go +++ b/pkg/storage/utils/indexer/map.go @@ -18,7 +18,10 @@ package indexer -import "github.com/cs3org/reva/v2/pkg/storage/utils/indexer/index" +import ( + "github.com/cs3org/reva/v2/pkg/storage/utils/indexer/index" + "github.com/iancoleman/strcase" +) // typeMap stores the indexer layout at runtime. @@ -33,13 +36,13 @@ type typeMapping struct { func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) { if val, ok := m[typeName]; ok { - val.IndicesByField[idx.IndexBy().String()] = append(val.IndicesByField[idx.IndexBy().String()], idx) + val.IndicesByField[strcase.ToCamel(idx.IndexBy().String())] = append(val.IndicesByField[strcase.ToCamel(idx.IndexBy().String())], idx) return } m[typeName] = typeMapping{ PKFieldName: pkName, IndicesByField: map[string][]index.Index{ - idx.IndexBy().String(): {idx}, + strcase.ToCamel(idx.IndexBy().String()): {idx}, }, } } diff --git a/pkg/share/manager/cs3/mocks/Indexer.go b/pkg/storage/utils/indexer/mocks/Indexer.go similarity index 100% rename from pkg/share/manager/cs3/mocks/Indexer.go rename to pkg/storage/utils/indexer/mocks/Indexer.go diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index dfbc0500bb..81062562ce 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -20,6 +20,7 @@ package metadata import ( "context" + "io/fs" "io/ioutil" "os" "path" @@ -66,6 +67,9 @@ func (disk *Disk) Delete(_ context.Context, path string) error { func (disk *Disk) ReadDir(_ context.Context, p string) ([]string, error) { infos, err := ioutil.ReadDir(disk.targetPath(p)) if err != nil { + if _, ok := err.(*fs.PathError); ok { + return []string{}, nil + } return nil, err } diff --git a/pkg/share/manager/cs3/mocks/Storage.go b/pkg/storage/utils/metadata/mocks/Storage.go similarity index 100% rename from pkg/share/manager/cs3/mocks/Storage.go rename to pkg/storage/utils/metadata/mocks/Storage.go diff --git a/pkg/storage/utils/metadata/storage.go b/pkg/storage/utils/metadata/storage.go index 599f6e3110..633f504ada 100644 --- a/pkg/storage/utils/metadata/storage.go +++ b/pkg/storage/utils/metadata/storage.go @@ -22,6 +22,8 @@ import ( "context" ) +//go:generate mockery -name Storage + // Storage is the interface to maintain metadata in a storage type Storage interface { Backend() string