From a9ce2e003c330a314b365fcb55e869252daa59ab Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 25 Jan 2024 12:08:00 +0100 Subject: [PATCH 1/4] allow bulkrestarting postprocessing Signed-off-by: jkoberg --- .../unreleased/postprocessing-bulk-restart.md | 5 + services/postprocessing/README.md | 8 ++ .../pkg/command/postprocessing.go | 36 +++--- .../pkg/postprocessing/postprocessing.go | 5 +- .../postprocessing/pkg/service/service.go | 114 +++++++++++++----- 5 files changed, 120 insertions(+), 48 deletions(-) create mode 100644 changelog/unreleased/postprocessing-bulk-restart.md diff --git a/changelog/unreleased/postprocessing-bulk-restart.md b/changelog/unreleased/postprocessing-bulk-restart.md new file mode 100644 index 00000000000..d2816f4f5aa --- /dev/null +++ b/changelog/unreleased/postprocessing-bulk-restart.md @@ -0,0 +1,5 @@ +Enhancement: Allow restarting multiple uploads with one command + +Allows to restart all commands in a specific state. + +https://github.com/owncloud/ocis/pull/8287 diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md index 282278d04c5..9e08ba21807 100644 --- a/services/postprocessing/README.md +++ b/services/postprocessing/README.md @@ -90,3 +90,11 @@ ocis storage-users uploads list ```bash ocis postprocessing restart -u ``` + +Instead of starting one specific upload, a system admin can also restart all uploads that are currently in a specific step. +Examples: +``` +ocis postprocessing restart # Restarts all uploads where postprocessing is finished, but upload is not finished +ocis postprocessing restart -s "finished" # Equivalent to the above +ocis postprocessing restart -s "virusscan" # Restart all uploads currently in virusscan step +``` diff --git a/services/postprocessing/pkg/command/postprocessing.go b/services/postprocessing/pkg/command/postprocessing.go index 99c853a0c90..ee155889630 100644 --- a/services/postprocessing/pkg/command/postprocessing.go +++ b/services/postprocessing/pkg/command/postprocessing.go @@ -2,7 +2,6 @@ package command import ( "context" - "time" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/events/stream" @@ -20,10 +19,15 @@ func RestartPostprocessing(cfg *config.Config) *cli.Command { Usage: "restart postprocessing for an uploadID", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "upload-id", - Aliases: []string{"u"}, - Required: true, - Usage: "the uploadid to restart", + Name: "upload-id", + Aliases: []string{"u"}, + Usage: "the uploadid to restart. Ignored if unset.", + }, + &cli.StringFlag{ + Name: "step", + Aliases: []string{"s"}, + Usage: "restarts all uploads in the given postprocessing step. Ignored if upload-id is set.", + Value: "finished", // Calling `ocis postprocessing restart` without any arguments will restart all uploads that are finished but failed to move the uploed from the upload area to the blobstore. }, }, Before: func(c *cli.Context) error { @@ -35,24 +39,18 @@ func RestartPostprocessing(cfg *config.Config) *cli.Command { return err } - ev := events.ResumePostprocessing{ - UploadID: c.String("upload-id"), - Timestamp: utils.TSNow(), + uid, step := c.String("upload-id"), "" + if uid == "" { + step = c.String("step") } - if err := events.Publish(context.Background(), stream, ev); err != nil { - return err + ev := events.ResumePostprocessing{ + UploadID: uid, + Step: events.Postprocessingstep(step), + Timestamp: utils.TSNow(), } - // go-micro nats implementation uses async publishing, - // therefore we need to manually wait. - // - // FIXME: upstream pr - // - // https://github.com/go-micro/plugins/blob/3e77393890683be4bacfb613bc5751867d584692/v4/events/natsjs/nats.go#L115 - time.Sleep(5 * time.Second) - - return nil + return events.Publish(context.Background(), stream, ev) }, } } diff --git a/services/postprocessing/pkg/postprocessing/postprocessing.go b/services/postprocessing/pkg/postprocessing/postprocessing.go index b2764f43f91..e3c9ca6bff7 100644 --- a/services/postprocessing/pkg/postprocessing/postprocessing.go +++ b/services/postprocessing/pkg/postprocessing/postprocessing.go @@ -65,14 +65,14 @@ func (pp *Postprocessing) NextStep(ev events.PostprocessingStepFinished) interfa // CurrentStep returns the current postprocessing step func (pp *Postprocessing) CurrentStep() interface{} { - if pp.Status.Outcome != "" { + if pp.Status.CurrentStep == events.PPStepFinished { return pp.finished(pp.Status.Outcome) } return pp.step(pp.Status.CurrentStep) } // Delay will sleep the configured time then continue -func (pp *Postprocessing) Delay(ev events.StartPostprocessingStep) interface{} { +func (pp *Postprocessing) Delay() interface{} { time.Sleep(pp.config.Delayprocessing) return pp.next(events.PPStepDelay) } @@ -106,6 +106,7 @@ func (pp *Postprocessing) step(next events.Postprocessingstep) events.StartPostp } func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events.PostprocessingFinished { + pp.Status.CurrentStep = events.PPStepFinished pp.Status.Outcome = outcome return events.PostprocessingFinished{ UploadID: pp.ID, diff --git a/services/postprocessing/pkg/service/service.go b/services/postprocessing/pkg/service/service.go index 50355001b17..f2a3c122e26 100644 --- a/services/postprocessing/pkg/service/service.go +++ b/services/postprocessing/pkg/service/service.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" "github.com/owncloud/ocis/v2/services/postprocessing/pkg/postprocessing" @@ -142,29 +143,20 @@ func (pps *PostprocessingService) processEvent(e events.Event) error { pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload") return fmt.Errorf("%w: cannot get upload", errEvent) } - next = pp.Delay(ev) + next = pp.Delay() case events.UploadReady: + if ev.Failed { + // the upload failed - let's keep it around for a while + return nil + } + // the storage provider thinks the upload is done - so no need to keep it any more if err := pps.store.Delete(ev.UploadID); err != nil { pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot delete upload") return fmt.Errorf("%w: cannot delete upload", errEvent) } case events.ResumePostprocessing: - pp, err = pps.getPP(pps.store, ev.UploadID) - if err != nil { - if err == store.ErrNotFound { - if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{ - UploadID: ev.UploadID, - Timestamp: ev.Timestamp, - }); err != nil { - pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot publish RestartPostprocessing event") - } - return fmt.Errorf("%w: cannot publish RestartPostprocessing event", errEvent) - } - pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload") - return fmt.Errorf("%w: cannot get upload", errEvent) - } - next = pp.CurrentStep() + return pps.handleResumePPEvent(ctx, ev) } if pp != nil { @@ -182,11 +174,30 @@ func (pps *PostprocessingService) processEvent(e events.Event) error { return nil } +func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) { + recs, err := sto.Read(uploadID) + if err != nil { + return nil, err + } + + if len(recs) != 1 { + return nil, fmt.Errorf("expected only one result for '%s', got %d", uploadID, len(recs)) + } + + pp := postprocessing.New(pps.c) + err = json.Unmarshal(recs[0].Value, pp) + if err != nil { + return nil, err + } + + return pp, nil +} + func getSteps(c config.Postprocessing) []events.Postprocessingstep { // NOTE: improved version only allows configuring order of postprocessing steps // But we aim for a system where postprocessing steps can be configured per space, ideally by the spaceadmin itself // We need to iterate over configuring PP service when we see fit - var steps []events.Postprocessingstep + steps := make([]events.Postprocessingstep, 0, len(c.Steps)) for _, s := range c.Steps { steps = append(steps, events.Postprocessingstep(s)) } @@ -206,21 +217,70 @@ func storePP(sto store.Store, pp *postprocessing.Postprocessing) error { }) } -func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) { - recs, err := sto.Read(uploadID) - if err != nil { - return nil, err +func (pps *PostprocessingService) handleResumePPEvent(ctx context.Context, ev events.ResumePostprocessing) error { + ids := []string{ev.UploadID} + if ev.Step != "" { + ids = pps.findUploadsByStep(ev.Step) } - if len(recs) != 1 { - return nil, fmt.Errorf("expected only one result for '%s', got %d", uploadID, len(recs)) + for _, id := range ids { + if err := pps.resumePP(ctx, id); err != nil { + pps.log.Error().Str("uploadID", id).Err(err).Msg("cannot resume upload") + return fmt.Errorf("%w: cannot resume upload", errEvent) + } } + return nil +} - pp := postprocessing.New(pps.c) - err = json.Unmarshal(recs[0].Value, pp) +func (pps *PostprocessingService) resumePP(ctx context.Context, uploadID string) error { + pp, err := pps.getPP(pps.store, uploadID) if err != nil { - return nil, err + if err == store.ErrNotFound { + if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{ + UploadID: uploadID, + Timestamp: utils.TSNow(), + }); err != nil { + return err + } + return nil + } + return fmt.Errorf("%w: cannot get upload", errEvent) } - return pp, nil + return events.Publish(ctx, pps.pub, pp.CurrentStep()) +} + +func (pps *PostprocessingService) findUploadsByStep(step events.Postprocessingstep) []string { + var ids []string + + keys, err := pps.store.List() + if err != nil { + pps.log.Error().Err(err).Msg("cannot list uploads") + } + + for _, k := range keys { + rec, err := pps.store.Read(k) + if err != nil { + pps.log.Error().Err(err).Msg("cannot read upload") + continue + } + + if len(rec) != 1 { + pps.log.Error().Err(err).Msg("expected only one result") + continue + } + + pp := &postprocessing.Postprocessing{} + err = json.Unmarshal(rec[0].Value, pp) + if err != nil { + pps.log.Error().Err(err).Msg("cannot unmarshal upload") + continue + } + + if pp.Status.CurrentStep == step { + ids = append(ids, pp.ID) + } + } + + return ids } From b7635f59ee9e3b0dc855bbae6ce37249cea2d98a Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 25 Jan 2024 12:32:23 +0100 Subject: [PATCH 2/4] bump reva Signed-off-by: jkoberg --- changelog/unreleased/bump-reva.md | 1 + go.mod | 2 +- go.sum | 4 ++-- .../internal/grpc/services/gateway/storageprovider.go | 2 +- .../publicshareprovider/publicshareprovider.go | 4 +--- .../sharesstorageprovider/sharesstorageprovider.go | 2 +- .../internal/http/services/appprovider/appprovider.go | 3 +-- .../http/services/owncloud/ocs/data/capabilities.go | 1 - .../ocs/handlers/apps/sharing/shares/shares.go | 3 +-- .../cs3org/reva/v2/pkg/events/postprocessing.go | 3 +++ .../cs3org/reva/v2/pkg/password/password_policies.go | 7 +------ .../reva/v2/pkg/rhttp/datatx/manager/simple/simple.go | 7 +++++++ .../v2/pkg/storage/utils/decomposedfs/decomposedfs.go | 10 +++++++--- .../pkg/storage/utils/decomposedfs/upload/session.go | 8 ++------ .../v2/pkg/storage/utils/decomposedfs/upload/upload.go | 2 +- vendor/go-micro.dev/v4/registry/cache/cache.go | 2 +- vendor/modules.txt | 2 +- 17 files changed, 32 insertions(+), 31 deletions(-) diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index 1e32339c103..1160d3471b0 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -2,6 +2,7 @@ Enhancement: Update reva to latest edge version We update reva to the latest edge version to get the latest fixes and features. +https://github.com/owncloud/ocis/pull/8287 https://github.com/owncloud/ocis/pull/8278 https://github.com/owncloud/ocis/pull/8264 https://github.com/owncloud/ocis/pull/8100 diff --git a/go.mod b/go.mod index 0692f6ef533..5a2b4e45e5b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 + github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e diff --git a/go.sum b/go.sum index 659ada4ed7f..5f68995a9cb 100644 --- a/go.sum +++ b/go.sum @@ -1018,8 +1018,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 h1:g7vQAbo64ziFqqhKcim3JCjDW1zqHy9imAm2HZmmK8w= -github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= +github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da h1:VgWIr/lE6cv2f5IjjWgR0LOAK41gsUytBsSZo/4DRq4= +github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go index 6c1ac742453..cf3a7387e3e 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go @@ -709,7 +709,7 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo if sourceProviderInfo.Address != destProviderInfo.Address { return &provider.MoveResponse{ - Status: status.NewPermissionDenied(ctx, nil, "cross storage moves are not permitted, use copy and delete"), + Status: status.NewUnimplemented(ctx, nil, "cross storage moves are not supported, use copy and delete"), }, nil } diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go index 201b2c0f974..3df749f8823 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -68,7 +68,6 @@ type config struct { } type passwordPolicy struct { - Disabled bool `mapstructure:"disabled"` MinCharacters int `mapstructure:"min_characters"` MinLowerCaseCharacters int `mapstructure:"min_lowercase_characters"` MinUpperCaseCharacters int `mapstructure:"min_uppercase_characters"` @@ -174,10 +173,9 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { func newPasswordPolicy(c *passwordPolicy) password.Validator { if c == nil { - return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( - c.Disabled, c.MinCharacters, c.MinLowerCaseCharacters, c.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go index 1d678d06bcf..da3624fd7ad 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go @@ -667,7 +667,7 @@ func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provide if dstReceivedShare.Share.Id.OpaqueId != srcReceivedShare.Share.Id.OpaqueId { return &provider.MoveResponse{ - Status: status.NewPermissionDenied(ctx, nil, "cross storage moves are not permitted, use copy and delete"), + Status: status.NewUnimplemented(ctx, nil, "cross storage moves are not supported, use copy and delete"), }, nil } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go index c121657885a..2181e98b95d 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go @@ -276,9 +276,8 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { return } defer httpRes.Body.Close() - if httpRes.StatusCode == http.StatusForbidden { + if httpRes.StatusCode == http.StatusBadRequest { // the file upload was already finished since it is a zero byte file - // TODO: why do we get a 401 then!? } else if httpRes.StatusCode != http.StatusOK { writeError(w, r, appErrorServerError, "failed to create the file", nil) return diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go index f6abda35b0c..369ca231792 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go @@ -117,7 +117,6 @@ type CapabilitiesGraph struct { // CapabilitiesPasswordPolicy hold the password policy capabilities type CapabilitiesPasswordPolicy struct { - Disabled bool `json:"disabled" xml:"disabled" mapstructure:"disabled"` MinCharacters int `json:"min_characters" xml:"min_characters" mapstructure:"min_characters"` MaxCharacters int `json:"max_characters" xml:"max_characters" mapstructure:"max_characters"` MinLowerCaseCharacters int `json:"min_lowercase_characters" xml:"min_lowercase_characters" mapstructure:"min_lowercase_characters"` diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index f75a30b3400..59c5ed2f470 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -1714,10 +1714,9 @@ func publicPwdEnforced(c *config.Config) passwordEnforced { func passwordPolicies(c *config.Config) password.Validator { if c.Capabilities.Capabilities == nil || c.Capabilities.Capabilities.PasswordPolicy == nil { - return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( - c.Capabilities.Capabilities.PasswordPolicy.Disabled, c.Capabilities.Capabilities.PasswordPolicy.MinCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinLowerCaseCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go b/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go index cd52a6356d8..69e3f14ac7b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go @@ -42,6 +42,8 @@ var ( PPStepPolicies Postprocessingstep = "policies" // PPStepDelay is the step that processing. Useful for testing or user annoyment PPStepDelay Postprocessingstep = "delay" + // PPStepFinished is the step that signals that postprocessing is finished, but storage provider hasn't acknowledged it yet + PPStepFinished Postprocessingstep = "finished" // PPOutcomeDelete means that the file and the upload should be deleted PPOutcomeDelete PostprocessingOutcome = "delete" @@ -193,6 +195,7 @@ func (UploadReady) Unmarshal(v []byte) (interface{}, error) { // ResumePostprocessing can be emitted to repair broken postprocessing type ResumePostprocessing struct { UploadID string + Step Postprocessingstep Timestamp *types.Timestamp } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go index ac126c9e873..ec98783d318 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go @@ -18,7 +18,6 @@ type Validator interface { // Policies represents a password validation rules type Policies struct { - disabled bool minCharacters int minLowerCaseCharacters int minUpperCaseCharacters int @@ -30,9 +29,8 @@ type Policies struct { } // NewPasswordPolicy returns a new NewPasswordPolicy instance -func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { +func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { p := &Policies{ - disabled: disabled, minCharacters: minCharacters, minLowerCaseCharacters: minLowerCaseCharacters, minUpperCaseCharacters: minUpperCaseCharacters, @@ -48,9 +46,6 @@ func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, min // Validate implements a password validation regarding the policy func (s Policies) Validate(str string) error { - if s.disabled { - return nil - } var allErr error if !utf8.ValidString(str) { return fmt.Errorf("the password contains invalid characters") diff --git a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go index a89ce71ecdd..6699c9f77a2 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go @@ -93,6 +93,13 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { defer func() { metrics.UploadsActive.Sub(1) }() + + if r.ContentLength == 0 { + sublog.Info().Msg("received invalid 0-byte PUT request") + w.WriteHeader(http.StatusBadRequest) + return + } + fn := r.URL.Path defer r.Body.Close() diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 6e709dfad7e..bedb629eeb5 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -286,8 +286,9 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } var ( - failed bool - keepUpload bool + failed bool + revertNodeMetadata bool + keepUpload bool ) unmarkPostprocessing := true @@ -297,12 +298,14 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { fallthrough case events.PPOutcomeAbort: failed = true + revertNodeMetadata = true keepUpload = true metrics.UploadSessionsAborted.Inc() case events.PPOutcomeContinue: if err := session.Finalize(); err != nil { log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not finalize upload") failed = true + revertNodeMetadata = false keepUpload = true // keep postprocessing status so the upload is not deleted during housekeeping unmarkPostprocessing = false @@ -311,6 +314,7 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } case events.PPOutcomeDelete: failed = true + revertNodeMetadata = true metrics.UploadSessionsDeleted.Inc() } @@ -337,7 +341,7 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } } - fs.sessionStore.Cleanup(ctx, session, failed, keepUpload, unmarkPostprocessing) + fs.sessionStore.Cleanup(ctx, session, revertNodeMetadata, keepUpload, unmarkPostprocessing) // remove cache entry in gateway fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go index 4107ea5e2dd..f438d684f9a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go @@ -295,13 +295,9 @@ func (s *OcisSession) MTime() time.Time { return t } -// IsProcessing returns true if the node has entered postprocessing state +// IsProcessing returns true if all bytes have been received. The session then has entered postprocessing state. func (s *OcisSession) IsProcessing() bool { - n, err := s.Node(context.Background()) - if err != nil { - return false - } - return n.IsProcessing(context.Background()) + return s.info.Size == s.info.Offset } // binPath returns the path to the file storing the binary data. diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go index 64c3cb047c6..2f2713b4544 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go @@ -191,7 +191,7 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error { n, err := session.store.CreateNodeForUpload(session, attrs) if err != nil { - session.store.Cleanup(ctx, session, true, false, true) + session.store.Cleanup(ctx, session, true, false, false) return err } diff --git a/vendor/go-micro.dev/v4/registry/cache/cache.go b/vendor/go-micro.dev/v4/registry/cache/cache.go index 5d17d020401..9a7b5e09cd3 100644 --- a/vendor/go-micro.dev/v4/registry/cache/cache.go +++ b/vendor/go-micro.dev/v4/registry/cache/cache.go @@ -459,7 +459,7 @@ func (c *cache) Stop() { } func (c *cache) String() string { - return "cache" + return "cached" } // New returns a new cache diff --git a/vendor/modules.txt b/vendor/modules.txt index fdca12de4da..bb6da18d069 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 +# github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime From 921f7d9d20efc701f26f7502385cfde07740baae Mon Sep 17 00:00:00 2001 From: Sawjan Gurung Date: Fri, 26 Jan 2024 20:56:39 +0545 Subject: [PATCH 3/4] test: adjust MOVE between spaces test expectations --- .../apiSpacesDavOperation/moveByFileId.feature | 12 ++++++------ .../features/apiSpacesShares/moveSpaces.feature | 10 +++++----- .../moveShareInsideAnotherShare.feature | 7 +++---- .../changingFilesShare.feature | 2 +- .../coreApiWebdavMove2/moveShareOnOcis.feature | 8 ++++---- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature index 0a92b25b02a..28ce813d740 100644 --- a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature +++ b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature @@ -94,8 +94,8 @@ Feature: moving/renaming file using file id And user "Brian" has uploaded file with content "some data" to "/test.txt" And we save it into "FILEID" When user "Brian" moves a file "test.txt" into "folder" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" - And the value of the item "/d:error/s:message" in the response about user "Brian" should be "cross storage moves are not permitted, use copy and delete" + Then the HTTP status code should be "502" + And the value of the item "/d:error/s:message" in the response about user "Brian" should be "move:error: not supported: cannot move across spaces" And for user "Brian" folder "/" of the space "Personal" should contain these files: | test.txt | But for user "Alice" folder "folder" of the space "Personal" should not contain these files: @@ -351,7 +351,7 @@ Feature: moving/renaming file using file id And user "Alice" has created folder "testshare" And user "Alice" has shared folder "testshare" with user "Brian" with permissions "" When user "Brian" moves a file "textfile.txt" into "testshare" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "/" of the space "project-space" should contain these files: | textfile.txt | But for user "Brian" folder "testshare" of the space "Shares" should not contain these files: @@ -475,7 +475,7 @@ Feature: moving/renaming file using file id And we save it into "FILEID" And user "Alice" has shared folder "folder" with user "Brian" with permissions "read" When user "Brian" moves a file "Shares/folder/test.txt" into "folder/sub-folder" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "folder/sub-folder" of the space "Shares" should not contain these files: | test.txt | And for user "Alice" folder "folder/sub-folder" of the space "Personal" should not contain these files: @@ -499,7 +499,7 @@ Feature: moving/renaming file using file id And user "Alice" has shared folder "testshare1" with user "Brian" with permissions "" And user "Alice" has shared folder "testshare2" with user "Brian" with permissions "" When user "Brian" moves a file "Shares/testshare1/textfile.txt" into "testshare2" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "testshare1" of the space "Shares" should contain these files: | textfile.txt | But for user "Brian" folder "testshare2" of the space "Shares" should not contain these files: @@ -697,7 +697,7 @@ Feature: moving/renaming file using file id And we save it into "FILEID" And user "Alice" has shared folder "/folder" with user "Brian" with permissions "read" When user "Brian" renames a file "Shares/folder/test.txt" into "folder/sub-folder/renamed.txt" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "folder" of the space "Shares" should contain these files: | test.txt | But for user "Brian" folder "folder/sub-folder" of the space "Shares" should not contain these files: diff --git a/tests/acceptance/features/apiSpacesShares/moveSpaces.feature b/tests/acceptance/features/apiSpacesShares/moveSpaces.feature index 47130d389d3..20413c713e6 100644 --- a/tests/acceptance/features/apiSpacesShares/moveSpaces.feature +++ b/tests/acceptance/features/apiSpacesShares/moveSpaces.feature @@ -125,7 +125,7 @@ Feature: move (rename) file | role | | And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "project.txt" from space "Project" to "/testshare/project.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Project" should contain these entries: | project.txt | But for user "Alice" folder "testshare" of the space "Shares" should not contain these entries: @@ -168,7 +168,7 @@ Feature: move (rename) file And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" And user "Alice" has uploaded file with content "personal content" to "personal.txt" When user "Alice" moves file "personal.txt" from space "Personal" to "/testshare/personal.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Personal" should contain these entries: | personal.txt | But for user "Alice" folder "testshare" of the space "Shares" should not contain these entries: @@ -185,7 +185,7 @@ Feature: move (rename) file And user "Brian" has uploaded file with content "testshare content" to "/testshare/testshare.txt" And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "/testshare/testshare.txt" from space "Shares" to "testshare.txt" inside space "Personal" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Personal" should not contain these entries: | testshare.txt | And for user "Alice" folder "testshare" of the space "Shares" should contain these entries: @@ -207,7 +207,7 @@ Feature: move (rename) file And user "Brian" has uploaded file with content "testshare content" to "/testshare/testshare.txt" And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "/testshare/testshare.txt" from space "Shares" to "testshare.txt" inside space "Project" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Project" should not contain these entries: | /testshare.txt | And for user "Alice" folder "testshare" of the space "Shares" should contain these entries: @@ -232,7 +232,7 @@ Feature: move (rename) file And user "Brian" has shared folder "/testshare1" with user "Alice" with permissions "" And user "Brian" has shared folder "/testshare2" with user "Alice" with permissions "" When user "Alice" moves file "/testshare1/testshare1.txt" from space "Shares" to "/testshare2/testshare1.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" folder "testshare1" of the space "Shares" should contain these entries: | testshare1.txt | But for user "Alice" folder "testshare2" of the space "Shares" should not contain these entries: diff --git a/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature b/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature index b13cbd7e17c..0c16fe0f04c 100644 --- a/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature +++ b/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature @@ -20,7 +20,7 @@ Feature: moving a share inside another share Scenario: share receiver cannot move a whole share inside another share When user "Brian" moves folder "Shares/folderB" to "Shares/folderA/folderB" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/folderB" should exist And as "Brian" folder "/Shares/folderB" should exist And as "Alice" file "/folderB/fileB.txt" should exist @@ -43,7 +43,7 @@ Feature: moving a share inside another share And user "Brian" has created folder "localFolder/subFolder" And user "Brian" has uploaded file with content "local text" to "/localFolder/localFile.txt" When user "Brian" moves folder "localFolder" to "Shares/folderA/localFolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Brian" folder "/Shares/folderA/localFolder" should not exist And as "Alice" folder "/folderA/localFolder" should not exist And as "Brian" folder "/localFolder" should exist @@ -52,8 +52,7 @@ Feature: moving a share inside another share Scenario: share receiver tries to move a whole share inside a local folder Given user "Brian" has created folder "localFolder" And user "Brian" has uploaded file with content "local text" to "/localFolder/localFile.txt" - # On oCIS you cannot move received shares out of the "Shares" folder When user "Brian" moves folder "Shares/folderB" to "localFolder/folderB" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "/folderB/fileB.txt" should exist And as "Brian" file "/Shares/folderB/fileB.txt" should exist diff --git a/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature b/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature index 54f5cf2c2a0..0aea127ba20 100644 --- a/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature +++ b/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature @@ -20,7 +20,7 @@ Feature: sharing And user "Alice" has shared folder "/share1" with user "Brian" And user "Alice" has shared folder "/share2" with user "Brian" When user "Brian" moves file "/Shares/share1/textfile0.txt" to "/Shares/share2/textfile0.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Brian" file "/Shares/share1/textfile0.txt" should exist And as "Alice" file "share1/textfile0.txt" should exist But as "Brian" file "/Shares/share2/textfile0.txt" should not exist diff --git a/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature b/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature index feba1ab4f6a..da4e86296d5 100644 --- a/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature +++ b/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature @@ -45,7 +45,7 @@ Feature: move (rename) file | shareWith | Alice | And user "Alice" has uploaded file with content "test data" to "/testfile.txt" When user "Alice" moves file "/testfile.txt" to "Shares/testshare/testfile.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "Shares/testshare/testfile.txt" should not exist And as "Brian" file "testshare/testfile.txt" should not exist But as "Alice" file "/testfile.txt" should exist @@ -95,7 +95,7 @@ Feature: move (rename) file | permissions | | | shareWith | Alice | When user "Alice" moves file "/Shares/testshare/testfile.txt" to "/testfile.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "/Shares/testshare/testfile.txt" should exist And as "Brian" file "/testshare/testfile.txt" should exist Examples: @@ -146,7 +146,7 @@ Feature: move (rename) file And user "Alice" has created folder "/testsubfolder" And user "Alice" has uploaded file with content "test data" to "/testsubfolder/testfile.txt" When user "Alice" moves folder "/testsubfolder" to "Shares/testshare/testsubfolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/Shares/testshare/testsubfolder" should not exist And as "Brian" folder "/testshare/testsubfolder" should not exist But as "Alice" folder "/testsubfolder" should exist @@ -202,7 +202,7 @@ Feature: move (rename) file | permissions | | | shareWith | Alice | When user "Alice" moves folder "/Shares/testshare/testsubfolder" to "/testsubfolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/Shares/testshare/testsubfolder" should exist And as "Brian" folder "/testshare/testsubfolder" should exist Examples: From d8d7795122ba6e919755a45d8608e39f36861195 Mon Sep 17 00:00:00 2001 From: Sawjan Gurung Date: Fri, 26 Jan 2024 21:19:11 +0545 Subject: [PATCH 4/4] test: revert error message --- .../features/apiSpacesDavOperation/moveByFileId.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature index 28ce813d740..7d46357de83 100644 --- a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature +++ b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature @@ -95,7 +95,7 @@ Feature: moving/renaming file using file id And we save it into "FILEID" When user "Brian" moves a file "test.txt" into "folder" inside space "Shares" using file-id path "" Then the HTTP status code should be "502" - And the value of the item "/d:error/s:message" in the response about user "Brian" should be "move:error: not supported: cannot move across spaces" + And the value of the item "/d:error/s:message" in the response about user "Brian" should be "cross storage moves are not supported, use copy and delete" And for user "Brian" folder "/" of the space "Personal" should contain these files: | test.txt | But for user "Alice" folder "folder" of the space "Personal" should not contain these files: