From f4009cc8b7d8c54b03f3ae538a877f8c1f690387 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 13 Feb 2020 14:17:58 +0100 Subject: [PATCH 01/14] new storage template package --- go.mod | 8 +- go.sum | 16 ++++ pkg/storage/templates/templates.go | 68 +++++++++++++++ pkg/storage/templates/templates_test.go | 105 ++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 pkg/storage/templates/templates.go create mode 100644 pkg/storage/templates/templates_test.go diff --git a/go.mod b/go.mod index fe01337b05..17521ad320 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,9 @@ require ( contrib.go.opencensus.io/exporter/jaeger v0.2.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/BurntSushi/toml v0.3.1 + github.com/Masterminds/goutils v1.1.0 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.22.0+incompatible github.com/aws/aws-sdk-go v1.28.13 github.com/cheggaaa/pb v1.0.28 github.com/coreos/go-oidc v2.2.1+incompatible @@ -16,10 +19,13 @@ require ( github.com/golang/protobuf v1.3.3 github.com/gomodule/redigo v2.0.0+incompatible github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 + github.com/huandu/xstrings v1.3.0 // indirect + github.com/imdario/mergo v0.3.8 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/ory/fosite v0.30.2 github.com/pkg/errors v0.9.1 @@ -30,7 +36,6 @@ require ( github.com/rs/zerolog v1.17.2 go.opencensus.io v0.22.3 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a - golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 google.golang.org/grpc v1.27.1 @@ -38,7 +43,6 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.2.2 // indirect - honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect ) go 1.13 diff --git a/go.sum b/go.sum index bd192f6939..b16c45c585 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,12 @@ contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= @@ -87,6 +93,10 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo= +github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= @@ -104,8 +114,12 @@ github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= @@ -165,6 +179,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -270,6 +285,7 @@ gopkg.in/square/go-jose.v2 v2.1.9/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go new file mode 100644 index 0000000000..bafd6874b4 --- /dev/null +++ b/pkg/storage/templates/templates.go @@ -0,0 +1,68 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/* +Package templates contains data-driven templates for path layouts. + +Templates can use functions from the gitbub.com/Masterminds/sprig library. +All templates are cleaned with path.Clean(). +*/ +package templates + +import ( + "bytes" + "fmt" + "path" + "text/template" + + "github.com/Masterminds/sprig" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + "github.com/pkg/errors" +) + +// UserData contains the template placeholders for a user. +// For example {{.Username}} or {{.Id.Idp}} +type UserData struct { + *userpb.User +} + +func WithUser(u *userpb.User, tpl string) string { + tpl = clean(tpl) + ut := newUserData(u) + // compile given template tpl + t, err := template.New("tpl").Funcs(sprig.TxtFuncMap()).Parse(tpl) + if err != nil { + err := errors.Wrap(err, fmt.Sprintf("error parsing template: user_template:%+v tpl:%s", ut, tpl)) + panic(err) + } + b := bytes.Buffer{} + if err := t.Execute(&b, u); err != nil { + err := errors.Wrap(err, fmt.Sprintf("error executing template: user_template:%+v tpl:%s", ut, tpl)) + panic(err) + } + return b.String() +} + +func newUserData(u *userpb.User) *UserData { + ut := &UserData{User: u} + return ut +} + +func clean(a string) string { + return path.Clean(a) +} diff --git a/pkg/storage/templates/templates_test.go b/pkg/storage/templates/templates_test.go new file mode 100644 index 0000000000..9743e5d132 --- /dev/null +++ b/pkg/storage/templates/templates_test.go @@ -0,0 +1,105 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package templates + +import ( + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" +) + +type testUnit struct { + expected string + template string + user *userpb.User +} + +var tests = []*testUnit{ + &testUnit{ + expected: "alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "{{.Username}}", + }, + &testUnit{ + expected: "a/alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "{{substr 0 1 .Username}}/{{.Username}}", + }, + &testUnit{ + expected: "idp@opaque", + user: &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp", + OpaqueId: "opaque", + }, + }, + template: "{{.Id.Idp}}@{{.Id.OpaqueId}}", + }, + &testUnit{ // test path clean + expected: "/alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "///{{.Username}}", + }, +} + +func TestLayout(t *testing.T) { + for _, u := range tests { + got := WithUser(u.user, u.template) + if u.expected != got { + t.Fatal("expected: " + u.expected + " got: " + got) + } + } +} + +func TestLayoutPanic(t *testing.T) { + assertPanic(t, testBadLayout) +} + +func TestUserPanic(t *testing.T) { + assertPanic(t, testBadUser) +} + +// should panic +func testBadLayout() { + layout := "{{ bad layout sintax" + user := &userpb.User{} + WithUser(user, layout) +} + +//should panic +func testBadUser() { + layout := "{{ .DoesNotExist }}" + user := &userpb.User{} + WithUser(user, layout) +} + +func assertPanic(t *testing.T, f func()) { + defer func() { + if r := recover(); r == nil { + t.Errorf("the code did not panic") + } + }() + f() +} From d14695152e947bc95187f4c9c119e9e74acac45f Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 13 Feb 2020 14:45:07 +0100 Subject: [PATCH 02/14] fix lint --- pkg/storage/templates/templates.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index bafd6874b4..5884276b21 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -41,6 +41,7 @@ type UserData struct { *userpb.User } +// WithUser generates a layout based on user data. func WithUser(u *userpb.User, tpl string) string { tpl = clean(tpl) ut := newUserData(u) From 2081c97959c975e0e822cd029628869b2df5b6b9 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 13 Feb 2020 15:54:41 +0100 Subject: [PATCH 03/14] remve pw magic --- cmd/revad/runtime/loader.go | 1 - go.sum | 1 + .../storageprovider/storageprovider.go | 16 -- pkg/storage/fs/eos/eos.go | 91 +++---- pkg/storage/fs/owncloud/owncloud.go | 233 ++++++++---------- pkg/storage/helper/helper.go | 89 ------- pkg/storage/pw/context/context.go | 99 -------- pkg/storage/pw/loader/loader.go | 25 -- pkg/storage/pw/registry/registry.go | 34 --- 9 files changed, 149 insertions(+), 440 deletions(-) delete mode 100644 pkg/storage/helper/helper.go delete mode 100644 pkg/storage/pw/context/context.go delete mode 100644 pkg/storage/pw/loader/loader.go delete mode 100644 pkg/storage/pw/registry/registry.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index ebdd419fe2..8a910f3381 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -32,7 +32,6 @@ import ( _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/share/manager/loader" _ "github.com/cs3org/reva/pkg/storage/fs/loader" - _ "github.com/cs3org/reva/pkg/storage/pw/loader" _ "github.com/cs3org/reva/pkg/storage/registry/loader" _ "github.com/cs3org/reva/pkg/token/manager/loader" _ "github.com/cs3org/reva/pkg/user/manager/loader" diff --git a/go.sum b/go.sum index b16c45c585..ee3c98072a 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 425f2420a5..c96de836e4 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - pwregistry "github.com/cs3org/reva/pkg/storage/pw/registry" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -134,10 +133,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { if err != nil { return nil, err } - pw, err := getPW(c) - if err != nil { - return nil, err - } // parse data server url u, err := url.Parse(c.DataServerURL) @@ -158,7 +153,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, storage: fs, - pathWrapper: pw, tmpFolder: tmpFolder, mountPath: mountPath, mountID: mountID, @@ -789,16 +783,6 @@ func getFS(c *config) (storage.FS, error) { return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getPW(c *config) (storage.PathWrapper, error) { - if c.PathWrapper == "" { - return nil, nil - } - if f, ok := pwregistry.NewFuncs[c.PathWrapper]; ok { - return f(c.PathWrappers[c.PathWrapper]) - } - return nil, fmt.Errorf("path wrapper not found: %s", c.Driver) -} - func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) { if ref.GetId() != nil { idRef := &provider.Reference{ diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index a1d85444d5..01bdb4bb8f 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -31,14 +31,13 @@ import ( "strconv" "strings" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/helper" - "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/acl" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/templates" "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -121,8 +120,11 @@ type config struct { // SingleUsername is the username to use when SingleUserMode is enabled SingleUsername string `mapstructure:"single_username"` - // Layout - Layout string `mapstructure:"layout"` + // UserLayout wraps the internal path with user information. + // Example: if mountpoint is /eos/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /eos/user//docs + UserLayout string `mapstructure:"layout"` } func getUser(ctx context.Context) (*userpb.User, error) { @@ -159,10 +161,6 @@ func (c *config) init() { if c.CacheDirectory == "" { c.CacheDirectory = os.TempDir() } - - if c.Layout == "" { - c.Layout = "{{.Username}}" - } } // New returns a new implementation of the storage.FS interface that connects to EOS. @@ -205,33 +203,43 @@ func New(m map[string]interface{}) (storage.FS, error) { return eosStorage, nil } -func (fs *eosStorage) getHomeForUser(u *userpb.User) (string, error) { - userhome, err := helper.GetUserHomePath(u, fs.conf.Layout) - if err != nil { - return "", err - } - - home := path.Join(fs.mountpoint, userhome) - return home, nil -} - func (fs *eosStorage) Shutdown(ctx context.Context) error { // TODO(labkode): in a grpc implementation we can close connections. return nil } -func (fs *eosStorage) getInternalPath(ctx context.Context, fn string) string { - internalPath := path.Join(fs.mountpoint, fn) - return internalPath +func (fs *eosStorage) wrap(ctx context.Context, fn string) (internal string) { + if fs.conf.EnableHome && fs.conf.UserLayout != "" { + u, err := getUser(ctx) + if err != nil { + err = errors.Wrap(err, "eos: wrap: no user in ctx and home is enabled") + panic(err) + } + layout := templates.WithUser(u, fs.conf.UserLayout) + internal = path.Join(fs.mountpoint, layout, fn) + } else { + internal = path.Join(fs.mountpoint, fn) + } + return } -func (fs *eosStorage) removeNamespace(ctx context.Context, np string) string { - p := strings.TrimPrefix(np, fs.mountpoint) - if p == "" { - p = "/" +func (fs *eosStorage) unwrap(ctx context.Context, np string) (external string) { + if fs.conf.EnableHome && fs.conf.UserLayout != "" { + u, err := getUser(ctx) + if err != nil { + err = errors.Wrap(err, "eos: unwrap: no user in ctx and home is enabled") + panic(err) + } + layout := templates.WithUser(u, fs.conf.UserLayout) + trim := path.Join(fs.mountpoint, layout) + external = strings.TrimPrefix(np, trim) + } else { + external = strings.TrimPrefix(np, fs.mountpoint) + if external == "" { + external = "/" + } } - - return p + return } func (fs *eosStorage) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { @@ -259,7 +267,7 @@ func (fs *eosStorage) GetPathByID(ctx context.Context, id *provider.ResourceId) // resolve takes in a request path or request id and converts it to a internal path. func (fs *eosStorage) resolve(ctx context.Context, u *userpb.User, ref *provider.Reference) (string, error) { if ref.GetPath() != "" { - return fs.getInternalPath(ctx, ref.GetPath()), nil + return fs.wrap(ctx, ref.GetPath()), nil } if ref.GetId() != nil { @@ -577,26 +585,25 @@ func (fs *eosStorage) GetQuota(ctx context.Context) (int, int, error) { } func (fs *eosStorage) GetHome(ctx context.Context) (string, error) { - u, err := getUser(ctx) - if err != nil { - return "", errors.Wrap(err, "eos: no user in ctx") - } - - home, err := fs.getHomeForUser(u) - if err != nil { - return "", err + if !fs.conf.EnableHome { + return "", errtypes.NotSupported("eos: get home not supported") } + home := fs.wrap(ctx, "/") return home, nil } func (fs *eosStorage) CreateHome(ctx context.Context) error { + if !fs.conf.EnableHome { + return errtypes.NotSupported("eos: create home not supported") + } + u, err := getUser(ctx) if err != nil { return errors.Wrap(err, "eos: no user in ctx") } - home, err := fs.getHomeForUser(u) + home := fs.wrap(ctx, "/") if err != nil { return err } @@ -665,7 +672,7 @@ func (fs *eosStorage) CreateDir(ctx context.Context, fn string) error { return errors.Wrap(err, "eos: no user in ctx") } - fn = fs.getInternalPath(ctx, fn) + fn = fs.wrap(ctx, fn) return fs.c.CreateDir(ctx, u.Username, fn) } @@ -810,7 +817,7 @@ func (fs *eosStorage) DownloadRevision(ctx context.Context, ref *provider.Refere return nil, errors.Wrap(err, "eos: error resolving reference") } - fn = fs.getInternalPath(ctx, fn) + fn = fs.wrap(ctx, fn) return fs.c.ReadVersion(ctx, u.Username, fn, revisionKey) } @@ -878,7 +885,7 @@ func (fs *eosStorage) RestoreRecycleItem(ctx context.Context, key string) error func (fs *eosStorage) convertToRecycleItem(ctx context.Context, eosDeletedItem *eosclient.DeletedEntry) *provider.RecycleItem { recycleItem := &provider.RecycleItem{ - Path: fs.removeNamespace(ctx, eosDeletedItem.RestorePath), + Path: fs.unwrap(ctx, eosDeletedItem.RestorePath), Key: eosDeletedItem.RestoreKey, Size: eosDeletedItem.Size, DeletionTime: &types.Timestamp{Seconds: eosDeletedItem.DeletionMTime / 1000}, // TODO(labkode): check if eos time is millis or nanos @@ -903,7 +910,7 @@ func (fs *eosStorage) convertToRevision(ctx context.Context, eosFileInfo *eoscli } func (fs *eosStorage) convertToResourceInfo(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { - path := fs.removeNamespace(ctx, eosFileInfo.File) + path := fs.unwrap(ctx, eosFileInfo.File) size := eosFileInfo.Size if eosFileInfo.IsDir { size = eosFileInfo.TreeSize diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 7b3f9e6192..f8f90e34fc 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -41,7 +41,6 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/gomodule/redigo/redis" @@ -218,29 +217,20 @@ func New(m map[string]interface{}) (storage.FS, error) { }, } - return &ocFS{c: c, pool: pool}, nil + return &ocfs{c: c, pool: pool}, nil } -type ocFS struct { +type ocfs struct { c *config pool *redis.Pool } -func (fs *ocFS) Shutdown(ctx context.Context) error { +func (fs *ocfs) Shutdown(ctx context.Context) error { return fs.pool.Close() } -func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - err := errors.Wrap(errtypes.UserRequired(""), "owncloud: error getting user from ctx") - return nil, err - } - return u, nil -} - // scan files and add uuid to path mapping to kv store -func (fs *ocFS) scanFiles(ctx context.Context, conn redis.Conn) { +func (fs *ocfs) scanFiles(ctx context.Context, conn redis.Conn) { if fs.c.Scan { fs.c.Scan = false // TODO ... in progress use mutex ? log := appctx.GetLogger(ctx) @@ -279,7 +269,7 @@ func (fs *ocFS) scanFiles(ctx context.Context, conn redis.Conn) { // the incoming path starts with /, so we need to insert the files subfolder into the path // and prefix the datadirectory // TODO the path handed to a storage provider should not contain the username -func (fs *ocFS) getInternalPath(ctx context.Context, fn string) string { +func (fs *ocfs) wrap(ctx context.Context, fn string) string { // trim all / fn = strings.Trim(fn, "/") // p = "" or @@ -305,7 +295,7 @@ func (fs *ocFS) getInternalPath(ctx context.Context, fn string) string { // the incoming path starts with /, so we need to insert the files subfolder into the path // and prefix the datadirectory // TODO the path handed to a storage provider should not contain the username -func (fs *ocFS) getVersionsPath(ctx context.Context, np string) string { +func (fs *ocfs) getVersionsPath(ctx context.Context, np string) string { // np = /path/to/data//files/foo/bar.txt // remove data dir if fs.c.DataDirectory != "/" { @@ -329,7 +319,7 @@ func (fs *ocFS) getVersionsPath(ctx context.Context, np string) string { } // ownloud stores trashed items in the files_trashbin subfolder of a users home -func (fs *ocFS) getRecyclePath(ctx context.Context) (string, error) { +func (fs *ocfs) getRecyclePath(ctx context.Context) (string, error) { u, ok := user.ContextGetUser(ctx) if !ok { err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") @@ -338,7 +328,7 @@ func (fs *ocFS) getRecyclePath(ctx context.Context) (string, error) { return path.Join(fs.c.DataDirectory, u.GetUsername(), "files_trashbin/files"), nil } -func (fs *ocFS) removeNamespace(ctx context.Context, np string) string { +func (fs *ocfs) unwrap(ctx context.Context, np string) string { // np = /data//files/foo/bar.txt // remove data dir if fs.c.DataDirectory != "/" { @@ -370,9 +360,9 @@ func getOwner(fn string) string { return "" } -func (fs *ocFS) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np string, c redis.Conn) *provider.ResourceInfo { +func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np string, c redis.Conn) *provider.ResourceInfo { id := readOrCreateID(ctx, np, c) - fn := fs.removeNamespace(ctx, path.Join("/", np)) + fn := fs.unwrap(ctx, path.Join("/", np)) etag := calcEtag(ctx, fi) @@ -471,7 +461,7 @@ func readOrCreateID(ctx context.Context, np string, conn redis.Conn) string { return uid.String() } -func (fs *ocFS) getPath(ctx context.Context, id *provider.ResourceId) (string, error) { +func (fs *ocfs) getPath(ctx context.Context, id *provider.ResourceId) (string, error) { c := fs.pool.Get() defer c.Close() fs.scanFiles(ctx, c) @@ -484,18 +474,18 @@ func (fs *ocFS) getPath(ctx context.Context, id *provider.ResourceId) (string, e } // GetPathByID returns the fn pointed by the file id, without the internal namespace -func (fs *ocFS) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { +func (fs *ocfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { np, err := fs.getPath(ctx, id) if err != nil { return "", err } - return fs.removeNamespace(ctx, np), nil + return fs.unwrap(ctx, np), nil } // resolve takes in a request path or request id and converts it to a internal path. -func (fs *ocFS) resolve(ctx context.Context, ref *provider.Reference) (string, error) { +func (fs *ocfs) resolve(ctx context.Context, ref *provider.Reference) (string, error) { if ref.GetPath() != "" { - return fs.getInternalPath(ctx, ref.GetPath()), nil + return fs.wrap(ctx, ref.GetPath()), nil } if ref.GetId() != nil { @@ -510,10 +500,10 @@ func (fs *ocFS) resolve(ctx context.Context, ref *provider.Reference) (string, e return "", fmt.Errorf("invalid reference %+v", ref) } -func (fs *ocFS) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { +func (fs *ocfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { np, err := fs.resolve(ctx, ref) if err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } e, err := fs.getACE(g) @@ -591,7 +581,7 @@ func getACEPerm(set *provider.ResourcePermissions) (string, error) { return b.String(), nil } -func (fs *ocFS) getACE(g *provider.Grant) (*ace, error) { +func (fs *ocfs) getACE(g *provider.Grant) (*ace, error) { permissions, err := getACEPerm(g.Permissions) if err != nil { return nil, err @@ -718,11 +708,11 @@ func getACEs(ctx context.Context, fsfn string, attrs []string) (entries []*ace, return entries, nil } -func (fs *ocFS) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { +func (fs *ocfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { log := appctx.GetLogger(ctx) var np string if np, err = fs.resolve(ctx, ref); err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving reference") + return nil, errors.Wrap(err, "ocfs: error resolving reference") } var attrs []string if attrs, err = xattr.List(np); err != nil { @@ -753,14 +743,14 @@ func (fs *ocFS) ListGrants(ctx context.Context, ref *provider.Reference) (grants return grants, nil } -func (fs *ocFS) getGranteeType(e *ace) provider.GranteeType { +func (fs *ocfs) getGranteeType(e *ace) provider.GranteeType { if strings.Contains(e.Flags, "g") { return provider.GranteeType_GRANTEE_TYPE_GROUP } return provider.GranteeType_GRANTEE_TYPE_USER } -func (fs *ocFS) getGrantPermissionSet(mode string) *provider.ResourcePermissions { +func (fs *ocfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { p := &provider.ResourcePermissions{} // r if strings.Contains(mode, "r") { @@ -829,11 +819,11 @@ func (fs *ocFS) getGrantPermissionSet(mode string) *provider.ResourcePermissions return p } -func (fs *ocFS) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { +func (fs *ocfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { var np string if np, err = fs.resolve(ctx, ref); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } var attr string @@ -846,33 +836,16 @@ func (fs *ocFS) RemoveGrant(ctx context.Context, ref *provider.Reference, g *pro return xattr.Remove(np, attr) } -func (fs *ocFS) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { +func (fs *ocfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { return fs.AddGrant(ctx, ref, g) } -func (fs *ocFS) GetQuota(ctx context.Context) (int, int, error) { +func (fs *ocfs) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, nil } -func (fs *ocFS) getHomeForUser(u *userpb.User) (string, error) { - userhome, err := helper.GetUserHomePath(u, fs.c.Layout) - if err != nil { - return "", err - } - - return path.Join("/", userhome), nil -} - -func (fs *ocFS) CreateHome(ctx context.Context) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "ocFS: no user in ctx") - } - - home, err := fs.getHomeForUser(u) - if err != nil { - return err - } +func (fs *ocfs) CreateHome(ctx context.Context) error { + home := fs.wrap(ctx, "/") homePaths := []string{ path.Join(fs.c.DataDirectory, home, "files"), @@ -881,59 +854,51 @@ func (fs *ocFS) CreateHome(ctx context.Context) error { } for _, v := range homePaths { - if err = os.MkdirAll(v, 0700); err != nil { - return errors.Wrap(err, "ocFS: error creating home path: "+v) + if err := os.MkdirAll(v, 0700); err != nil { + return errors.Wrap(err, "ocfs: error creating home path: "+v) } } return nil } -func (fs *ocFS) GetHome(ctx context.Context) (string, error) { - u, err := getUser(ctx) - if err != nil { - return "", errors.Wrap(err, "ocFS: no user in ctx") - } - - home, err := fs.getHomeForUser(u) - if err != nil { - return "", err - } +func (fs *ocfs) GetHome(ctx context.Context) (string, error) { + home := fs.wrap(ctx, "/") return home, nil } -func (fs *ocFS) CreateDir(ctx context.Context, fn string) (err error) { - np := fs.getInternalPath(ctx, fn) +func (fs *ocfs) CreateDir(ctx context.Context, fn string) (err error) { + np := fs.wrap(ctx, fn) if err = os.Mkdir(np, 0700); err != nil { if os.IsNotExist(err) { return errtypes.NotFound(fn) } // FIXME we also need already exists error, webdav expects 405 MethodNotAllowed - return errors.Wrap(err, "ocFS: error creating dir "+np) + return errors.Wrap(err, "ocfs: error creating dir "+np) } return nil } -func (fs *ocFS) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { +func (fs *ocfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { // TODO(jfd): implement return errtypes.NotSupported("owncloud: operation not supported") } -func (fs *ocFS) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { +func (fs *ocfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { log := appctx.GetLogger(ctx) var np string if np, err = fs.resolve(ctx, ref); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } var fi os.FileInfo fi, err = os.Stat(np) if err != nil { if os.IsNotExist(err) { - return errtypes.NotFound(fs.removeNamespace(ctx, np)) + return errtypes.NotFound(fs.unwrap(ctx, np)) } - return errors.Wrap(err, "ocFS: error stating "+np) + return errors.Wrap(err, "ocfs: error stating "+np) } errs := []error{} @@ -1061,20 +1026,20 @@ func parseMTime(v string) (t time.Time, err error) { return time.Unix(sec, nsec), err } -func (fs *ocFS) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { +func (fs *ocfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { log := appctx.GetLogger(ctx) var np string if np, err = fs.resolve(ctx, ref); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } _, err = os.Stat(np) if err != nil { if os.IsNotExist(err) { - return errtypes.NotFound(fs.removeNamespace(ctx, np)) + return errtypes.NotFound(fs.unwrap(ctx, np)) } - return errors.Wrap(err, "ocFS: error stating "+np) + return errors.Wrap(err, "ocfs: error stating "+np) } errs := []error{} @@ -1130,24 +1095,24 @@ func (fs *ocFS) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refere } // Delete is actually only a move to trash -func (fs *ocFS) Delete(ctx context.Context, ref *provider.Reference) (err error) { +func (fs *ocfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { var np string if np, err = fs.resolve(ctx, ref); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } rp, err := fs.getRecyclePath(ctx) if err != nil { - return errors.Wrap(err, "ocFS: error resolving recycle path") + return errors.Wrap(err, "ocfs: error resolving recycle path") } if err := os.MkdirAll(rp, 0700); err != nil { - return errors.Wrap(err, "ocFS: error creating trashbin dir "+rp) + return errors.Wrap(err, "ocfs: error creating trashbin dir "+rp) } // np is the path on disk ... we need only the path relative to root - origin := path.Dir(fs.removeNamespace(ctx, np)) + origin := path.Dir(fs.unwrap(ctx, np)) // and we need to get rid of the user prefix parts := strings.SplitN(origin, "/", 3) @@ -1159,7 +1124,7 @@ func (fs *ocFS) Delete(ctx context.Context, ref *provider.Reference) (err error) case 3: fp = path.Join("/", parts[2]) default: - return errors.Wrap(err, "ocFS: error creating trashbin dir "+rp) + return errors.Wrap(err, "ocfs: error creating trashbin dir "+rp) } // set origin location in metadata @@ -1170,40 +1135,40 @@ func (fs *ocFS) Delete(ctx context.Context, ref *provider.Reference) (err error) // move to trash location tgt := path.Join(rp, fmt.Sprintf("%s.d%d", path.Base(np), time.Now().Unix())) if err := os.Rename(np, tgt); err != nil { - return errors.Wrap(err, "ocFS: could not restore item") + return errors.Wrap(err, "ocfs: could not restore item") } // TODO(jfd) move versions to trash return nil } -func (fs *ocFS) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { +func (fs *ocfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { var oldName string if oldName, err = fs.resolve(ctx, oldRef); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } var newName string if newName, err = fs.resolve(ctx, newRef); err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } if err = os.Rename(oldName, newName); err != nil { - return errors.Wrap(err, "ocFS: error moving "+oldName+" to "+newName) + return errors.Wrap(err, "ocfs: error moving "+oldName+" to "+newName) } return nil } -func (fs *ocFS) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) { +func (fs *ocfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) { np, err := fs.resolve(ctx, ref) if err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving reference") + return nil, errors.Wrap(err, "ocfs: error resolving reference") } md, err := os.Stat(np) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(fs.removeNamespace(ctx, np)) + return nil, errtypes.NotFound(fs.unwrap(ctx, np)) } - return nil, errors.Wrap(err, "ocFS: error stating "+np) + return nil, errors.Wrap(err, "ocfs: error stating "+np) } c := fs.pool.Get() defer c.Close() @@ -1212,18 +1177,18 @@ func (fs *ocFS) GetMD(ctx context.Context, ref *provider.Reference) (*provider.R return m, nil } -func (fs *ocFS) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) { +func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) { np, err := fs.resolve(ctx, ref) if err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving reference") + return nil, errors.Wrap(err, "ocfs: error resolving reference") } mds, err := ioutil.ReadDir(np) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(fs.removeNamespace(ctx, np)) + return nil, errtypes.NotFound(fs.unwrap(ctx, np)) } - return nil, errors.Wrap(err, "ocFS: error listing "+np) + return nil, errors.Wrap(err, "ocfs: error listing "+np) } finfos := make([]*provider.ResourceInfo, 0, len(mds)) @@ -1238,10 +1203,10 @@ func (fs *ocFS) ListFolder(ctx context.Context, ref *provider.Reference) ([]*pro return finfos, nil } -func (fs *ocFS) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { +func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { np, err := fs.resolve(ctx, ref) if err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } // we cannot rely on /tmp as it can live in another partition and we can @@ -1249,21 +1214,21 @@ func (fs *ocFS) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCl // the file is supposed to be written. tmp, err := ioutil.TempFile(path.Dir(np), "._reva_atomic_upload") if err != nil { - return errors.Wrap(err, "ocFS: error creating tmp fn at "+path.Dir(np)) + return errors.Wrap(err, "ocfs: error creating tmp fn at "+path.Dir(np)) } defer os.RemoveAll(tmp.Name()) _, err = io.Copy(tmp, r) tmp.Close() if err != nil { - return errors.Wrap(err, "ocFS: error writing to tmp file "+tmp.Name()) + return errors.Wrap(err, "ocfs: error writing to tmp file "+tmp.Name()) } // if destination exists if _, err := os.Stat(np); err == nil { // copy attributes of existing file to tmp file if err := fs.copyMD(np, tmp.Name()); err != nil { - return errors.Wrap(err, "ocFS: error copying metadata from "+np+" to "+tmp.Name()) + return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+tmp.Name()) } // create revision if err := fs.archiveRevision(ctx, np); err != nil { @@ -1273,28 +1238,28 @@ func (fs *ocFS) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCl // TODO(jfd): make sure rename is atomic, missing fsync ... if err := os.Rename(tmp.Name(), np); err != nil { - return errors.Wrap(err, "ocFS: error renaming from "+tmp.Name()+" to "+np) + return errors.Wrap(err, "ocfs: error renaming from "+tmp.Name()+" to "+np) } return nil } -func (fs *ocFS) archiveRevision(ctx context.Context, np string) error { +func (fs *ocfs) archiveRevision(ctx context.Context, np string) error { // move existing file to versions dir vp := fmt.Sprintf("%s.v%d", fs.getVersionsPath(ctx, np), time.Now().Unix()) if err := os.MkdirAll(path.Dir(vp), 0700); err != nil { - return errors.Wrap(err, "ocFS: error creating versions dir "+vp) + return errors.Wrap(err, "ocfs: error creating versions dir "+vp) } // TODO(jfd): make sure rename is atomic, missing fsync ... if err := os.Rename(np, vp); err != nil { - return errors.Wrap(err, "ocFS: error renaming from "+np+" to "+vp) + return errors.Wrap(err, "ocfs: error renaming from "+np+" to "+vp) } return nil } -func (fs *ocFS) copyMD(s string, t string) (err error) { +func (fs *ocfs) copyMD(s string, t string) (err error) { var attrs []string if attrs, err = xattr.List(s); err != nil { return err @@ -1313,25 +1278,25 @@ func (fs *ocFS) copyMD(s string, t string) (err error) { return nil } -func (fs *ocFS) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { +func (fs *ocfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { np, err := fs.resolve(ctx, ref) if err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving reference") + return nil, errors.Wrap(err, "ocfs: error resolving reference") } r, err := os.Open(np) if err != nil { if os.IsNotExist(err) { - return nil, errtypes.NotFound(fs.removeNamespace(ctx, np)) + return nil, errtypes.NotFound(fs.unwrap(ctx, np)) } - return nil, errors.Wrap(err, "ocFS: error reading "+np) + return nil, errors.Wrap(err, "ocfs: error reading "+np) } return r, nil } -func (fs *ocFS) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { +func (fs *ocfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { np, err := fs.resolve(ctx, ref) if err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving reference") + return nil, errors.Wrap(err, "ocfs: error resolving reference") } vp := fs.getVersionsPath(ctx, np) @@ -1340,7 +1305,7 @@ func (fs *ocFS) ListRevisions(ctx context.Context, ref *provider.Reference) ([]* revisions := []*provider.FileVersion{} mds, err := ioutil.ReadDir(path.Dir(vp)) if err != nil { - return nil, errors.Wrap(err, "ocFS: error reading"+path.Dir(vp)) + return nil, errors.Wrap(err, "ocfs: error reading"+path.Dir(vp)) } for i := range mds { rev := fs.filterAsRevision(ctx, bn, mds[i]) @@ -1352,7 +1317,7 @@ func (fs *ocFS) ListRevisions(ctx context.Context, ref *provider.Reference) ([]* return revisions, nil } -func (fs *ocFS) filterAsRevision(ctx context.Context, bn string, md os.FileInfo) *provider.FileVersion { +func (fs *ocfs) filterAsRevision(ctx context.Context, bn string, md os.FileInfo) *provider.FileVersion { if strings.HasPrefix(md.Name(), bn) { // versions have filename.ext.v12345678 version := md.Name()[len(bn)+2:] // truncate ".v" to get version mtime @@ -1372,14 +1337,14 @@ func (fs *ocFS) filterAsRevision(ctx context.Context, bn string, md os.FileInfo) return nil } -func (fs *ocFS) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { +func (fs *ocfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { return nil, errtypes.NotSupported("download revision") } -func (fs *ocFS) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { +func (fs *ocfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { np, err := fs.resolve(ctx, ref) if err != nil { - return errors.Wrap(err, "ocFS: error resolving reference") + return errors.Wrap(err, "ocfs: error resolving reference") } vp := fs.getVersionsPath(ctx, np) rp := vp + ".v" + revisionKey @@ -1417,43 +1382,43 @@ func (fs *ocFS) RestoreRevision(ctx context.Context, ref *provider.Reference, re return err } -func (fs *ocFS) PurgeRecycleItem(ctx context.Context, key string) error { +func (fs *ocfs) PurgeRecycleItem(ctx context.Context, key string) error { rp, err := fs.getRecyclePath(ctx) if err != nil { - return errors.Wrap(err, "ocFS: error resolving recycle path") + return errors.Wrap(err, "ocfs: error resolving recycle path") } ip := path.Join(rp, path.Clean(key)) err = os.Remove(ip) if err != nil { - return errors.Wrap(err, "ocFS: error deleting recycle item") + return errors.Wrap(err, "ocfs: error deleting recycle item") } err = os.RemoveAll(path.Join(path.Dir(rp), "versions", path.Clean(key))) if err != nil { - return errors.Wrap(err, "ocFS: error deleting recycle item versions") + return errors.Wrap(err, "ocfs: error deleting recycle item versions") } // TODO delete keyfiles, keys, share-keys return nil } -func (fs *ocFS) EmptyRecycle(ctx context.Context) error { +func (fs *ocfs) EmptyRecycle(ctx context.Context) error { rp, err := fs.getRecyclePath(ctx) if err != nil { - return errors.Wrap(err, "ocFS: error resolving recycle path") + return errors.Wrap(err, "ocfs: error resolving recycle path") } err = os.RemoveAll(rp) if err != nil { - return errors.Wrap(err, "ocFS: error deleting recycle files") + return errors.Wrap(err, "ocfs: error deleting recycle files") } err = os.RemoveAll(path.Join(path.Dir(rp), "versions")) if err != nil { - return errors.Wrap(err, "ocFS: error deleting recycle files versions") + return errors.Wrap(err, "ocfs: error deleting recycle files versions") } // TODO delete keyfiles, keys, share-keys ... or just everything? return nil } -func (fs *ocFS) convertToRecycleItem(ctx context.Context, rp string, md os.FileInfo) *provider.RecycleItem { +func (fs *ocfs) convertToRecycleItem(ctx context.Context, rp string, md os.FileInfo) *provider.RecycleItem { // trashbin items have filename.ext.d12345678 suffix := path.Ext(md.Name()) if len(suffix) == 0 || !strings.HasPrefix(suffix, ".d") { @@ -1492,10 +1457,10 @@ func (fs *ocFS) convertToRecycleItem(ctx context.Context, rp string, md os.FileI } } -func (fs *ocFS) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { +func (fs *ocfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { rp, err := fs.getRecyclePath(ctx) if err != nil { - return nil, errors.Wrap(err, "ocFS: error resolving recycle path") + return nil, errors.Wrap(err, "ocfs: error resolving recycle path") } // list files folder @@ -1518,7 +1483,7 @@ func (fs *ocFS) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error return items, nil } -func (fs *ocFS) RestoreRecycleItem(ctx context.Context, key string) error { +func (fs *ocfs) RestoreRecycleItem(ctx context.Context, key string) error { log := appctx.GetLogger(ctx) u, ok := user.ContextGetUser(ctx) if !ok { @@ -1526,7 +1491,7 @@ func (fs *ocFS) RestoreRecycleItem(ctx context.Context, key string) error { } rp, err := fs.getRecyclePath(ctx) if err != nil { - return errors.Wrap(err, "ocFS: error resolving recycle path") + return errors.Wrap(err, "ocfs: error resolving recycle path") } src := path.Join(rp, path.Clean(key)) @@ -1542,11 +1507,11 @@ func (fs *ocFS) RestoreRecycleItem(ctx context.Context, key string) error { } else { origin = path.Clean(string(v)) } - tgt := path.Join(fs.getInternalPath(ctx, path.Join("/", u.GetUsername(), origin)), strings.TrimSuffix(path.Base(src), suffix)) + tgt := path.Join(fs.wrap(ctx, path.Join("/", u.GetUsername(), origin)), strings.TrimSuffix(path.Base(src), suffix)) // move back to original location if err := os.Rename(src, tgt); err != nil { log.Error().Err(err).Str("path", src).Msg("could not restore item") - return errors.Wrap(err, "ocFS: could not restore item") + return errors.Wrap(err, "ocfs: could not restore item") } // unset trash origin location in metadata if err := xattr.Remove(tgt, trashOriginPrefix); err != nil { diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go deleted file mode 100644 index 8fabe13e6b..0000000000 --- a/pkg/storage/helper/helper.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package helper - -import ( - "bytes" - "fmt" - "regexp" - "strconv" - "strings" - "text/template" - - "github.com/cs3org/reva/pkg/errtypes" - "github.com/pkg/errors" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" -) - -type layoutTemplate struct { - Username string //the username - UsernameLower string //the username in lowercase - UsernamePrefixCount string //first letters of username in lowercase eg: {{.UsernamePrefixCount.3}} will take the first 3 chars and make them lowercase, defaults to 1 - UsernameFirstLetter string //first letter of username in lowercase, equivalent as {{.UsernamePrefixCount.1}} but easy to read - Provider string //Provider/domain of user in lowercase -} - -// GetUserHomePath converts username into user's home path according to layout -func GetUserHomePath(u *userpb.User, layout string) (string, error) { - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - - usernameSplit := strings.Split(u.Username, "@") - if len(usernameSplit) == 1 { - usernameSplit = append(usernameSplit, "_unknown") - } - if usernameSplit[1] == "" { - usernameSplit[1] = "_unknown" - } - - // handle {{.UsernamePrefixCount.x}} - // where x is an int, pull it out and remove it from the go template - letters := 1 - reg := regexp.MustCompile(`\{\{\.UsernamePrefixCount\.[0-9]+\}\}`) - rmatches := reg.FindAllString(layout, -1) - if rmatches != nil { - reg := regexp.MustCompile("[^0-9]+") - f, _ := strconv.ParseInt(reg.ReplaceAllString(rmatches[0], ""), 10, 64) - if f > 1 { - letters = int(f) - } - layout = strings.Replace(layout, "{{.UsernamePrefixCount."+strconv.Itoa(letters)+"}}", "{{.UsernamePrefixCount}}", -1) - } - - pathTemplate := layoutTemplate{ - Username: u.Username, - UsernameLower: strings.ToLower(u.Username), - UsernamePrefixCount: strings.ToLower(string([]rune(usernameSplit[0])[0:letters])), - UsernameFirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), - Provider: strings.ToLower(usernameSplit[1]), - } - tmpl, err := template.New("userhomepath").Parse(layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error())) - } - buf := new(bytes.Buffer) - err = tmpl.Execute(buf, pathTemplate) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error())) - } - - return buf.String(), nil -} diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go deleted file mode 100644 index 6b81f2796c..0000000000 --- a/pkg/storage/pw/context/context.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package context - -import ( - "context" - "fmt" - "path" - "strings" - - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/helper" - "github.com/cs3org/reva/pkg/storage/pw/registry" - "github.com/cs3org/reva/pkg/user" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -func init() { - registry.Register("context", New) -} - -type config struct { - Prefix string `mapstructure:"prefix"` - Layout string `mapstructure:"layout"` -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{Layout: "{{.Username}}"} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -// New returns an implementation to of the storage.PathWrapper interface that -// is used to wrap and unwrap storage paths -func New(m map[string]interface{}) (storage.PathWrapper, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - return &pw{prefix: c.Prefix, layout: c.Layout}, nil -} - -type pw struct { - prefix string - layout string -} - -// Only works when a user is in context -func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userhome, err := helper.GetUserHomePath(u, pw.layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) - } - return path.Join("/", pw.prefix, userhome, rp), nil -} - -func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userhome, err := helper.GetUserHomePath(u, pw.layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) - } - return strings.TrimPrefix(rp, path.Join("/", userhome)), nil -} diff --git a/pkg/storage/pw/loader/loader.go b/pkg/storage/pw/loader/loader.go deleted file mode 100644 index b284bfda2f..0000000000 --- a/pkg/storage/pw/loader/loader.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package loader - -import ( - // Load core storage path wrapper backends. - _ "github.com/cs3org/reva/pkg/storage/pw/context" - // Add your own here -) diff --git a/pkg/storage/pw/registry/registry.go b/pkg/storage/pw/registry/registry.go deleted file mode 100644 index fe68ba03a6..0000000000 --- a/pkg/storage/pw/registry/registry.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package registry - -import "github.com/cs3org/reva/pkg/storage" - -// NewFunc is the function that storage implementations -// should register at init time. -type NewFunc func(map[string]interface{}) (storage.PathWrapper, error) - -// NewFuncs is a map containing all the registered storage backends. -var NewFuncs = map[string]NewFunc{} - -// Register registers a new storage backend new function. -// Not safe for concurrent use. Safe for use from package init. -func Register(name string, f NewFunc) { - NewFuncs[name] = f -} From 7db19c448e56cf32d544dd0f89a947dd296d3698 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 13 Feb 2020 15:56:41 +0100 Subject: [PATCH 04/14] simplify eos storage --- pkg/storage/fs/eos/eos.go | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 01bdb4bb8f..3cb8b5f3e1 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -55,10 +55,8 @@ func init() { var hiddenReg = regexp.MustCompile(`\.sys\..#.`) type eosStorage struct { - c *eosclient.Client - mountpoint string - showHiddenSys bool - conf *config + c *eosclient.Client + conf *config } func parseConfig(m map[string]interface{}) (*config, error) { @@ -108,9 +106,6 @@ type config struct { // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. UseKeytab bool `mapstrucuture:"use_keytab"` - // EnableHome enables the creation of home directories. - EnableHome bool `mapstructure:"enable_home"` - // SecProtocol specifies the xrootd security protocol to use between the server and EOS. SecProtocol string `mapstructure:"sec_protocol"` @@ -120,11 +115,14 @@ type config struct { // SingleUsername is the username to use when SingleUserMode is enabled SingleUsername string `mapstructure:"single_username"` + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` + // UserLayout wraps the internal path with user information. - // Example: if mountpoint is /eos/user and received path is /docs + // Example: if conf.Namespace is /eos/user and received path is /docs // and the UserLayout is {{.Username}} the internal path will be: // /eos/user//docs - UserLayout string `mapstructure:"layout"` + UserLayout string `mapstructure:"user_layout"` } func getUser(ctx context.Context) (*userpb.User, error) { @@ -194,10 +192,8 @@ func New(m map[string]interface{}) (storage.FS, error) { eosClient := eosclient.New(eosClientOpts) eosStorage := &eosStorage{ - c: eosClient, - mountpoint: c.Namespace, - showHiddenSys: c.ShowHiddenSysFiles, - conf: c, + c: eosClient, + conf: c, } return eosStorage, nil @@ -216,9 +212,9 @@ func (fs *eosStorage) wrap(ctx context.Context, fn string) (internal string) { panic(err) } layout := templates.WithUser(u, fs.conf.UserLayout) - internal = path.Join(fs.mountpoint, layout, fn) + internal = path.Join(fs.conf.Namespace, layout, fn) } else { - internal = path.Join(fs.mountpoint, fn) + internal = path.Join(fs.conf.Namespace, fn) } return } @@ -231,10 +227,10 @@ func (fs *eosStorage) unwrap(ctx context.Context, np string) (external string) { panic(err) } layout := templates.WithUser(u, fs.conf.UserLayout) - trim := path.Join(fs.mountpoint, layout) + trim := path.Join(fs.conf.Namespace, layout) external = strings.TrimPrefix(np, trim) } else { - external = strings.TrimPrefix(np, fs.mountpoint) + external = strings.TrimPrefix(np, fs.conf.Namespace) if external == "" { external = "/" } @@ -564,7 +560,7 @@ func (fs *eosStorage) ListFolder(ctx context.Context, ref *provider.Reference) ( finfos := []*provider.ResourceInfo{} for _, eosFileInfo := range eosFileInfos { // filter out sys files - if !fs.showHiddenSys { + if !fs.conf.ShowHiddenSysFiles { base := path.Base(eosFileInfo.File) if hiddenReg.MatchString(base) { continue @@ -862,7 +858,7 @@ func (fs *eosStorage) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, } recycleEntries := []*provider.RecycleItem{} for _, entry := range eosDeletedEntries { - if !fs.showHiddenSys { + if !fs.conf.ShowHiddenSysFiles { base := path.Base(entry.RestorePath) if hiddenReg.MatchString(base) { continue From fd600613559a66b93a5fb739a04a50d762652c20 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 13 Feb 2020 16:08:38 +0100 Subject: [PATCH 05/14] optimize struct --- pkg/storage/fs/eos/eos.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 3cb8b5f3e1..e18bc9931f 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -92,6 +92,21 @@ type config struct { // Defaults to os.TempDir() CacheDirectory string `mapstructure:"cache_directory"` + // SecProtocol specifies the xrootd security protocol to use between the server and EOS. + SecProtocol string `mapstructure:"sec_protocol"` + + // Keytab specifies the location of the keytab to use to authenticate to EOS. + Keytab string `mapstructure:"keytab"` + + // SingleUsername is the username to use when SingleUserMode is enabled + SingleUsername string `mapstructure:"single_username"` + + // UserLayout wraps the internal path with user information. + // Example: if conf.Namespace is /eos/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /eos/user//docs + UserLayout string `mapstructure:"user_layout"` + // Enables logging of the commands executed // Defaults to false EnableLogging bool `mapstructure:"enable_logging"` @@ -106,23 +121,8 @@ type config struct { // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. UseKeytab bool `mapstrucuture:"use_keytab"` - // SecProtocol specifies the xrootd security protocol to use between the server and EOS. - SecProtocol string `mapstructure:"sec_protocol"` - - // Keytab specifies the location of the keytab to use to authenticate to EOS. - Keytab string `mapstructure:"keytab"` - - // SingleUsername is the username to use when SingleUserMode is enabled - SingleUsername string `mapstructure:"single_username"` - // EnableHome enables the creation of home directories. EnableHome bool `mapstructure:"enable_home"` - - // UserLayout wraps the internal path with user information. - // Example: if conf.Namespace is /eos/user and received path is /docs - // and the UserLayout is {{.Username}} the internal path will be: - // /eos/user//docs - UserLayout string `mapstructure:"user_layout"` } func getUser(ctx context.Context) (*userpb.User, error) { From 43b67bf2188e59f6ef75e6f27303bb17afc065cd Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Mon, 17 Feb 2020 11:50:17 +1100 Subject: [PATCH 06/14] Add provider and lowercase --- pkg/storage/templates/templates.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index 5884276b21..21a84850af 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -39,6 +39,8 @@ import ( // For example {{.Username}} or {{.Id.Idp}} type UserData struct { *userpb.User + UsernameLower string + Provider string } // WithUser generates a layout based on user data. @@ -52,7 +54,7 @@ func WithUser(u *userpb.User, tpl string) string { panic(err) } b := bytes.Buffer{} - if err := t.Execute(&b, u); err != nil { + if err := t.Execute(&b, ut); err != nil { err := errors.Wrap(err, fmt.Sprintf("error executing template: user_template:%+v tpl:%s", ut, tpl)) panic(err) } @@ -60,8 +62,20 @@ func WithUser(u *userpb.User, tpl string) string { } func newUserData(u *userpb.User) *UserData { - ut := &UserData{User: u} - return ut + usernameSplit := strings.Split(u.Username, "@") + if len(usernameSplit) == 1 { + usernameSplit = append(usernameSplit, "_unknown") + } + if usernameSplit[1] == "" { + usernameSplit[1] = "_unknown" + } + + ut := &UserData{ + User: u, + UsernameLower: strings.ToLower(u.Username), + Provider: usernameSplit[1], + } + return ut } func clean(a string) string { From 59cab2bd888b17225b83aaddf720edda2c0bf418 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Mon, 17 Feb 2020 00:53:34 +0000 Subject: [PATCH 07/14] lint --- pkg/storage/templates/templates.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index 21a84850af..be345afd56 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -28,6 +28,7 @@ import ( "bytes" "fmt" "path" + "strings" "text/template" "github.com/Masterminds/sprig" @@ -39,8 +40,8 @@ import ( // For example {{.Username}} or {{.Id.Idp}} type UserData struct { *userpb.User - UsernameLower string - Provider string + UsernameLower string + Provider string } // WithUser generates a layout based on user data. @@ -74,8 +75,8 @@ func newUserData(u *userpb.User) *UserData { User: u, UsernameLower: strings.ToLower(u.Username), Provider: usernameSplit[1], - } - return ut + } + return ut } func clean(a string) string { From 18ac3318a88a898bdf371c297aa0c749a0668917 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Mon, 17 Feb 2020 00:53:55 +0000 Subject: [PATCH 08/14] add tests for provider and lowercase --- go.mod | 2 ++ pkg/storage/templates/templates_test.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/go.mod b/go.mod index 17521ad320..3e69a06bd9 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/rs/zerolog v1.17.2 go.opencensus.io v0.22.3 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a + golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 google.golang.org/grpc v1.27.1 @@ -43,6 +44,7 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.2.2 // indirect + honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect ) go 1.13 diff --git a/pkg/storage/templates/templates_test.go b/pkg/storage/templates/templates_test.go index 9743e5d132..3f39dcf271 100644 --- a/pkg/storage/templates/templates_test.go +++ b/pkg/storage/templates/templates_test.go @@ -62,6 +62,20 @@ var tests = []*testUnit{ }, template: "///{{.Username}}", }, + &testUnit{ + expected: "michael", + user: &userpb.User{ + Username: "MICHAEL", + }, + template: "{{.UsernameLower}}", + }, + &testUnit{ + expected: "somewhere.com/michael@somewhere.com", + user: &userpb.User{ + Username: "michael@somewhere.com", + }, + template: "{{.Provider}}/{{.Username}}", + }, } func TestLayout(t *testing.T) { From f043b6ed10dcd2f4399eeee37f3408f0426c0d1b Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Tue, 18 Feb 2020 22:44:34 +0000 Subject: [PATCH 09/14] removed UsernameLower in favor of built in lower function --- pkg/storage/templates/templates.go | 8 +++----- pkg/storage/templates/templates_test.go | 9 ++++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index be345afd56..806f49ab9d 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -40,8 +40,7 @@ import ( // For example {{.Username}} or {{.Id.Idp}} type UserData struct { *userpb.User - UsernameLower string - Provider string + Provider string } // WithUser generates a layout based on user data. @@ -72,9 +71,8 @@ func newUserData(u *userpb.User) *UserData { } ut := &UserData{ - User: u, - UsernameLower: strings.ToLower(u.Username), - Provider: usernameSplit[1], + User: u, + Provider: strings.ToLower(usernameSplit[1]), } return ut } diff --git a/pkg/storage/templates/templates_test.go b/pkg/storage/templates/templates_test.go index 3f39dcf271..2da0c5d66c 100644 --- a/pkg/storage/templates/templates_test.go +++ b/pkg/storage/templates/templates_test.go @@ -67,7 +67,7 @@ var tests = []*testUnit{ user: &userpb.User{ Username: "MICHAEL", }, - template: "{{.UsernameLower}}", + template: "{{lower .Username}}", }, &testUnit{ expected: "somewhere.com/michael@somewhere.com", @@ -76,6 +76,13 @@ var tests = []*testUnit{ }, template: "{{.Provider}}/{{.Username}}", }, + &testUnit{ + expected: "_unknown/michael", + user: &userpb.User{ + Username: "michael", + }, + template: "{{.Provider}}/{{.Username}}", + }, } func TestLayout(t *testing.T) { From bda44b86dbfa01db80453284279f68cbf52c971d Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Tue, 25 Feb 2020 22:44:39 +0000 Subject: [PATCH 10/14] Changed provider to Email.Domain, added Email.Local --- pkg/storage/templates/templates.go | 14 +++++++++++--- pkg/storage/templates/templates_test.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index 806f49ab9d..e502b232cd 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -40,7 +40,12 @@ import ( // For example {{.Username}} or {{.Id.Idp}} type UserData struct { *userpb.User - Provider string + Email EmailData +} + +type EmailData struct { + Local string + Domain string } // WithUser generates a layout based on user data. @@ -71,8 +76,11 @@ func newUserData(u *userpb.User) *UserData { } ut := &UserData{ - User: u, - Provider: strings.ToLower(usernameSplit[1]), + User: u, + Email: EmailData{ + Local: strings.ToLower(usernameSplit[0]), + Domain: strings.ToLower(usernameSplit[1]), + }, } return ut } diff --git a/pkg/storage/templates/templates_test.go b/pkg/storage/templates/templates_test.go index 2da0c5d66c..1e5c55ed59 100644 --- a/pkg/storage/templates/templates_test.go +++ b/pkg/storage/templates/templates_test.go @@ -74,14 +74,21 @@ var tests = []*testUnit{ user: &userpb.User{ Username: "michael@somewhere.com", }, - template: "{{.Provider}}/{{.Username}}", + template: "{{.Email.Domain}}/{{.Username}}", + }, + &testUnit{ + expected: "somewhere.com/michael", + user: &userpb.User{ + Username: "michael@somewhere.com", + }, + template: "{{.Email.Domain}}/{{.Email.Local}}", }, &testUnit{ expected: "_unknown/michael", user: &userpb.User{ Username: "michael", }, - template: "{{.Provider}}/{{.Username}}", + template: "{{.Email.Domain}}/{{.Username}}", }, } From b8aea62d80bdf6fd7a378e94ec56baff99cecad9 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 26 Feb 2020 13:58:03 +0100 Subject: [PATCH 11/14] clean go.mod --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 3e69a06bd9..17521ad320 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,6 @@ require ( github.com/rs/zerolog v1.17.2 go.opencensus.io v0.22.3 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a - golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 google.golang.org/grpc v1.27.1 @@ -44,7 +43,6 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.2.2 // indirect - honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect ) go 1.13 From 66bbf67bab4c19d41a8af5456b788188ce0c8880 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 26 Feb 2020 13:58:15 +0100 Subject: [PATCH 12/14] clean go.mod --- go.mod | 1 - go.sum | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/go.mod b/go.mod index 17521ad320..2d08930e44 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/fatih/color v1.7.0 // indirect github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.2.0+incompatible - github.com/gogo/protobuf v1.2.0 // indirect github.com/golang/protobuf v1.3.3 github.com/gomodule/redigo v2.0.0+incompatible github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 diff --git a/go.sum b/go.sum index ee3c98072a..c3bbd16db4 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzs github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.26.7 h1:ObjEnmzvSdYy8KVd3me7v/UMyCn81inLy2SyoIPoBkg= -github.com/aws/aws-sdk-go v1.26.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.28.2 h1:j5IXG9CdyLfcVfICqo1PXVv+rua+QQHbkXuvuU/JF+8= -github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.28.13 h1:JyCQQ86yil3hg7MtWdNH8Pbcgx92qlUV2v22Km63Mf4= github.com/aws/aws-sdk-go v1.28.13/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -32,15 +28,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cs3org/go-cs3apis v0.0.0-20200113082535-e58b99ae8f80 h1:ui+ztsFshmOv1yR3HRPaDRz1J/G6VQLd0ZUaBG33fWU= -github.com/cs3org/go-cs3apis v0.0.0-20200113082535-e58b99ae8f80/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20200113170445-f5790f408edc h1:qY/nHxpK3mJHkmWveEpwQCWHeNBjlDgk+o4t/5QeZM4= -github.com/cs3org/go-cs3apis v0.0.0-20200113170445-f5790f408edc/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20200115100126-824a5f718250 h1:N/WWs9OegcgFlsUo7/iahxq+e3luhZKu0B8wLrWBsTQ= github.com/cs3org/go-cs3apis v0.0.0-20200115100126-824a5f718250/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -64,7 +54,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= @@ -189,8 +178,6 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From e75dabcdf4a9330b54243044c3fb778d7f0965b6 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 26 Feb 2020 14:02:47 +0100 Subject: [PATCH 13/14] lint --- pkg/storage/templates/templates.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go index e502b232cd..df143a6323 100644 --- a/pkg/storage/templates/templates.go +++ b/pkg/storage/templates/templates.go @@ -43,6 +43,9 @@ type UserData struct { Email EmailData } +// EmailData contains mail data +// split into local and domain part. +// It is extracted from spliting the username by @. type EmailData struct { Local string Domain string From e0de9015900200daca663f8168f864e9efb4c152 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 26 Feb 2020 14:06:30 +0100 Subject: [PATCH 14/14] tidy deps --- go.sum | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go.sum b/go.sum index 52c99107cd..69617f4efc 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzs github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.28.13 h1:JyCQQ86yil3hg7MtWdNH8Pbcgx92qlUV2v22Km63Mf4= -github.com/aws/aws-sdk-go v1.28.13/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.29.5 h1:PddgnlgWgNI6x/weTnfk1fGYkhcs363gieDzK+Cf91Q= -github.com/aws/aws-sdk-go v1.29.5/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go v1.29.9 h1:PHq9ddjfZYfCOXyqHKiCZ1CHRAk7nXhV7WTqj5l+bmQ= github.com/aws/aws-sdk-go v1.29.9/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -206,8 +202,6 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=