Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement quota fetching #3745

Closed
wants to merge 6 commits into from
Closed

implement quota fetching #3745

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0108042
implement quota fetching
butonic May 9, 2022
7971839
omit empty values, comment code
butonic May 9, 2022
396f0d6
prevent nil
butonic May 9, 2022
f9efeb2
disable quota checking in unit tests
butonic May 9, 2022
3378e9e
set quota definition to 'none' on missing total
butonic May 9, 2022
14b4a11
Merge branch 'master' into quota-fetching
butonic May 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions extensions/ocs/pkg/service/v0/data/user.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,22 @@ type Users struct {
// User holds the payload for a GetUser response
type User struct {
Enabled string `json:"enabled" xml:"enabled"`
UserID string `json:"id" xml:"id"`// UserID is mapped to the preferred_name attribute in accounts
UserID string `json:"id" xml:"id"` // UserID is mapped to the preferred_name attribute in accounts
DisplayName string `json:"display-name" xml:"display-name"`
LegacyDisplayName string `json:"displayname" xml:"displayname"`
Email string `json:"email" xml:"email"`
Quota *Quota `json:"quota" xml:"quota"`
Quota *Quota `json:"quota,omitempty" xml:"quota,omitempty"`
UIDNumber int64 `json:"uidnumber" xml:"uidnumber"`
GIDNumber int64 `json:"gidnumber" xml:"gidnumber"`
}

// Quota holds quota information
type Quota struct {
Free int64 `json:"free" xml:"free"`
Used int64 `json:"used" xml:"used"`
Total int64 `json:"total" xml:"total"`
Relative float32 `json:"relative" xml:"relative"`
Definition string `json:"definition" xml:"definition"`
Free int64 `json:"free,omitempty" xml:"free,omitempty"`
Used int64 `json:"used,omitempty" xml:"used,omitempty"`
Total int64 `json:"total,omitempty" xml:"total,omitempty"`
Relative float32 `json:"relative,omitempty" xml:"relative,omitempty"`
Definition string `json:"definition,omitempty" xml:"definition,omitempty"`
}

// SigningKey holds the Payload for a GetSigningKey response
Expand Down
126 changes: 113 additions & 13 deletions extensions/ocs/pkg/service/v0/users.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,14 @@ import (

storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0"
storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
"google.golang.org/grpc/metadata"

cs3 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
cs3gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
cs3identity "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
cs3rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
cs3storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/go-chi/chi/v5"
"github.com/go-micro/plugins/v4/client/grpc"
"github.com/owncloud/ocis/v2/extensions/ocs/pkg/service/v0/data"
Expand Down Expand Up @@ -48,12 +53,20 @@ func (o Ocs) GetUser(w http.ResponseWriter, r *http.Request) {
o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
}

var user *cs3.User
currentUser, ok := revactx.ContextGetUser(r.Context())
if !ok {
response.ErrRender(data.MetaServerError.StatusCode, "missing user in context")
return
}

var user *cs3identity.User
switch {
case userid == "":
o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "missing user in context"))
o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "missing username"))
case userid == currentUser.Username:
user = currentUser
case o.config.AccountBackend == "cs3":
user, err = o.fetchAccountFromCS3Backend(r.Context(), userid)
user, err = o.fetchUserFromCS3Backend(r.Context(), userid)
default:
o.logger.Fatal().Msgf("Invalid accounts backend type '%s'", o.config.AccountBackend)
}
Expand All @@ -79,14 +92,12 @@ func (o Ocs) GetUser(w http.ResponseWriter, r *http.Request) {
UIDNumber: user.UidNumber,
GIDNumber: user.GidNumber,
Enabled: "true", // TODO include in response only when admin?
// TODO query storage registry for free space? of home storage, maybe...
Quota: &data.Quota{
Free: 2840756224000,
Used: 5059416668,
Total: 2845815640668,
Relative: 0.18,
Definition: "default",
},
Quota: &data.Quota{},
}

// lightweight and federated users don't have access to their storage space
if currentUser.Id.Type != cs3identity.UserType_USER_TYPE_LIGHTWEIGHT && currentUser.Id.Type != cs3identity.UserType_USER_TYPE_FEDERATED {
o.fillPersonalQuota(r.Context(), d, user)
}

_, span := ocstracing.TraceProvider.
Expand All @@ -97,6 +108,95 @@ func (o Ocs) GetUser(w http.ResponseWriter, r *http.Request) {
o.mustRender(w, r, response.DataRender(d))
}

func (o Ocs) fillPersonalQuota(ctx context.Context, d *data.User, u *cs3identity.User) {

gc, err := pool.GetGatewayServiceClient(o.config.Reva.Address)
if err != nil {
o.logger.Error().Err(err).Msg("error getting gateway client")
return
}

aRes, err := gc.Authenticate(ctx, &cs3gateway.AuthenticateRequest{
Type: "machine",
ClientId: "userid:" + u.Id.OpaqueId,
ClientSecret: o.config.MachineAuthAPIKey,
})

switch {
case err != nil:
o.logger.Error().Err(err).Msg("could not fill personal quota")
return
case aRes.Status.Code != cs3rpc.Code_CODE_OK:
o.logger.Error().Interface("status", aRes.Status).Msg("could not fill personal quota")
}

ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, aRes.Token)

res, err := gc.ListStorageSpaces(ctx, &cs3storage.ListStorageSpacesRequest{
Filters: []*cs3storage.ListStorageSpacesRequest_Filter{
{
Type: cs3storage.ListStorageSpacesRequest_Filter_TYPE_OWNER,
Term: &cs3storage.ListStorageSpacesRequest_Filter_Owner{
Owner: aRes.User.Id,
},
},
{
Type: cs3storage.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE,
Term: &cs3storage.ListStorageSpacesRequest_Filter_SpaceType{
SpaceType: "personal",
},
},
},
})
if err != nil {
o.logger.Error().Err(err).Msg("error calling ListStorageSpaces")
return
}
if res.Status.Code != cs3rpc.Code_CODE_OK {
o.logger.Debug().Interface("status", res.Status).Msg("ListStorageSpaces returned non OK result")
return
}

if len(res.StorageSpaces) == 0 {
o.logger.Debug().Err(err).Msg("list spaces returned empty list")
return
}

getQuotaRes, err := gc.GetQuota(ctx, &cs3gateway.GetQuotaRequest{Ref: &cs3storage.Reference{
ResourceId: res.StorageSpaces[0].Root,
Path: ".",
}})
if err != nil {
o.logger.Error().Err(err).Msg("error calling GetQuota")
return
}
if res.Status.Code != cs3rpc.Code_CODE_OK {
o.logger.Debug().Interface("status", res.Status).Msg("GetQuota returned non OK result")
return
}

total := getQuotaRes.TotalBytes
used := getQuotaRes.UsedBytes

d.Quota = &data.Quota{
Used: int64(used),
// TODO support negative values or flags for the quota to carry special meaning: -1 = uncalculated, -2 = unknown, -3 = unlimited
// for now we can only report total and used
Total: int64(total),
Comment on lines +183 to +185
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the oc10 code has this: https://github.com/owncloud/core/blob/817f54f53d64249166feaa0eb9231c50ca1ed7e0/lib/public/Files/FileInfo.php#L44-L58

it is returned for quota-available-bytes in propfind requests:

 $ curl https://demo.owncloud.com/remote.php/webdav/ -udemo:demo -X PROPFIND -d '<?xml version="1.0" ?><d:propfind xmlns:d="DAV:"><d:prop><d:quota-available-bytes/><d:quota-used-bytes/></d:prop></d:propfind>' -v --trace-time | xmllint -format -

...

  <d:response>
    <d:href>/remote.php/webdav/Photos/</d:href>
    <d:propstat>
      <d:prop>
        <d:quota-available-bytes>-3</d:quota-available-bytes>
        <d:quota-used-bytes>1011464</d:quota-used-bytes>
      </d:prop>
      <d:status>HTTP/1.1 200 OK</d:status>
    </d:propstat>
  </d:response>
...

the ocs api only returns quota on the /ocs/v[1|2].php/cloud/users/{username} endpoint:

$ curl https://demo.owncloud.com/ocs/v1.php/cloud/users/demo?format=json -u demo:demo -v | jq

...

      "quota": {
        "free": 27273637888,
        "used": 7716359,
        "total": 27281354247,
        "relative": 0.03,
        "definition": "none"
      },

...

but it always returns the actually available bytes, not -3 ...

// we cannot differentiate between `default` or a human readable `1 GB` defanation.
// The web ui can create a human readable string from the actual total if it is sot. Otherwise it has to leave out relative and total anyway.
// Definition: "default",
}

// only calculate free and relative when total is available
if total > 0 {
d.Quota.Free = int64(total - used)
d.Quota.Relative = float32(float64(used) / float64(total))
} else {
d.Quota.Definition = "none" // this indicates no quota / unlimited to the ui
}
}

// AddUser creates a new user account
func (o Ocs) AddUser(w http.ResponseWriter, r *http.Request) {
switch o.config.AccountBackend {
Expand Down Expand Up @@ -241,7 +341,7 @@ func escapeValue(value string) string {
return strings.ReplaceAll(value, "'", "''")
}

func (o Ocs) fetchAccountFromCS3Backend(ctx context.Context, name string) (*cs3.User, error) {
func (o Ocs) fetchUserFromCS3Backend(ctx context.Context, name string) (*cs3identity.User, error) {
backend := o.getCS3Backend()
u, _, err := backend.GetUserByClaims(ctx, "username", name, false)
if err != nil {
Expand Down