Skip to content

Commit

Permalink
Merge pull request #3569 from owncloud/storage-config
Browse files Browse the repository at this point in the history
[full-ci] Refactor storage configuration to make it similar to other oCIS configuration
  • Loading branch information
David Christofas authored Apr 26, 2022
2 parents 485fa6d + e2ea74c commit 4b65e6d
Show file tree
Hide file tree
Showing 77 changed files with 4,359 additions and 1,525 deletions.
82 changes: 40 additions & 42 deletions .drone.star
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,19 @@ config = {
"suites": [
"apiShareManagement",
],
"skip": False,
# The tests fail after the storage config changes
# They will be fixed later.
"skip": True,
"earlyFail": True,
"cron": "nightly",
},
"apiWebdav": {
"suites": [
"apiWebdavOperations",
],
"skip": False,
# The tests fail after the storage config changes
# They will be fixed later.
"skip": True,
"earlyFail": True,
"cron": "nightly",
},
Expand Down Expand Up @@ -1624,8 +1628,8 @@ def ocisServerWithIdp():
"GRAPH_LDAP_SERVER_WRITE_ENABLED": "true",
"LDAP_URI": "ldaps://0.0.0.0:9235",
"LDAP_INSECURE": "true",
"LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
"LDAP_BIND_PASSWORD": "idm",
"GRAPH_LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
"GRAPH_LDAP_BIND_PASSWORD": "idm",
"LDAP_USER_BASE_DN": "ou=users,o=libregraph-idm",
"LDAP_USER_SCHEMA_ID": "ownclouduuid",
"LDAP_USER_SCHEMA_MAIL": "mail",
Expand All @@ -1644,8 +1648,8 @@ def ocisServerWithIdp():
"IDP_LDAP_LOGIN_ATTRIBUTE": "uid",
"PROXY_ACCOUNT_BACKEND_TYPE": "cs3",
"PROXY_ENABLE_BASIC_AUTH": "true",
"STORAGE_LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
"STORAGE_LDAP_BIND_PASSWORD": "reva",
"LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
"LDAP_BIND_PASSWORD": "reva",
"OCS_ACCOUNT_BACKEND_TYPE": "cs3",
"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,idm,ocdav",
"OCIS_LOG_LEVEL": "error",
Expand Down Expand Up @@ -1679,13 +1683,13 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
user = "0:0"
environment = {
"OCIS_URL": "https://ocis-server:9200",
"STORAGE_GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"STORAGE_HOME_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
"STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"PROXY_ENABLE_BASIC_AUTH": True,
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
"IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
Expand All @@ -1708,42 +1712,38 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
environment = {
# Keycloak IDP specific configuration
"PROXY_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud",
"LDAP_IDP": "https://keycloak/auth/realms/owncloud",
"WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud",
"WEB_OIDC_CLIENT_ID": "ocis-web",
"WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration",
"STORAGE_OIDC_ISSUER": "https://keycloak",
"STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud",
"AUTH_BEARER_OIDC_ISSUER": "https://keycloak",
"WEB_OIDC_SCOPE": "openid profile email owncloud",
# LDAP bind
"STORAGE_LDAP_URI": "ldaps://openldap",
"STORAGE_LDAP_INSECURE": "true",
"STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
"STORAGE_LDAP_BIND_PASSWORD": "admin",
"LDAP_URI": "ldaps://openldap",
"LDAP_INSECURE": "true",
"LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
"LDAP_BIND_PASSWORD": "admin",
# LDAP user settings
"PROXY_AUTOPROVISION_ACCOUNTS": "true", # automatically create users when they login
"PROXY_ACCOUNT_BACKEND_TYPE": "cs3", # proxy should get users from CS3APIS (which gets it from LDAP)
"PROXY_USER_OIDC_CLAIM": "ocis.user.uuid", # claim was added in Keycloak
"PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID
"STORAGE_LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
"STORAGE_LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
"STORAGE_LDAP_GROUPFILTER": "(objectclass=owncloud)",
"STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
"STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber",
"STORAGE_LDAP_GROUP_SCHEMA_ID": "cn",
"STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail",
"STORAGE_LDAP_GROUP_SCHEMA_MEMBER": "cn",
"STORAGE_LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
"STORAGE_LDAP_USER_OBJECTCLASS": "posixAccount",
"STORAGE_LDAP_USERFILTER": "(objectclass=owncloud)",
"STORAGE_LDAP_USER_SCHEMA_USERNAME": "cn",
"STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
"STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber",
"STORAGE_LDAP_USER_SCHEMA_MAIL": "mail",
"STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber",
"STORAGE_LDAP_USER_SCHEMA_ID": "ownclouduuid",
"STORAGE_LDAP_LOGIN_ATTRIBUTES": "uid,mail",
"LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
"LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
"LDAP_GROUPFILTER": "(objectclass=owncloud)",
"LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
"LDAP_GROUP_SCHEMA_ID": "cn",
"LDAP_GROUP_SCHEMA_MAIL": "mail",
"LDAP_GROUP_SCHEMA_MEMBER": "cn",
"LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
"LDAP_USER_OBJECTCLASS": "posixAccount",
"LDAP_USERFILTER": "(objectclass=owncloud)",
"LDAP_USER_SCHEMA_USERNAME": "cn",
"LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
"LDAP_USER_SCHEMA_MAIL": "mail",
"LDAP_USER_SCHEMA_ID": "ownclouduuid",
"LDAP_LOGIN_ATTRIBUTES": "uid,mail",
# ownCloudSQL storage driver
"STORAGE_HOME_DRIVER": "owncloudsql",
"STORAGE_USERS_DRIVER": "owncloudsql",
"STORAGE_METADATA_DRIVER": "ocis",
"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files",
Expand All @@ -1758,19 +1758,19 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
# TODO: redis is not yet supported
"STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379",
# ownCloudSQL sharing driver
"STORAGE_SHARING_USER_DRIVER": "owncloudsql",
"STORAGE_SHARING_USER_SQL_USERNAME": "owncloud",
"STORAGE_SHARING_USER_SQL_PASSWORD": "owncloud",
"STORAGE_SHARING_USER_SQL_HOST": "oc10-db",
"STORAGE_SHARING_USER_SQL_PORT": 3306,
"STORAGE_SHARING_USER_SQL_NAME": "owncloud",
"SHARING_USER_DRIVER": "owncloudsql",
"SHARING_USER_SQL_USERNAME": "owncloud",
"SHARING_USER_SQL_PASSWORD": "owncloud",
"SHARING_USER_SQL_HOST": "oc10-db",
"SHARING_USER_SQL_PORT": 3306,
"SHARING_USER_SQL_NAME": "owncloud",
# ownCloud storage readonly
# TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
"OCIS_STORAGE_READ_ONLY": "false",
# General oCIS config
# OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,nats,ocdav",
"OCIS_LOG_LEVEL": "error",
"OCIS_LOG_LEVEL": "info",
"OCIS_URL": OCIS_URL,
"PROXY_TLS": "true",
"OCIS_BASE_DATA_PATH": "/mnt/data/ocis",
Expand All @@ -1780,8 +1780,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
"OCIS_MACHINE_AUTH_API_KEY": "change-me-please",
"OCIS_INSECURE": "true",
"PROXY_ENABLE_BASIC_AUTH": "true",
"ACCOUNTS_DEMO_USERS_AND_GROUPS": True, # deprecated, remove after switching to LibreIDM
"IDM_CREATE_DEMO_USERS": True,
}
wait_for_ocis = {
"name": "wait-for-ocis-server",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ import (
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/appprovider/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/owncloud/ocis/ocis-pkg/tracing"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
Expand All @@ -23,12 +24,15 @@ func AppProvider(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "app-provider",
Usage: "start appprovider for providing apps",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-app-provider")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
logCfg := cfg.Logging
logger := log.NewLogger(
log.Level(logCfg.Level),
log.File(logCfg.File),
log.Pretty(logCfg.Pretty),
log.Color(logCfg.Color),
)
tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -51,10 +55,12 @@ func AppProvider(cfg *config.Config) *cli.Command {

debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AppProvider.DebugAddr),
debug.Addr(cfg.Debug.Addr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
debug.Pprof(cfg.Debug.Pprof),
debug.Zpages(cfg.Debug.Zpages),
debug.Token(cfg.Debug.Token),
)

if err != nil {
Expand All @@ -66,7 +72,7 @@ func AppProvider(cfg *config.Config) *cli.Command {
cancel()
})

if !cfg.Reva.AppProvider.Supervised {
if !cfg.Supervised {
sync.Trap(&gr, cancel)
}

Expand All @@ -80,38 +86,36 @@ func appProviderConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]

rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AppProvider.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
"jwt_secret": cfg.JWTSecret,
"gatewaysvc": cfg.GatewayEndpoint,
"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AppProvider.GRPCNetwork,
"address": cfg.Reva.AppProvider.GRPCAddr,
"network": cfg.GRPC.Protocol,
"address": cfg.GRPC.Addr,
// TODO build services dynamically
"services": map[string]interface{}{
"appprovider": map[string]interface{}{
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"app_provider_url": cfg.Reva.AppProvider.ExternalAddr,
"driver": cfg.Reva.AppProvider.Driver,
"app_provider_url": cfg.ExternalAddr,
"driver": cfg.Driver,
"drivers": map[string]interface{}{
"wopi": map[string]interface{}{
"app_api_key": cfg.Reva.AppProvider.WopiDriver.AppAPIKey,
"app_desktop_only": cfg.Reva.AppProvider.WopiDriver.AppDesktopOnly,
"app_icon_uri": cfg.Reva.AppProvider.WopiDriver.AppIconURI,
"app_int_url": cfg.Reva.AppProvider.WopiDriver.AppInternalURL,
"app_name": cfg.Reva.AppProvider.WopiDriver.AppName,
"app_url": cfg.Reva.AppProvider.WopiDriver.AppURL,
"insecure_connections": cfg.Reva.AppProvider.WopiDriver.Insecure,
"iop_secret": cfg.Reva.AppProvider.WopiDriver.IopSecret,
"jwt_secret": cfg.Reva.AppProvider.WopiDriver.JWTSecret,
"wopi_url": cfg.Reva.AppProvider.WopiDriver.WopiURL,
"app_api_key": cfg.Drivers.WOPI.AppAPIKey,
"app_desktop_only": cfg.Drivers.WOPI.AppDesktopOnly,
"app_icon_uri": cfg.Drivers.WOPI.AppIconURI,
"app_int_url": cfg.Drivers.WOPI.AppInternalURL,
"app_name": cfg.Drivers.WOPI.AppName,
"app_url": cfg.Drivers.WOPI.AppURL,
"insecure_connections": cfg.Drivers.WOPI.Insecure,
"iop_secret": cfg.Drivers.WOPI.IopSecret,
"jwt_secret": cfg.JWTSecret,
"wopi_url": cfg.Drivers.WOPI.WopiURL,
},
},
},
Expand All @@ -128,28 +132,28 @@ type AppProviderSutureService struct {

// NewAppProvider creates a new store.AppProviderSutureService
func NewAppProvider(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
cfg.AppProvider.Commons = cfg.Commons
return AppProviderSutureService{
cfg: cfg.Storage,
cfg: cfg.AppProvider,
}
}

func (s AppProviderSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AppProvider.Context = ctx
cmd := AppProvider(s.cfg)
f := &flag.FlagSet{}
cmdFlags := AppProvider(s.cfg).Flags
cmdFlags := cmd.Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AppProvider(s.cfg).Before != nil {
if err := AppProvider(s.cfg).Before(cliCtx); err != nil {
if cmd.Before != nil {
if err := cmd.Before(cliCtx); err != nil {
return err
}
}
if err := AppProvider(s.cfg).Action(cliCtx); err != nil {
if err := cmd.Action(cliCtx); err != nil {
return err
}

Expand Down
67 changes: 67 additions & 0 deletions extensions/appprovider/pkg/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package config

import "github.com/owncloud/ocis/ocis-pkg/shared"

type Config struct {
*shared.Commons `yaml:"-"`
Service Service `yaml:"-"`
Tracing *Tracing `yaml:"tracing"`
Logging *Logging `yaml:"log"`
Debug Debug `yaml:"debug"`
Supervised bool

GRPC GRPCConfig `yaml:"grpc"`

JWTSecret string
GatewayEndpoint string
SkipUserGroupsInToken bool
ExternalAddr string
Driver string
Drivers Drivers
}

type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;APP_PROVIDER_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;APP_PROVIDER_TRACING_TYPE"`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;APP_PROVIDER_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;APP_PROVIDER_TRACING_COLLECTOR"`
}

type Logging struct {
Level string `yaml:"level" env:"OCIS_LOG_LEVEL;APP_PROVIDER_LOG_LEVEL" desc:"The log level."`
Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;APP_PROVIDER_LOG_PRETTY" desc:"Activates pretty log output."`
Color bool `yaml:"color" env:"OCIS_LOG_COLOR;APP_PROVIDER_LOG_COLOR" desc:"Activates colorized log output."`
File string `yaml:"file" env:"OCIS_LOG_FILE;APP_PROVIDER_LOG_FILE" desc:"The target log file."`
}

type Service struct {
Name string `yaml:"-"`
}

type Debug struct {
Addr string `yaml:"addr" env:"APP_PROVIDER_DEBUG_ADDR"`
Token string `yaml:"token" env:"APP_PROVIDER_DEBUG_TOKEN"`
Pprof bool `yaml:"pprof" env:"APP_PROVIDER_DEBUG_PPROF"`
Zpages bool `yaml:"zpages" env:"APP_PROVIDER_DEBUG_ZPAGES"`
}

type GRPCConfig struct {
Addr string `yaml:"addr" env:"APP_PROVIDER_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"APP_PROVIDER_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}

type Drivers struct {
WOPI WOPIDriver
}

type WOPIDriver struct {
AppAPIKey string `yaml:"app_api_key"`
AppDesktopOnly bool `yaml:"app_desktop_only"`
AppIconURI string `yaml:"app_icon_uri"`
AppInternalURL string `yaml:"app_internal_url"`
AppName string `yaml:"app_name"`
AppURL string `yaml:"app_url"`
Insecure bool `yaml:"insecure"`
IopSecret string `yaml:"ipo_secret"`
WopiURL string `yaml:"wopi_url"`
}
Loading

0 comments on commit 4b65e6d

Please sign in to comment.