Merge pull request #3551 from owncloud/ocis-init

[full-ci] ocis init & remove default secrets
owncloud · May 2, 2022 · 9934895 · 9934895
2 parents b013974 + f9dcf37
commit 9934895
Show file tree

Hide file tree

Showing 204 changed files with 2,720 additions and 757 deletions.
diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk
@@ -1,4 +1,4 @@
-# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.5.2. DO NOT EDIT.
+# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.6. DO NOT EDIT.
 # All tools are designed to be build inside $GOBIN.
 BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
 GOPATH ?= $(shell go env GOPATH)

diff --git a/.bingo/variables.env b/.bingo/variables.env
@@ -1,4 +1,4 @@
-# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.5.2. DO NOT EDIT.
+# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.6. DO NOT EDIT.
 # All tools are designed to be build inside $GOBIN.
 # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk.
 GOBIN=${GOBIN:=$(go env GOBIN)}

diff --git a/.drone.star b/.drone.star
@@ -1680,6 +1680,7 @@ def ocisServerWithAccounts(storage, accounts_hash_difficulty = 4, volumes = [],
             "detach": True,
             "environment": environment,
             "commands": [
+                "ocis/bin/ocis init --insecure true",
                 "ocis/bin/ocis server",
             ],
             "volumes": volumes,
@@ -1700,8 +1701,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
         user = "0:0"
         environment = {
             "OCIS_URL": "https://ocis-server:9200",
-            "GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
-            "STORAGE_HOME_DRIVER": "%s" % (storage),
+            "GATEWAY_GRPC_ADDR": "0.0.0.0:9142",  # cs3api-validator needs the cs3api gatway exposed
             "STORAGE_USERS_DRIVER": "%s" % (storage),
             "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
             "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
@@ -1712,8 +1712,8 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
             "OCIS_LOG_LEVEL": "error",
             "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
-            "OCIS_INSECURE": "true",
             "IDM_CREATE_DEMO_USERS": True,
+            "IDM_ADMIN_PASSWORD": "admin",  # override the random admin password from `ocis init`
         }
         wait_for_ocis = {
             "name": "wait-for-ocis-server",
@@ -1782,24 +1782,16 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "SHARING_USER_SQL_HOST": "oc10-db",
             "SHARING_USER_SQL_PORT": 3306,
             "SHARING_USER_SQL_NAME": "owncloud",
-            # ownCloud storage readonly
-            # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
-            "OCIS_STORAGE_READ_ONLY": "false",
             # General oCIS config
             # OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
             "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,nats,ocdav",
             "OCIS_LOG_LEVEL": "info",
             "OCIS_URL": OCIS_URL,
-            "PROXY_TLS": "true",
             "OCIS_BASE_DATA_PATH": "/mnt/data/ocis",
             "OCIS_CONFIG_DIR": "/etc/ocis",
-            # change default secrets
-            "OCIS_JWT_SECRET": "Pive-Fumkiu4",
-            "STORAGE_TRANSFER_SECRET": "replace-me-with-a-transfer-secret",
-            "OCIS_MACHINE_AUTH_API_KEY": "change-me-please",
-            "OCIS_INSECURE": "true",
             "PROXY_ENABLE_BASIC_AUTH": "true",
             "IDM_CREATE_DEMO_USERS": True,
+            "IDM_ADMIN_PASSWORD": "admin",  # override the random admin password from `ocis init`
         }
         wait_for_ocis = {
             "name": "wait-for-ocis-server",
@@ -1825,6 +1817,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "environment": environment,
             "user": user,
             "commands": [
+                "ocis/bin/ocis init --insecure true",
                 "ocis/bin/ocis server",
             ],
             "volumes": volumes,

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -19,9 +19,25 @@
                 "PROXY_ENABLE_BASIC_AUTH": "true",
                 // set insecure options because we don't have valid certificates in dev environments
                 "OCIS_INSECURE": "true",
+                // set some hardcoded secrets
+                "OCIS_JWT_SECRET": "some-ocis-jwt-secret",
+                "STORAGE_TRANSFER_SECRET": "some-ocis-transfer-secret",
+                "OCIS_MACHINE_AUTH_API_KEY": "some-ocis-machine-auth-api-key",
+                // idm ldap
+                "IDM_SVC_PASSWORD": "some-ldap-idm-password",
+                "GRAPH_LDAP_BIND_PASSWORD": "some-ldap-idm-password",
+                // reva ldap
+                "IDM_REVASVC_PASSWORD": "some-ldap-reva-password",
+                "GROUPS_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
+                "USERS_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
+                "AUTH_BASIC_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
+                // idp ldap
+                "IDM_IDPSVC_PASSWORD": "some-ldap-idp-password",
+                "IDP_LDAP_BIND_PASSWORD": "some-ldap-idp-password",
+                // admin user default password
+                "IDM_ADMIN_PASSWORD": "admin",
                 // demo users
-                "ACCOUNTS_DEMO_USERS_AND_GROUPS": "true",
-                "IDM_CREATE_DEMO_USERS": "true"
+                "IDM_CREATE_DEMO_USERS": "true",
                 // OCIS_RUN_EXTENSIONS allows to start a subset of extensions even in the supervised mode
                 //"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,glauth,graph,graph-explorer,idp,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,accounts,proxy,ocdav",
             }

diff --git a/changelog/unreleased/change-ocis-init.md b/changelog/unreleased/change-ocis-init.md
@@ -0,0 +1,10 @@
+Change: Introduce `ocis init` and remove all default secrets
+
+We've removed all default secrets. This means you can't start oCIS any longer
+without setting these via environment variable or configuration file.
+
+In order to make this easy for you, we introduced a new command: `ocis init`.
+You can run this command before starting oCIS with `ocis server` and it will
+bootstrap you a configuration file for a secure oCIS instance.
+
+https://github.com/owncloud/ocis/pull/3551
diff --git a/deployments/examples/ocis_traefik/.env b/deployments/examples/ocis_traefik/.env
@@ -2,10 +2,6 @@
 # It skips certificate validation for various parts of oCIS and is needed if you use self signed certificates.
 INSECURE=true
 
-# The demo users should not be created on a production instance
-# because their passwords are public
-DEMO_USERS=true
-
 ### Traefik settings ###
 # Serve Traefik dashboard. Defaults to "false".
 TRAEFIK_DASHBOARD=
@@ -21,16 +17,11 @@ TRAEFIK_ACME_MAIL=
 OCIS_DOCKER_TAG=
 # Domain of oCIS, where you can find the frontend. Defaults to "ocis.owncloud.test"
 OCIS_DOMAIN=
-# IDP LDAP bind password. Must be changed in order to have a secure oCIS. Defaults to "idp".
-IDP_LDAP_BIND_PASSWORD=
-# Storage LDAP bind password. Must be changed in order to have a secure oCIS. Defaults to "reva".
-STORAGE_LDAP_BIND_PASSWORD=
-# JWT secret which is used for the storage provider. Must be changed in order to have a secure oCIS. Defaults to "Pive-Fumkiu4"
-OCIS_JWT_SECRET=
-# JWT secret which is used for uploads to create transfer tokens. Must be changed in order to have a secure oCIS. Defaults to "replace-me-with-a-transfer-secret"
-STORAGE_TRANSFER_SECRET=
-# Machine auth api key secret. Must be changed in order to have a secure oCIS. Defaults to "change-me-please"
-OCIS_MACHINE_AUTH_API_KEY=
+# oCIS admin user password. Defaults to "admin".
+ADMIN_PASSWORD=
+# The demo users should not be created on a production instance
+# because their passwords are public. Defaults to "false".
+DEMO_USERS=
 
 # If you want to use debugging and tracing with this stack,
 # you need uncomment following line. Please see documentation at

diff --git a/deployments/examples/ocis_traefik/config/ocis/entrypoint-override.sh b/deployments/examples/ocis_traefik/config/ocis/entrypoint-override.sh
@@ -1,24 +1,5 @@
 #!/bin/sh
-
 set -e
 
-ocis server&
-sleep 10
-
-echo "##################################################"
-echo "change default secrets:"
-
-# IDP
-IDP_USER_UUID=$(ocis accounts list | grep "| Kopano IDP " | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' -o)
-echo "  IDP user UUID: $IDP_USER_UUID"
-ocis accounts update --password $IDP_LDAP_BIND_PASSWORD $IDP_USER_UUID
-
-# REVA
-REVA_USER_UUID=$(ocis accounts list | grep " | Reva Inter " | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' -o)
-echo "  Reva user UUID: $REVA_USER_UUID"
-ocis accounts update --password $STORAGE_LDAP_BIND_PASSWORD $REVA_USER_UUID
-
-echo "default secrets changed"
-echo "##################################################"
-
-wait # wait for oCIS to exit
+ocis init || true # will only initialize once
+ocis server
diff --git a/deployments/examples/ocis_traefik/docker-compose.yml b/deployments/examples/ocis_traefik/docker-compose.yml
@@ -53,21 +53,17 @@ services:
       OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
       OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
       PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
-      # change default secrets
-      IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
-      STORAGE_LDAP_BIND_PASSWORD: ${STORAGE_LDAP_BIND_PASSWORD:-reva}
-      OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4}
-      STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}
-      OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
       # INSECURE: needed if oCIS / Traefik is using self generated certificates
       OCIS_INSECURE: "${INSECURE:-false}"
       # basic auth (not recommended, but needed for eg. WebDav clients that do not support OpenID Connect)
       PROXY_ENABLE_BASIC_AUTH: "${PROXY_ENABLE_BASIC_AUTH:-false}"
+      # admin user password
+      IDM_ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}" # this overrides the admin password from the configuration file
       # demo users
-      ACCOUNTS_DEMO_USERS_AND_GROUPS: "${DEMO_USERS:-false}"  # deprecated, remove after switching to LibreIDM
       IDM_CREATE_DEMO_USERS: "${DEMO_USERS:-false}"
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
+      - ocis-config:/etc/ocis
       - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
@@ -82,7 +78,9 @@ services:
 
 volumes:
   certs:
+  ocis-config:
   ocis-data:
 
+
 networks:
   ocis-net:
diff --git a/docs/extensions/accounts/tests.md b/docs/extensions/accounts/tests.md
@@ -19,9 +19,10 @@ Make sure you've cloned the [web frontend repo](https://github.com/owncloud/web/
 {{< hint info >}}
 For now, an IDP configuration file gets generated once and will fail upon changing the oCIS url as done below. To avoid any clashes, remove this file before starting the tests:
 
-```
+```bash
 rm ~/.ocis/idp/identifier-registration.yaml
 ```
+
 {{< /hint >}}
 
 ### In the web repo
@@ -30,7 +31,7 @@ rm ~/.ocis/idp/identifier-registration.yaml
 
 Install dependencies and bundle the frontend with a watcher by running
 
-```
+```bash
 yarn && yarn build:w
 ```
 
@@ -40,7 +41,7 @@ If you skip the step above, the currently bundled frontend from the oCIS binary
 
 Start the necessary acceptance test services by using Docker (Compose):
 
-```
+```bash
 docker compose up selenium middleware-ocis vnc
 ```
 
@@ -50,21 +51,22 @@ docker compose up selenium middleware-ocis vnc
 
 Navigate into the accounts service via `cd ../accounts/` and install dependencies and build the bundled accounts UI with a watcher by running
 
-```
+```bash
 yarn && yarn watch
 ```
 
 #### Start oCIS from binary
 
 Navigate into the oCIS directory inside the oCIS repository and build the oCIS binary by running
 
-```
+```bash
 make clean build
 ```
 
 Then, start oCIS from the binary via
 
-```
+```bash
+./bin/ocis init
 OCIS_URL=https://host.docker.internal:9200 OCIS_INSECURE=true PROXY_ENABLE_BASIC_AUTH=true WEB_UI_CONFIG=../../web/dev/docker/ocis.web.config.json ./bin/ocis server
 ```
 
@@ -78,6 +80,6 @@ If you want visual feedback on the test run, visit http://host.docker.internal:6
 
 Navigate into the accounts service via `cd ../accounts/` and start the acceptance tests by running
 
-```
+```bash
 SERVER_HOST=https://host.docker.internal:9200 BACKEND_HOST=https://host.docker.internal:9200 RUN_ON_OCIS=true NODE_TLS_REJECT_UNAUTHORIZED=0 WEB_PATH=../../web WEB_UI_CONFIG=../../web/tests/drone/config-ocis.json MIDDLEWARE_HOST=http://host.docker.internal:3000 ./ui/tests/run-acceptance-test.sh ./ui/tests/acceptance/features/
 ```
diff --git a/docs/extensions/settings/tests.md b/docs/extensions/settings/tests.md
@@ -19,9 +19,10 @@ Make sure you've cloned the [web frontend repo](https://github.com/owncloud/web/
 {{< hint info >}}
 For now, an IDP configuration file gets generated once and will fail upon changing the oCIS url as done below. To avoid any clashes, remove this file before starting the tests:
 
-```
+```bash
 rm ~/.ocis/idp/identifier-registration.yaml
 ```
+
 {{< /hint >}}
 
 ### In the web repo
@@ -30,7 +31,7 @@ rm ~/.ocis/idp/identifier-registration.yaml
 
 Install dependencies and bundle the frontend with a watcher by running
 
-```
+```bash
 yarn && yarn build:w
 ```
 
@@ -40,7 +41,7 @@ If you skip the step above, the currently bundled frontend from the oCIS binary
 
 Start the necessary acceptance test services by using Docker (Compose):
 
-```
+```bash
 docker compose up selenium middleware-ocis vnc
 ```
 
@@ -50,21 +51,22 @@ docker compose up selenium middleware-ocis vnc
 
 Navigate into the settings service via `cd ../settings/` and install dependencies and build the bundled settings UI with a watcher by running
 
-```
+```bash
 yarn && yarn watch
 ```
 
 #### Start oCIS from binary
 
 Navigate into the oCIS directory inside the oCIS repository and build the oCIS binary by running
 
-```
+```bash
 make clean build
 ```
 
 Then, start oCIS from the binary via
 
-```
+```bash
+ocis init
 OCIS_URL=https://host.docker.internal:9200 OCIS_INSECURE=true PROXY_ENABLE_BASIC_AUTH=true WEB_UI_CONFIG=../../web/dev/docker/ocis.web.config.json ./bin/ocis server
 ```
 
@@ -78,6 +80,6 @@ If you want visual feedback on the test run, visit http://host.docker.internal:6
 
 Navigate into the settings service via `cd ../settings/` and start the acceptance tests by running
 
-```
+```bash
 SERVER_HOST=https://host.docker.internal:9200 BACKEND_HOST=https://host.docker.internal:9200 RUN_ON_OCIS=true NODE_TLS_REJECT_UNAUTHORIZED=0 WEB_PATH=../../web WEB_UI_CONFIG=../../web/tests/drone/config-ocis.json MIDDLEWARE_HOST=http://host.docker.internal:3000 ./ui/tests/run-acceptance-test.sh ./ui/tests/acceptance/features/
 ```
diff --git a/docs/helpers/example-config-generator.go.tmpl b/docs/helpers/example-config-generator.go.tmpl
@@ -22,9 +22,7 @@ func main() {
 	    {{- range $key, $value := .}}
 		replacer.Replace("{{$value}}"): func() string {
 			fmt.Println("Generating example YAML config for {{ $value -}}")
-			c := pkg{{$key}}.DefaultConfig()
-			pkg{{$key}}.EnsureDefaults(c)
-			pkg{{$key}}.Sanitize(c)
+			c := pkg{{$key}}.FullDefaultConfig()
 			yml, err := yaml.Marshal(c)
 			if err != nil {
 				log.Fatalf("Marshalling yaml for pkg0 failed: %s\n", err)
@@ -50,4 +48,3 @@ func main() {
 		}
 	}
 }
-
diff --git a/docs/ocis/deployment/_index.md b/docs/ocis/deployment/_index.md
@@ -27,38 +27,7 @@ oCIS deployments are super simple, yet there are many configurations possible fo
 
 ## Secure an oCIS instance
 
-### Change default secrets
-oCIS uses two system users which are needed for being operational:
-- Reva Inter Operability Platform (bc596f3c-c955-4328-80a0-60d018b4ad57)
-- Kopano IDP (820ba2a1-3f54-4538-80a4-2d73007e30bf)
+oCIS no longer has any default secrets in versions later than oCIS 1.20.0. Therefore you're no
+longer able to start oCIS without generating / setting all needed secrets.
 
-Both have simple default passwords which need to be changed. Currently, changing a password is only possible on the command line. You need to run `ocis accounts update --password <new-password> <id>` for both users.
-
-The new password for the Reva Inter Operability Platform user must be made available to oCIS by using the environment variable `STORAGE_LDAP_BIND_PASSWORD`. The same applies to the new Kopano IDP user password, which needs to be made available to oCIS in `IDP_LDAP_BIND_PASSWORD`.
-
-Furthermore, oCIS uses a shared secret to sign JWT tokens for inter service authorization, which also needs to be changed by the user.
-You can change it by setting the `OCIS_JWT_SECRET` environment variable for oCIS to a random string.
-
-Another is used secret for singing JWT tokens for uploads and downloads, which also needs to be changed by the user.
-You can change it by setting the `STORAGE_TRANSFER_SECRET` environment variable for oCIS to a random string.
-
-One more secret is used for machine auth, so that external applications can authenticate with an API key.
-You can change it by setting the `OCIS_MACHINE_AUTH_API_KEY` environment variable for oCIS to a random string.
-
-### Delete demo users
-
-{{< hint info >}}
-Before deleting the demo users mentioned below, you must create a new account for yourself and assign it to the administrator role.
-
-By default, oCIS doesn't create any demo users. During the first startup, it generates only the admin and one user for IDP and Reva respectively.
-{{< /hint >}}
-
-oCIS ships with a few demo users besides the system users:
-- Admin (ddc2004c-0977-11eb-9d3f-a793888cd0f8)
-- Albert Einstein (4c510ada-c86b-4815-8820-42cdf82c3d51)
-- Richard Feynman (932b4540-8d16-481e-8ef4-588e4b6b151c)
-- Maurice Moss (058bff95-6708-4fe5-91e4-9ea3d377588b)
-- Marie Curie (f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c)
-
-You can view them in ownCloud Web if you log in as Admin user or list them by running `ocis accounts list`.
-After adding your own user it is safe to delete the demo users in the web UI or with the command `ocis accounts remove <id>`. Please do not delete the system users (see [change default secrets]({{< ref "./#change-default-secrets" >}})) or oCIS will not function properly anymore.
+The recommended way is to use `ocis init` for that. It will generate a secure config file for you.