Merge pull request #2730 from owncloud/improve-tooling

improve tooling

.drone.star

@@ -255,7 +255,7 @@ def testPipelines(ctx):
     return pipelines
 
 def testOcisModule(ctx, module):
-    steps = skipIfUnchanged(ctx, "unit-tests") + makeGenerate(module) + [
+    steps = skipIfUnchanged(ctx, "unit-tests") + makeGoGenerate(module) + [
         {
             "name": "golangci-lint",
             "image": OC_CI_GOLANG,
@@ -327,7 +327,8 @@ def buildOcisBinaryForTesting(ctx):
             "arch": "amd64",
         },
         "steps": skipIfUnchanged(ctx, "acceptance-tests") +
-                 makeGenerate("") +
+                 makeNodeGenerate("") +
+                 makeGoGenerate("") +
                  build() +
                  rebuildBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis"),
         "trigger": {
@@ -622,7 +623,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
                     "cd /srv/app/web",
                     "git checkout $WEB_COMMITID",
                     "cp -r tests/acceptance/filesForUpload/* /uploads",
-                    "yarn install --frozen-lockfile",
+                    "yarn install --immutable",
                     "yarn build",
                     "./tests/acceptance/run.sh",
                 ],
@@ -685,10 +686,10 @@ def accountsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
                     "cd /srv/app/web",
                     "git checkout $WEB_COMMITID",
                     "cp -r tests/acceptance/filesForUpload/* /uploads",
-                    "yarn install --frozen-lockfile",
+                    "yarn install --immutable",
                     "yarn build",
                     "cd /drone/src/accounts",
-                    "yarn install --frozen-lockfile",
+                    "yarn install --immutable",
                     "make test-acceptance-webui",
                 ],
                 "volumes": [stepVolumeOC10Tests] +
@@ -749,9 +750,9 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
                     "cp -r /srv/app/web/tests/acceptance/filesForUpload/* /uploads",
                     "cd /srv/app/web",
                     "git checkout $WEB_COMMITID",
-                    "yarn install --frozen-lockfile",
+                    "yarn install --immutable",
                     "cd /drone/src/settings",
-                    "yarn install --frozen-lockfile",
+                    "yarn install --immutable",
                     "make test-acceptance-webui",
                 ],
                 "volumes": [stepVolumeOC10Tests] +
@@ -875,12 +876,13 @@ def dockerRelease(ctx, arch):
             "arch": arch,
         },
         "steps": skipIfUnchanged(ctx, "build-docker") +
-                 makeGenerate("") + [
+                 makeNodeGenerate("") +
+                 makeGoGenerate("") + [
             {
                 "name": "build",
                 "image": OC_CI_GOLANG,
                 "commands": [
-                    "make -C ocis release-linux-docker",
+                    "make -C ocis release-linux-docker-%s" % (arch),
                 ],
             },
             {
@@ -949,12 +951,13 @@ def dockerEos(ctx):
             "arch": "amd64",
         },
         "steps": skipIfUnchanged(ctx, "build-docker") +
-                 makeGenerate("") + [
+                 makeNodeGenerate("") +
+                 makeGoGenerate("") + [
             {
                 "name": "build",
                 "image": OC_CI_GOLANG,
                 "commands": [
-                    "make -C ocis release-linux-docker",
+                    "make -C ocis release-linux-docker-amd64",
                 ],
             },
             {
@@ -1052,7 +1055,8 @@ def binaryRelease(ctx, name):
             "arch": "amd64",
         },
         "steps": skipIfUnchanged(ctx, "build-binary") +
-                 makeGenerate("") + [
+                 makeNodeGenerate("") +
+                 makeGoGenerate("") + [
             {
                 "name": "build",
                 "image": OC_CI_GOLANG,
@@ -1395,7 +1399,7 @@ def docs(ctx):
         },
     }
 
-def makeGenerate(module):
+def makeNodeGenerate(module):
     if module == "":
         make = "make"
     else:
@@ -1409,6 +1413,14 @@ def makeGenerate(module):
             ],
             "volumes": [stepVolumeGo],
         },
+    ]
+
+def makeGoGenerate(module):
+    if module == "":
+        make = "make"
+    else:
+        make = "make -C %s" % (module)
+    return [
         {
             "name": "generate go",
             "image": OC_CI_GOLANG,

.make/docs.mk

@@ -1,7 +1,12 @@
+
+SKIP_CONFIG_DOCS_GENERATE ?= 0
+CONFIG_DOCS_BASE_PATH ?= ../docs/extensions
+
 .PHONY: config-docs-generate
 config-docs-generate: $(FLAEX)
-	@echo "$(NAME): generating config docs"
-	@$(FLAEX) >| ../docs/extensions/$(NAME)/configuration.md
+	@if [ $(SKIP_CONFIG_DOCS_GENERATE) -ne 1 ]; then \
+		$(FLAEX) >| $(CONFIG_DOCS_BASE_PATH)/$(NAME)/configuration.md \
+	; fi;
 
 .PHONY: grpc-docs-generate
 grpc-docs-generate: buf-generate

.make/l10n.mk

@@ -12,7 +12,7 @@ OUTPUT_DIR = ./l10n
 TEMPLATE_FILE = ./l10n/template.pot
 
 # Name of the generated .po files for each available locale.
-LOCALE_FILES = $(shell find l10n/locale -name '*.po')
+LOCALE_FILES = $(shell find l10n/locale -name '*.po' 2> /dev/null)
 
 .PHONY: l10n-push
 l10n-push:

.make/recursion.mk

@@ -0,0 +1,8 @@
+
+ifeq ($(MAKE_DEPTH),)
+MAKE_DEPTH := 0
+else
+$(eval MAKE_DEPTH := $(shell echo "$$(( $(MAKE_DEPTH) + 1 ))" ) )
+endif
+
+export

.make/release.mk

@@ -7,24 +7,30 @@ release: release-dirs release-linux release-windows release-darwin release-copy
 
 .PHONY: release-dirs
 release-dirs:
-	mkdir -p $(DIST)/binaries $(DIST)/release
-
-.PHONY: release-linux
-release-linux: $(GOX) release-dirs
-	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+	@mkdir -p $(DIST)/binaries $(DIST)/release
 
 # docker specific packaging flags
 DOCKER_LDFLAGS += -X "$(OCIS_REPO)/ocis-pkg/config/defaults.BaseDataPathType=path" -X "$(OCIS_REPO)/ocis-pkg/config/defaults.BaseDataPathValue=/var/lib/ocis"
-release-linux-docker: $(GOX) release-dirs
-	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS) $(DOCKER_LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+release-linux-docker-amd64: $(GOX) release-dirs
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS) $(DOCKER_LDFLAGS)' -os 'linux' -arch 'amd64' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+
+release-linux-docker-arm: $(GOX) release-dirs
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS) $(DOCKER_LDFLAGS)' -os 'linux' -arch 'arm' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+
+release-linux-docker-arm64: $(GOX) release-dirs
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS) $(DOCKER_LDFLAGS)' -os 'linux' -arch 'arm64' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+
+.PHONY: release-linux
+release-linux: $(GOX) release-dirs
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
 
 .PHONY: release-windows
 release-windows: $(GOX) release-dirs
-	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'windows' -arch 'amd64' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'windows' -arch 'amd64' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
 
 .PHONY: release-darwin
 release-darwin: $(GOX) release-dirs
-	$(GOX) -tags 'netgo $(TAGS)' -ldflags '$(LDFLAGS)' -os 'darwin' -arch 'amd64' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+	@$(GOX) -tags 'netgo $(TAGS)' -ldflags '$(LDFLAGS)' -os 'darwin' -arch 'amd64 arm64' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
 
 .PHONY: release-copy
 release-copy:

Dockerfile

@@ -1,3 +1,19 @@
+# Please use this Dockerfile only if
+# you want to build an image from source without
+# Yarn and Go installed on your dev machine.
+
+# You can build oCIS using this Dockerfile
+# by running following command:
+# `docker build -t owncloud/ocis:custom .`
+
+# In most other cases you might want to run the
+# following command instead:
+# `make -C ocis dev-docker`
+# It will build a `owncloud/ocis:dev` image for you
+# and use your local Yarn and Go caches and therefore
+# is a lot faster than the below build steps.
+
+
 FROM owncloudci/nodejs:14 as generate
 
 COPY ./ /ocis/
@@ -12,7 +28,6 @@ COPY --from=generate /ocis /ocis
 WORKDIR /ocis/ocis
 RUN make ci-go-generate build
 
-
 FROM alpine:3.13
 
 RUN apk update && \

Makefile

@@ -43,6 +43,8 @@ ifneq (, $(shell which go 2> /dev/null)) # suppress `command not found warnings`
 include .bingo/Variables.mk
 endif
 
+include .make/recursion.mk
+
 .PHONY: help
 help:
 	@echo "Please use 'make <target>' where <target> is one of the following:"
@@ -130,9 +132,10 @@ ci-go-generate:
 
 .PHONY: ci-node-generate
 ci-node-generate:
-	@for mod in $(OCIS_MODULES); do \
+	@if [ $(MAKE_DEPTH) -le 1 ]; then \
+	for mod in $(OCIS_MODULES); do \
         $(MAKE) --no-print-directory -C $$mod ci-node-generate; \
-    done
+    done; fi;
 
 .PHONY: go-mod-tidy
 go-mod-tidy:
@@ -175,31 +178,31 @@ endif
 .PHONY: l10n-push
 l10n-push:
 	@for extension in $(L10N_MODULES); do \
-    	make -C $$extension l10n-push; \
+		$(MAKE) -C $$extension l10n-push; \
 	done
 
 .PHONY: l10n-pull
 l10n-pull:
 	@for extension in $(L10N_MODULES); do \
-    	make -C $$extension l10n-pull; \
+		$(MAKE) -C $$extension l10n-pull; \
 	done
 
 .PHONY: l10n-clean
 l10n-clean:
 	@for extension in $(L10N_MODULES); do \
-		make -C $$extension l10n-clean; \
+		$(MAKE) -C $$extension l10n-clean; \
 	done
 
 .PHONY: l10n-read
 l10n-read:
 	@for extension in $(L10N_MODULES); do \
-    	make -C $$extension l10n-read; \
+		$(MAKE) -C $$extension l10n-read; \
     done
 
 .PHONY: l10n-write
 l10n-write:
 	@for extension in $(L10N_MODULES); do \
-    	make -C $$extension l10n-write; \
+		$(MAKE) -C $$extension l10n-write; \
     done
 
 .PHONY: ci-format

accounts/.gitignore

@@ -0,0 +1,17 @@
+# yarn2 with Zero-Installs: https://yarnpkg.com/features/zero-installs
+#.yarn/*
+#!.yarn/cache
+#!.yarn/patches
+#!.yarn/plugins
+#!.yarn/releases
+#!.yarn/sdks
+#!.yarn/versions
+
+# yarn2 not using Zero-Installs: https://yarnpkg.com/features/zero-installs
+.yarn/*
+!.yarn/patches
+!.yarn/releases
+!.yarn/plugins
+!.yarn/sdks
+!.yarn/versions
+.pnp.*

accounts/.yarnrc.yml

@@ -0,0 +1,6 @@
+nodeLinker: node-modules
+
+yarnPath: .yarn/releases/yarn-3.1.0.cjs
+
+enableScripts: false
+enableTelemetry: false

accounts/Makefile

@@ -1,6 +1,8 @@
 SHELL := bash
 NAME := accounts
 
+include ../.make/recursion.mk
+
 .PHONY: test-acceptance-webui
 test-acceptance-webui:
 	./ui/tests/run-acceptance-test.sh $(FEATURE_PATH)
@@ -44,7 +46,7 @@ yarn-build: node_modules
 
 .PHONY: node_modules
 node_modules:
-	yarn install --frozen-lockfile
+	yarn install --immutable
 
 ############ protobuf ############
 PROTO_VERSION := v0

accounts/package.json

@@ -80,5 +80,6 @@
   },
   "peerDependencies": {
     "owncloud-design-system": "^11.0.0"
-  }
+  },
+  "packageManager": "[email protected]"
 }