[full-ci] feat(smoke tests): automated parallel smoke tests (#5675)

* feat(bdd smoke testing): introduce multi-user playwright bdd tests

ocis and oc10 bdd smoke tests are now able to with playwright and in parallel,
this means running tests with many users at the same time are now possible.
Sessions won't conflict, this introduces a whole new level of test possibilities and speed

* Bump commit IDs & core img with node14 & update cypress in CI yarn command

* wrap file unlinking in try catch

Co-authored-by: pwengerter <[email protected]>

.drone.env

@@ -1,7 +1,7 @@
 # The version of OCIS to use in pipelines that test against OCIS
-OCIS_COMMITID=45bd133a706ea031f905a159184e6fdb63faa885
+OCIS_COMMITID=a07c1e77a0a3bcb10b109ea4b468472cea10fbb3
 OCIS_BRANCH=master
 
 # The test runner source for API tests
-CORE_COMMITID=08e27f0d73bca5352d87b42936d6d956bc0eaea5
+CORE_COMMITID=d2d63df18170e5e62dbc4d77492ba9a9156af80e
 CORE_BRANCH=master

.drone.star

@@ -1505,7 +1505,7 @@ def installCore(version, db):
 
     stepDefinition = {
         "name": "install-core",
-        "image": "owncloudci/core",
+        "image": "owncloudci/core:nodejs14",
         "pull": "always",
     }
 
@@ -1553,7 +1553,7 @@ def installFederatedServer(version, db, dbSuffix = "-federated"):
 
     stepDefinition = {
         "name": "install-federated",
-        "image": "owncloudci/core",
+        "image": "owncloudci/core:nodejs14",
         "pull": "always",
     }
     if version:

.eslintrc.js

@@ -21,6 +21,7 @@ module.exports = {
      */
     'require-await': 'warn',
     'no-new': 'off',
+    'jest/no-standalone-expect': 'off',
     'node/no-callback-literal': 'off'
   },
   globals: {

cucumber.js

@@ -0,0 +1,12 @@
+module.exports = {
+  smoke: `
+  --require ./tests/smoke/**/*.ts
+  --require-module ts-node/register
+  --format @cucumber/pretty-formatter
+  --publish-quiet
+  --format-options ${JSON.stringify({
+    snippetInterface: 'async-await',
+    snippetSyntax: './tests/smoke/support/cucumber/snippets-syntax.js'
+  })}
+  `
+}

package.json

@@ -19,8 +19,9 @@
     "depcheck": "depcheck",
     "lint": "eslint '{packages,tests}/**/*.{js,ts,vue}' --color",
     "serve": "SERVER=true yarn build:w",
-    "test:acceptance:oc10": "cucumber-js --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format node_modules/cucumber-pretty -t \"${TEST_TAGS:-not @skip and not @skipOnOC10}\" -f json:tests/report/cucumber_report.json",
-    "test:acceptance:ocis": "NODE_TLS_REJECT_UNAUTHORIZED=0 RUN_ON_OCIS=true cucumber-js --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format node_modules/cucumber-pretty -t \"${TEST_TAGS:-not @skip and not @skipOnOCIS and not @notToImplementOnOCIS}\" -f json:tests/report/cucumber_report.json",
+    "test:acceptance:oc10": "cucumber-js --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format @cucumber/pretty-formatter -t \"${TEST_TAGS:-not @skip and not @skipOnOC10}\" -f json:tests/report/cucumber_report.json",
+    "test:acceptance:ocis": "NODE_TLS_REJECT_UNAUTHORIZED=0 RUN_ON_OCIS=true cucumber-js --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format @cucumber/pretty-formatter -t \"${TEST_TAGS:-not @skip and not @skipOnOCIS and not @notToImplementOnOCIS}\" -f json:tests/report/cucumber_report.json",
+    "test:smoke:experimental": "NODE_TLS_REJECT_UNAUTHORIZED=0 cucumber-js --profile=smoke",
     "test:unit": "jest --coverage --config ./tests/unit/config/jest.config.js",
     "test:integration": "jest --config ./tests/integration/config/jest.config.js"
   },
@@ -36,16 +37,21 @@
     "@babel/polyfill": "^7.12.1",
     "@babel/preset-env": "^7.14.7",
     "@babel/register": "^7.13.16",
+    "@cucumber/cucumber": "^7.3.1",
+    "@cucumber/pretty-formatter": "^1.0.0-alpha.1",
     "@erquhart/rollup-plugin-node-builtins": "^2.1.5",
+    "@playwright/test": "^1.14.0",
     "@rollup/plugin-commonjs": "^17.0.0",
     "@rollup/plugin-html": "^0.2.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-typescript": "^8.2.5",
     "@testing-library/jest-dom": "^5.13.0",
     "@testing-library/vue": "^5.6.2",
+    "@types/cucumber": "^7.0.0",
     "@types/jest": "^26.0.23",
     "@types/jest-axe": "^3.5.2",
     "@types/lodash-es": "^4.17.4",
+    "@types/node-fetch": "^2.5.12",
     "@types/vue": "^2.0.0",
     "@typescript-eslint/eslint-plugin": "^4.28.3",
     "@typescript-eslint/parser": "^4.28.3",
@@ -58,9 +64,7 @@
     "chromedriver": "^89.0.0",
     "commander": "^8.1.0",
     "core-js": "^3.15.2",
-    "cucumber": ">=6.0.5",
     "cucumber-html-reporter": "^5.4.0",
-    "cucumber-pretty": "^6.0.0",
     "depcheck": "^1.3.1",
     "ejs": "^3.1.5",
     "eslint": "^7.30.0",
@@ -90,6 +94,7 @@
     "nightwatch-api": "3.0.1",
     "nightwatch-vrt": "^0.2.10",
     "node-fetch": "^2.6.1",
+    "playwright": "^1.14.0",
     "postcss": "^8.3.6",
     "regenerator-runtime": "^0.13.7",
     "requirejs": "^2.3.6",
@@ -111,6 +116,7 @@
     "rollup-plugin-vue": "^5.1.4",
     "sync-fetch": "^0.3.0",
     "ts-jest": "^26.5.6",
+    "ts-node": "^10.2.0",
     "tslib": "^2.2.0",
     "typescript": "^4.3.2",
     "url-search-params-polyfill": "^8.0.0",

tests/acceptance/run.sh

@@ -161,13 +161,13 @@ then
 	TEST_PATHS+=( "${FEATURES_DIR}" )
 fi
 
-RUN_ACCEPTANCE_TESTS="cucumber-js --retry 1 --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format node_modules/cucumber-pretty"
+RUN_ACCEPTANCE_TESTS="cucumber-js --retry 1 --require-module @babel/register --require-module @babel/polyfill --require tests/acceptance/setup.js --require tests/acceptance/stepDefinitions --format @cucumber/pretty-formatter"
 
 if [ -z "${TEST_TAGS}" ]
 then
-	yarn ${RUN_ACCEPTANCE_TESTS} ${TEST_PATHS[@]} | tee -a 'logfile.txt'
+	CUCUMBER_PUBLISH_ENABLED=false yarn ${RUN_ACCEPTANCE_TESTS} ${TEST_PATHS[@]} | tee -a 'logfile.txt'
 else
-	yarn ${RUN_ACCEPTANCE_TESTS} ${TEST_PATHS[@]} -t "${TEST_TAGS}" | tee -a 'logfile.txt'
+	CUCUMBER_PUBLISH_ENABLED=false yarn ${RUN_ACCEPTANCE_TESTS} ${TEST_PATHS[@]} -t "${TEST_TAGS}" | tee -a 'logfile.txt'
 fi
 
 ACCEPTANCE_TESTS_EXIT_STATUS=${PIPESTATUS[0]}

tests/acceptance/setup.js

@@ -1,4 +1,4 @@
-import { setDefaultTimeout, After, Before, defineParameterType } from 'cucumber'
+import { setDefaultTimeout, After, Before, defineParameterType } from '@cucumber/cucumber'
 import { createSession, closeSession, client, startWebDriver, stopWebDriver } from 'nightwatch-api'
 import { rollbackConfigs, cacheConfigs } from './helpers/config'
 import { getAllLogsWithDateTime } from './helpers/browserConsole.js'

tests/acceptance/stepDefinitions/accountContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When, Then } = require('cucumber')
+const { When, Then } = require('@cucumber/cucumber')
 const assert = require('assert')
 const _ = require('lodash')
 

tests/acceptance/stepDefinitions/filesContext.js

@@ -1,7 +1,7 @@
 /* eslint-disable no-unused-expressions */
 const { client } = require('nightwatch-api')
 const assert = require('assert')
-const { Given, When, Then, Before } = require('cucumber')
+const { Given, When, Then, Before } = require('@cucumber/cucumber')
 const webdav = require('../helpers/webdavHelper')
 const _ = require('lodash')
 const loginHelper = require('../helpers/loginHelper')

tests/acceptance/stepDefinitions/generalContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { After, Before, Given, Then, When } = require('cucumber')
+const { After, Before, Given, Then, When } = require('@cucumber/cucumber')
 const webdavHelper = require('../helpers/webdavHelper')
 const httpHelper = require('../helpers/httpHelper')
 const backendHelper = require('../helpers/backendHelper')
@@ -8,7 +8,7 @@ const fs = require('fs')
 const occHelper = require('../helpers/occHelper')
 
 let initialConfigJsonSettings
-let createdFiles = []
+const createdFiles = []
 
 Given(
   'a file with the size of {string} bytes and the name {string} has been created locally',
@@ -244,7 +244,7 @@ After(async function(testCase) {
 })
 
 Before(function(testCase) {
-  createdFiles = []
+  testCase.pickle.createdFiles = []
   if (
     typeof process.env.SCREEN_RESOLUTION !== 'undefined' &&
     process.env.SCREEN_RESOLUTION.trim() !== ''
@@ -264,7 +264,8 @@ Before(function(testCase) {
   } else {
     client.maximizeWindow()
   }
-  console.log('  ' + testCase.sourceLocation.uri + ':' + testCase.sourceLocation.line + '\n')
+  // todo
+  // console.log('  ' + testCase.sourceLocation.uri + ':' + testCase.sourceLocation.line + '\n')
 })
 
 After(async function(testCase) {
@@ -273,7 +274,13 @@ After(async function(testCase) {
   }
   console.log('\n  Result: ' + testCase.result.status + '\n')
 
-  createdFiles.forEach(fileName => fs.unlinkSync(fileName))
+  createdFiles.forEach(fileName => {
+    try {
+      fs.unlinkSync(fileName)
+    } catch (err) {
+      console.info(err.message)
+    }
+  })
 
   // clear file locks
   const body = new URLSearchParams()

tests/acceptance/stepDefinitions/loginContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { Given, Then, When } = require('cucumber')
+const { Given, Then, When } = require('@cucumber/cucumber')
 const loginHelper = require('../helpers/loginHelper')
 const assert = require('assert')
 

tests/acceptance/stepDefinitions/markdownEditorContext.js

@@ -1,7 +1,7 @@
 const { client } = require('nightwatch-api')
 const assert = require('assert')
 const _ = require('lodash')
-const { Given, When, Then } = require('cucumber')
+const { Given, When, Then } = require('@cucumber/cucumber')
 
 const markdownEditor = client.page.markdownEditorPage()
 const filesList = client.page.FilesPageElement.filesList()

tests/acceptance/stepDefinitions/messagesContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When } = require('cucumber')
+const { When } = require('@cucumber/cucumber')
 
 When('the user closes the message', function() {
   return client.page.webPage().closeMessage()

tests/acceptance/stepDefinitions/notificationsContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { Given, When, Then } = require('cucumber')
+const { Given, When, Then } = require('@cucumber/cucumber')
 const httpHelper = require('../helpers/httpHelper')
 const codify = require('../helpers/codify')
 const assert = require('assert')

tests/acceptance/stepDefinitions/previewContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { Given, When, Then } = require('cucumber')
+const { Given, When, Then } = require('@cucumber/cucumber')
 const mediaViewerPage = client.page.FilesPageElement.mediaViewerPage()
 const filesList = client.page.FilesPageElement.filesList()
 const assert = require('assert')

tests/acceptance/stepDefinitions/privateLinksContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When } = require('cucumber')
+const { When } = require('@cucumber/cucumber')
 const webdav = require('../helpers/webdavHelper')
 
 When(

tests/acceptance/stepDefinitions/provisioningContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { Given, After } = require('cucumber')
+const { Given, After } = require('@cucumber/cucumber')
 const fs = require('fs-extra')
 const assert = require('assert')
 require('url-search-params-polyfill')

tests/acceptance/stepDefinitions/publicLinkContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When, Then } = require('cucumber')
+const { When, Then } = require('@cucumber/cucumber')
 require('url-search-params-polyfill')
 const sharingHelper = require('../helpers/sharingHelper')
 const assert = require('assert')

tests/acceptance/stepDefinitions/searchContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When } = require('cucumber')
+const { When } = require('@cucumber/cucumber')
 
 When('the user searches for {string} using the webUI', function(searchTerm) {
   return client.page.webPage().search(searchTerm)

tests/acceptance/stepDefinitions/sharingContext.js

@@ -1,5 +1,5 @@
 const { client } = require('nightwatch-api')
-const { When, Given, Then } = require('cucumber')
+const { When, Given, Then } = require('@cucumber/cucumber')
 const assert = require('assert')
 const { URLSearchParams } = require('url')
 require('url-search-params-polyfill')

tests/acceptance/stepDefinitions/visualContext.js

@@ -1,4 +1,4 @@
-const { Then } = require('cucumber')
+const { Then } = require('@cucumber/cucumber')
 const { client } = require('nightwatch-api')
 const _ = require('lodash')
 const path = require('path')

tests/acceptance/stepDefinitions/webdavContext.js

@@ -1,4 +1,4 @@
-const { Given, Then } = require('cucumber')
+const { Given, Then } = require('@cucumber/cucumber')
 require('url-search-params-polyfill')
 const httpHelper = require('../helpers/httpHelper')
 const backendHelper = require('../helpers/backendHelper')

tests/smoke/README.md

@@ -0,0 +1,100 @@
+# WEB-Smoke-Tests (EXPERIMENTAL POC)
+
+Use with care, the smoke-test package introduces many new concepts and is still under heavy development.
+
+## Why smoke tests
+
+Before we release a new web version, we always take care that nothing breaks and the core features are working as
+expected. Till now this was a manual process which consumed a lot of time and effort.
+
+Manual steps are error-prone by nature, on the one hand they can bring up unknown issues, but on the other it's easy to
+forget some test steps. The main problem we faced, was that it's never a good thing to manually test your own code.
+
+To get closer to the point where we are able to release faster and more often, we started to develop the WEB-Smoke-Tests
+package.
+
+## Why not as part of the tests/acceptance package
+
+The tests/acceptance package has a much broader test scope and is here to test every little element // interaction in
+ci. To guarantee that process, a lot of steps are required which takes a lot of time and isn't the perfect fit for local
+development.
+
+The WEB-Smoke-Tests are much smaller and are still in an early phase. Many things are still not there compared to the
+mature tests/acceptance package.
+
+**The main reasons why we decided to split out the smoke tests are:**
+
+* we wanted to test on a persona // use-case level
+* testing should be easy, fast and reliable
+* no external dependencies expect a backend
+* strongly typed page-objects, steps and helpers
+* develop, test manually, extend automatic tests, release, repeat
+
+## Run a smoke test
+
+Please make sure to point http://host.docker.internal/ to 127.0.0.1 by adding it to your hosts.
+
+```shell
+$ yarn && yarn build:w
+$ docker-compose up oc10 ocis
+$ yarn test:smoke:experimental tests/smoke/features/kindergarten.feature
+```
+
+## Available options
+
+To run the tests with below options, you have to set them as environment variable in you shell.
+
+```shell
+$ OPTION_1=foo OPTION_2=bar yarn test:smoke:experimental ...
+```
+
+* **OCIS=boolean**
+    * defines if the tests should use ocis as backend
+    * **default**: false
+* **SLOW_MO=time-in-ms**
+    * run the tests with a timeout between every interaction
+    * **default**: 0
+* **HEADLESS=boolean**
+    * run the tests without opening a browser
+    * **default**: false
+* **BROWSER=string**
+    * define which browser is used to run the tests
+    * **default**: chrome
+    * **values**: firefox | webkit | chrome | chromium
+
+## Package structure
+
+This package is still in an early stage, design could change over time.
+
+### Features
+`./tests/smoke/features`
+[Gherkin features](https://cucumber.io/docs/gherkin/reference/) and corresponding [Cucumber step definitions](https://cucumber.io/docs/cucumber/step-definitions/) placed in there, it follows
+
+### Support
+`./tests/smoke/support`
+everything that is needed to run the tests organize as package 
+
+`./tests/smoke/support/api`
+Every call that is used to provision, configure or even request data from backend will be placed here
+
+`./tests/smoke/support/cta`
+(Call to action) - Snippets that can be used across the entire livecycle of a test
+
+`./tests/smoke/support/cucumber`
+Mainly all cucumber related things, for now only setup is done here
+
+`./tests/smoke/support/page`
+With the option in mind that the smoke-test package could grow, we decided to already group interactions inside page
+objects. To get more background what page objects are, you can read
+the [introduction here](https://playwright.dev/docs/pom/)
+
+`./tests/smoke/support/store`
+Simple store to persist data for the lifetime of a test
+
+`./tests/smoke/support/world`
+Custom world that is used across the tests, global objects like services get initialized there. [Read more](https://github.com/cucumber/cucumber-js/blob/main/docs/support_files/world.md)
+
+## Packages we use
+
+* [cucumber-js](https://github.com/cucumber/cucumber-js)
+* [playwright](https://github.com/microsoft/playwright)