Issue/3315 New Integration test facility & add integration tests for auth

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
     -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
   # If you changed dependencies (especially for scala), you might want to prevent your branch from using old cache by increase the version blew
   # gitlab actually use the same technique when you click the `Clear Cache Button`
-  CACHE_VERSION: ts-12-scala-7
+  CACHE_VERSION: ts-13-scala-7
   DOCKER_HOST: tcp://docker:2376
   DOCKER_TLS_CERTDIR: "/certs"
   DOCKER_TLS_VERIFY: 1
@@ -31,7 +31,7 @@ stages:
 
 yarn-install:
   stage: builders
-  image: data61/magda-builder-docker:buildx
+  image: data61/magda-builder-docker:master
   retry: 1
   needs: []
   cache:
@@ -50,7 +50,7 @@ yarn-install:
 
 build-builder-image:
   stage: builders
-  image: data61/magda-builder-docker:buildx
+  image: data61/magda-builder-docker:master
   retry: 1
   needs: []
   before_script:
@@ -465,6 +465,57 @@ buildtest:storage-api:
       - "*/dist"
     expire_in: 7 days
 
+buildtest:integration-tests:
+  stage: buildtest
+  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$BUILDER_IMG_TAG
+  retry: 1
+  needs:
+    - yarn-install
+    - build-builder-image
+    - registry-typescript-api
+    - sbt-prebuild
+    - dockerize:dockerExtensions
+  before_script:
+    - |
+      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+        export DOCKER_HOST='tcp://localhost:2375'
+      fi
+    - docker info
+    - docker-compose --version
+    # we can remove socat installation in future once the change is merged into master as it's installed in base image
+    - apk add socat 
+  cache:
+    key: $CI_JOB_NAME-$CACHE_VERSION
+    paths:
+      - "$CI_PROJECT_DIR/pip-cache"
+  services:
+    - docker:dind
+  variables:
+    POSTGRES_URL: "jdbc:postgresql://localhost/postgres"
+    POSTGRES_DB: postgres
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: "password"
+  tags:
+    - heavy
+  script:
+    - cd magda-authorization-api && yarn build
+    - cd ..
+    - cd magda-storage-api && yarn build
+    - cd ..
+    - cd magda-migrator-registry-aspects && yarn build
+    - cd ..
+    - cd magda-int-test-ts
+    - yarn test
+  artifacts:
+    paths:
+      - "target"
+      - "project/target"
+      - "project/project/target"
+      - "*/target"
+      - "*/project/target"
+      - "*/project/project/target"
+    expire_in: 7 days
+
 buildtest:opa-policies:
   stage: buildtest
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
@@ -629,7 +680,7 @@ dockerize:opa:
   - yarn docker-build-prod --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG  --platform linux/arm64,linux/amd64
 
 dockerize:dockerExtensions:
-  stage: buildtest
+  stage: prebuild
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
   retry: 2
   needs:
@@ -646,83 +697,6 @@ dockerize:dockerExtensions:
     - cd $CI_PROJECT_DIR/magda-elastic-search && yarn docker-build-prod --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --platform linux/arm64,linux/amd64
     - cd $CI_PROJECT_DIR/magda-postgres && yarn docker-build-prod --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG
 
-inttest:registryAuth:
-  # https://github.com/kind-ci/examples/blob/master/.gitlab-ci.yml
-  stage: preview
-  retry: 1
-  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$BUILDER_IMG_TAG
-  allow_failure: true
-  tags:
-    - heavy
-  needs:
-    - yarn-install
-    - sbt-prebuild
-    - buildtest:registry
-    - buildtest:helm-charts
-    - dockerize:dockerExtensions
-  services:
-    - docker:dind
-  cache:
-    key: $CI_JOB_NAME-$CACHE_VERSION
-    paths:
-      - /usr/local/bin/kind
-      - /usr/local/bin/kubectl
-      - /usr/local/bin/helm
-  variables:
-    KUBECTL: v1.19.14
-    KIND: v0.11.1
-  before_script:
-    - echo "Waiting for docker cli to respond before continuing build..."
-    - | 
-      for i in $(seq 1 30); do
-        if ! docker info &> /dev/null; then
-          echo "Docker not responding yet. Sleeping for 2s..." && sleep 2s
-        else
-          echo "Docker ready. Continuing build..."
-          break
-        fi
-      done
-  script: |
-    apk add --update postgresql-client sed curl pwgen
-    curl -Lo /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-$(uname)-amd64 -C -
-    chmod +x /usr/local/bin/kind
-    curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL}/bin/linux/amd64/kubectl -C -
-    chmod +x /usr/local/bin/kubectl
-    curl -L https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz -C - | tar -xzO linux-amd64/helm > /usr/local/bin/helm
-    chmod +x /usr/local/bin/helm
-
-    # create k8s cluster
-    kind create cluster --config=./kind-config.yaml --wait 20m --image kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861
-
-    # Set up KUBECONFIG that links to kind - we use sed to edit it so it's pointing to gitlab's docker host
-    kind get kubeconfig | sed -E -e 's/localhost|0\.0\.0\.0/docker/g' > ./kubeconfig
-
-    export KUBECONFIG="$(pwd)/kubeconfig"
-    export JWT_SECRET=udIsbcYaKs1G4n6AdiMSIvPx5KpxQAy8FA2aIcD46iCipNAZvds4jeXFLZKhVvSJZvhYb5Pvgvmtonk7UFfhGnYcd3DXM7KzHG7gBmGO8PCsOZ4t7icqZoJbpdDqYWMmd9XnrVXtJhR6HVFBmEmbk9AmFJ1Gz9ipYPGYLoFcavPs9iZ63KPXgdt4aBdWQcmICkGPYiY8CQOvqOoiU7hUhKDTkJgRRTSaax6UQDOveTQvQnd5uyXuV4os0tlahzRX
-    kubectl create ns test2
-    echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"auth-secrets\"}, \"type\": \"Opaque\", \"data\": {\"jwt-secret\": \"dWRJc2JjWWFLczFHNG42QWRpTVNJdlB4NUtweFFBeThGQTJhSWNENDZpQ2lwTkFadmRzNGplWEZMWktoVnZTSlp2aFliNVB2Z3ZtdG9uazdVRmZoR25ZY2QzRFhNN0t6SEc3Z0JtR084UENzT1o0dDdpY3Fab0picGREcVlXTW1kOVhuclZYdEpoUjZIVkZCbUVtYms5QW1GSjFHejlpcFlQR1lMb0ZjYXZQczlpWjYzS1BYZ2R0NGFCZFdRY21JQ2tHUFlpWThDUU92cU9vaVU3aFVoS0RUa0pnUlJUU2FheDZVUURPdmVUUXZRbmQ1dXlYdVY0b3MwdGxhaHpSWA==\"}}" | kubectl apply --namespace test2 -f -
-    echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"regcred\"}, \"type\": \"kubernetes.io/dockerconfigjson\", \"data\": { \".dockerconfigjson\": \"$DOCKERCONFIGJSON\" }}" | kubectl apply --namespace test2 -f -
-
-    # Create DB password secret
-    export PGPASSWORD="${DB_PASSWORD:-$(pwgen 16 1)}"
-    kubectl create secret generic db-main-account-secret --from-literal=postgresql-password=$PGPASSWORD --namespace test2
-
-    helm upgrade test2 deploy/helm/local-auth-test-deployment --debug --namespace test2 --install --timeout 1200s --set global.image.tag=$CI_COMMIT_REF_SLUG,magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG
-
-    # Forward local ports to kind
-    kubectl port-forward combined-db-postgresql-0 5432 --namespace test2 >/dev/null 2>&1 < /dev/null &
-    kubectl port-forward deployment/authorization-api 6104:80 --namespace test2 >/dev/null 2>&1 < /dev/null &
-    # Wait for port forwarding to start
-    timeout 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 5432
-    timeout 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 6104
-    # Set up some values
-    psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/organizations.sql
-    psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/users.sql
-    # Test!
-    export POSTGRES_USER=postgres
-    export POSTGRES_PASSWORD=$PGPASSWORD
-    sbt "registryApi/testOnly au.csiro.data61.magda.opa.*"
-
 (Full) Run As Preview: &runAsPreview
   stage: preview
   when: manual
@@ -834,6 +808,7 @@ Deploy Master To Dev:
     - dockerize:typescript
     - dockerize:migrators
     - dockerize:dockerExtensions
+    - buildtest:integration-tests
   image:
     name: dtzar/helm-kubectl:3.5.2
   retry: 1
@@ -882,6 +857,7 @@ Release-to-Docker-Hub:
     - dockerize:typescript
     - dockerize:migrators
     - dockerize:dockerExtensions
+    - buildtest:integration-tests
   cache: {}
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
   retry: 1
@@ -906,6 +882,7 @@ Release-Master-Tag-to-Docker-Hub:
     - dockerize:typescript
     - dockerize:migrators
     - dockerize:dockerExtensions
+    - buildtest:integration-tests
   cache: {}
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
   retry: 1
@@ -936,6 +913,7 @@ Release-Latest-Tag-to-Docker-Hub:
     - dockerize:migrators
     - dockerize:dockerExtensions
     - Release-to-Docker-Hub
+    - buildtest:integration-tests
   cache: {}
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
   retry: 1
@@ -964,6 +942,7 @@ Publish NPM Packages:
     - dockerize:typescript
     - dockerize:migrators
     - dockerize:dockerExtensions
+    - buildtest:integration-tests
   image:
     name: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$BUILDER_IMG_TAG
   cache: {}
@@ -991,6 +970,7 @@ Publish Helm Chart:
     - dockerize:typescript
     - dockerize:migrators
     - dockerize:dockerExtensions
+    - buildtest:integration-tests
   image:
     name: dtzar/helm-kubectl:3.5.2
   cache:

CHANGES.md

@@ -21,6 +21,13 @@
 - #3340 OPA AST parser takes too long to evaluate large response
 - Increase registry default request timeout to 60s (from 30s)
 - Bump Dataset index version to 49 & publishers index version to 7
+- Fixed registry API generated ts client patchRecord API response type
+- add `allowAutoCrawlOnStartingUp` option to indexer
+- add `/v0/status/live` & `/v0/status/ready` endpoint to indexer
+- Related to #3315, introduce `ServiceRunner` to launch an integrated test & dev environment
+- #3315 rewrite auth related integration tests and execute tests using `ServiceRunner` rather than mini k8s cluster `kind`
+- #3345 Add dataset index / delete API endpoints to indexer
+- Add `debug` switch to registry API, search API & Auth API
 
 ## 1.2.1
 

deploy/helm/internal-charts/indexer/templates/deployment.yaml

@@ -50,6 +50,8 @@ spec:
             "-Dregistry.webhookUrl=http://indexer/v0/registry-hook",
             "-Dregistry.baseUrl=http://registry-api",
             "-Dregistry.readOnlyBaseUrl=http://registry-api-read-only",
+            "-DauthApi.baseUrl=http://authorization-api",
+            "-Dopa.baseUrl=http://authorization-api/v0/opa/",
             "-Dindexer.readSnapshots={{ .Values.readSnapshots }}",
             "-Dindexer.makeSnapshots={{ .Values.makeSnapshots }}",
             "-Dakka.loglevel={{ .Values.global.logLevel }}",

deploy/helm/internal-charts/search-api/README.md

@@ -17,6 +17,7 @@ Kubernetes: `>= 1.14.0-0`
 | autoscaler.minReplicas | int | `1` |  |
 | autoscaler.targetCPUUtilizationPercentage | int | `80` |  |
 | datasetsIndexVersion | string | `nil` | Manually set dataset index version. If not specify, default version will be used. you want to manually set this setting when upgrade to a Magda version that involves dataset index version changes. As it takes time to rebuild the index, you could use this setting to make search API query existing old version index before the new version index is built. |
+| debug | bool | `false` | when set to true, search API will print verbose debug info (e.g. ES DSL query) to log |
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
 | defaultImage.repository | string | `"docker.io/data61"` |  |

deploy/helm/internal-charts/search-api/templates/deployment.yaml

@@ -32,6 +32,9 @@ spec:
           - "-DauthApi.baseUrl=http://authorization-api"
           - "-Dopa.baseUrl=http://authorization-api/v0/opa/"
           - "-DelasticSearch.serverUrl=elasticsearch://elasticsearch:9200"
+{{- if .Values.debug }}
+          - "-DsearchApi.debug=true"
+{{- end }}
 {{- if .Values.datasetsIndexVersion }}
           - "-DelasticSearch.indices.datasets.version={{ .Values.datasetsIndexVersion }}"
 {{- end }}

deploy/helm/internal-charts/search-api/values.yaml

@@ -22,6 +22,9 @@ resources:
   limits:
     cpu: 200m
 
+# -- when set to true, search API will print verbose debug info (e.g. ES DSL query) to log
+debug: false
+
 # -- Manually set dataset index version.
 # If not specify, default version will be used.
 # you want to manually set this setting when upgrade to a Magda version that involves dataset index version changes.

magda-authorization-api/package.json

@@ -15,14 +15,15 @@
   },
   "dependencies": {
     "@magda/typescript-common": "^2.0.0-alpha.0",
+    "uuid": "^8.2.0",
     "bcrypt": "^5.0.0",
     "body-parser": "^1.18.3",
     "express": "^4.17.1",
     "is-uuid": "^1.0.2",
     "jsonwebtoken": "^8.4.0",
     "lodash": "^4.17.4",
     "object-path": "^0.11.4",
-    "pg": "^6.4.0",
+    "pg": "^7.18.2",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.7",
     "sql-syntax": "^1.1.3",
@@ -42,13 +43,14 @@
     "@types/mocha": "^2.2.47",
     "@types/nock": "^9.1.2",
     "@types/object-path": "^0.11.0",
-    "@types/pg": "^6.1.41",
+    "@types/pg": "^7.14.4",
     "@types/recursive-readdir": "^2.2.0",
     "@types/request": "^2.48.1",
     "@types/request-promise-native": "^1.0.16",
     "@types/sinon": "^7.0.11",
     "@types/supertest": "^2.0.4",
     "@types/yargs": "^12.0.8",
+    "@types/uuid": "^8.0.0",
     "chai": "^4.1.2",
     "chai-as-promised": "^7.1.1",
     "fs-extra": "^8.0.1",