[MM-62065] e2e HA support (#913)

* Refactor e2e scripts * HA Support * Fix signaling after WS reconnect in non-RTCD HA (#920)
mattermost · Dec 10, 2024 · 2cae664 · 2cae664
1 parent fef920d
commit 2cae664
Show file tree

Hide file tree

Showing 9 changed files with 234 additions and 151 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -73,13 +73,15 @@ jobs:
       COMPOSE_PROJECT_NAME: playwright_tests
       DOCKER_NETWORK: playwright_tests
       CONTAINER_SERVER: playwright_tests_server
+      CONTAINER_PROXY: playwright_tests_proxy
       IMAGE_CALLS_OFFLOADER: mattermost/calls-offloader:v0.8.0
       IMAGE_CALLS_RECORDER: mattermost/calls-recorder:v0.7.7
       IMAGE_CALLS_TRANSCRIBER: mattermost/calls-transcriber:v0.5.0
       IMAGE_SERVER: mattermostdevelopment/mattermost-enterprise-edition:master
       IMAGE_CURL: curlimages/curl:8.7.1
       CI_NODE_INDEX: ${{ matrix.run_id }}
       CI_NODE_TOTAL: ${{ needs.generate-matrix.outputs.parallelism }}
+      WORKSPACE: ${{github.workspace}}
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.parallelism-matrix) }}
     steps:
@@ -108,14 +110,6 @@ jobs:
           mkdir -p ${{ github.workspace }}/config
           go run main.go
 
-      - name: e2e/patch-generated-configuration
-        env:
-          OUTPUT_CONFIG: ${{ github.workspace }}/config/config.json
-        run: |
-          cp ${OUTPUT_CONFIG} ${OUTPUT_CONFIG}.orig
-          jq -f ${{ github.workspace }}/e2e/overlay-config.jq ${OUTPUT_CONFIG}.orig > ${OUTPUT_CONFIG}
-          diff ${OUTPUT_CONFIG}.orig ${OUTPUT_CONFIG} || true
-
       - name: e2e/download-mattermost-plugin-calls-package
         uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
         with:
@@ -130,41 +124,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_DEV_USERNAME }}
           password: ${{ secrets.DOCKERHUB_DEV_TOKEN }}
 
-      - name: e2e/prepare-server
+      - name: e2e/run-server
         env:
           DOCKER_CLIENT_TIMEOUT: 120
           COMPOSE_HTTP_TIMEOUT: 120
           DOCKER_COMPOSE_FILE: ${{ github.workspace }}/e2e/docker/docker-compose.yaml
+          MM_PLUGIN_CALLS_TEST_LICENSE: ${{ secrets.MM_PLUGIN_CALLS_TEST_LICENSE }}
         run: |
-          mkdir -p ${{ github.workspace }}/logs
-          mkdir -p ${{ github.workspace }}/config
-          mkdir -p ${{ github.workspace }}/dotenv
-
-          ## Navigate to mattermost server build folder to spawn the environment
-          cd mattermost/server/build/
           ${{ github.workspace }}/e2e/scripts/prepare-server.sh
-          cd -
-
-          ## Add extra environment variables for mattermost server
-          echo "MM_LICENSE=${{ secrets.MM_PLUGIN_CALLS_TEST_LICENSE }}" >> dotenv/app.private.env
-          echo "MM_FEATUREFLAGS_BoardsProduct=true" >> dotenv/app.private.env
-          echo "MM_SERVICEENVIRONMENT=test" >> dotenv/app.private.env
-          echo "MM_CALLS_JOB_SERVICE_URL=http://calls-offloader:4545" >> dotenv/app.private.env
-
-          sudo chown -R 2000:2000 ${{ github.workspace }}/logs
-          sudo chown -R 2000:2000 ${{ github.workspace }}/config
-
-          # Spawn mattermost server
-          echo "Spawning mattermost server ... "
-          docker run -d --quiet --name ${CONTAINER_SERVER} \
-          --net ${DOCKER_NETWORK} \
-          --net-alias mm-server \
-          --user mattermost:mattermost \
-          --env-file="${{ github.workspace }}/dotenv/app.private.env" \
-          -v ${{ github.workspace }}/config:/mattermost/config:rw \
-          -v ${{ github.workspace }}/logs:/mattermost/logs:rw \
-          ${IMAGE_SERVER} \
-          sh -c "/mattermost/bin/mattermost server"
 
       - name: e2e/build-e2e-image
         working-directory: e2e
@@ -174,17 +141,13 @@ jobs:
           docker build --quiet -t mm-playwright -f Playwright.Dockerfile .
 
       - name: e2e/test
+        env:
+          WORKSPACE: ${{github.workspace}}
+          RUN_ID: ${{matrix.run_id}}
         id: test
         run: |
-          mkdir -p ${{ github.workspace }}/results
           ${{ github.workspace }}/e2e/scripts/run.sh
 
-          ## Check if we have an early failures in order to upload logs
-          NUM_FAILURES=0
-          NUM_FAILURES=$((NUM_FAILURES + $(jq '.suites[].suites[].specs[].tests[] | last(.results[]) | select(.status != "passed").status' < ${{ github.workspace }}/results/pw-results-${{ matrix.run_id }}.json | wc -l)))
-          echo "FAILURES=${NUM_FAILURES}" >> ${GITHUB_OUTPUT}
-          sudo chown -R 1001:1001 ${{ github.workspace }}/logs
-
       - name: e2e/persist-report-results
         uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
         with:

diff --git a/e2e/docker/docker-compose.yaml b/e2e/docker/docker-compose.yaml
@@ -21,10 +21,24 @@ services:
       timeout: 10s
       retries: 3
 
+  haproxy:
+    image: nginx
+    container_name: ${CONTAINER_PROXY}
+    volumes:
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+    restart: on-failure
+    networks:
+      default:
+        aliases:
+          - mm-server
+    ports:
+      - "8065:8065"
+
   start_dependencies:
     image: mattermost/mattermost-wait-for-dep:latest
     depends_on:
       - postgres
+      - haproxy
     command: postgres:5432
     networks:
       default:

diff --git a/e2e/docker/nginx.conf b/e2e/docker/nginx.conf
@@ -0,0 +1,37 @@
+upstream app_cluster {
+  server mm-server1:8065 fail_timeout=10s max_fails=10;
+  server mm-server2:8065 fail_timeout=10s max_fails=10;
+}
+
+server {
+  listen 8065;
+
+  location ~ /api/v[0-9]+/(users/)?websocket$ {
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_http_version 1.1;
+    client_max_body_size 50M;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Frame-Options SAMEORIGIN;
+    proxy_buffers 256 16k;
+    proxy_buffer_size 16k;
+    proxy_read_timeout 600s;
+    proxy_pass http://app_cluster;
+  }
+
+  location / {
+    client_max_body_size 100M;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+	  proxy_http_version 1.1;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Frame-Options SAMEORIGIN;
+    proxy_pass http://app_cluster;
+  }
+}
diff --git a/e2e/overlay-config.jq b/e2e/overlay-config.jq
diff --git a/e2e/scripts/prepare-server.sh b/e2e/scripts/prepare-server.sh
@@ -2,11 +2,23 @@
 set -eu
 set -o pipefail
 
+function print_logs {
+	docker logs ${CONTAINER_SERVER}1
+	docker logs ${CONTAINER_SERVER}2
+	docker logs ${CONTAINER_PROXY}
+}
+
+trap print_logs EXIT
+
+mkdir -p ${WORKSPACE}/logs
+mkdir -p ${WORKSPACE}/config
+mkdir -p ${WORKSPACE}/dotenv
+
 docker network create ${DOCKER_NETWORK}
 
 # Start server dependencies
 echo "Starting server dependencies ... "
-DOCKER_NETWORK=${DOCKER_NETWORK} docker compose -f ${DOCKER_COMPOSE_FILE} run -d --rm start_dependencies
+DOCKER_NETWORK=${DOCKER_NETWORK} CONTAINER_PROXY=${CONTAINER_PROXY} docker compose -f ${DOCKER_COMPOSE_FILE} run -d --rm start_dependencies
 timeout --foreground 90s bash -c "until docker compose -f ${DOCKER_COMPOSE_FILE} exec -T postgres pg_isready ; do sleep 5 ; done"
 
 echo "Pulling ${IMAGE_CALLS_RECORDER} in order to be quickly accessible ... "
@@ -29,15 +41,87 @@ docker images
 echo "Spawning calls-offloader service with docker host access ..."
 # Spawn calls offloader image as root to access local docker socket
 docker run -d --quiet --user root --name "${COMPOSE_PROJECT_NAME}_callsoffloader" \
-  -v /var/run/docker.sock:/var/run/docker.sock:rw \
-  --net ${DOCKER_NETWORK} \
-  --env "API_SECURITY_ALLOWSELFREGISTRATION=true" \
-  --env "JOBS_MAXCONCURRENTJOBS=20" \
-  --env "LOGGER_ENABLEFILE=false" \
-  --env "LOGGER_CONSOLELEVEL=DEBUG" \
-  --env "DEV_MODE=true" \
-  --env "DOCKER_NETWORK=${DOCKER_NETWORK}" \
-  --network-alias=calls-offloader ${IMAGE_CALLS_OFFLOADER}
+	-v /var/run/docker.sock:/var/run/docker.sock:rw \
+	--net ${DOCKER_NETWORK} \
+	--env "API_SECURITY_ALLOWSELFREGISTRATION=true" \
+	--env "JOBS_MAXCONCURRENTJOBS=20" \
+	--env "LOGGER_ENABLEFILE=false" \
+	--env "LOGGER_CONSOLELEVEL=DEBUG" \
+	--env "DEV_MODE=true" \
+	--env "DOCKER_NETWORK=${DOCKER_NETWORK}" \
+	--network-alias=calls-offloader ${IMAGE_CALLS_OFFLOADER}
 
 # Check that calls-offloader is up and ready
 docker run --rm --quiet --name "${COMPOSE_PROJECT_NAME}_curl_callsoffloader" --net ${DOCKER_NETWORK} ${IMAGE_CURL} sh -c "until curl -fs http://calls-offloader:4545/version; do echo Waiting for calls-offloader; sleep 5; done; echo calls-offloader is up"
+
+## Add extra environment variables for mattermost server. This is needed to override configuration in HA since
+## the config is stored in DB.
+echo "MM_LICENSE=${MM_PLUGIN_CALLS_TEST_LICENSE}" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_FEATUREFLAGS_BoardsProduct=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICEENVIRONMENT=test" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_CALLS_JOB_SERVICE_URL=http://calls-offloader:4545" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_CONFIG=postgres://mmuser:mostest@postgres/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_SITEURL=http://mm-server:8065" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_ENABLELOCALMODE=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_ENABLEDEVELOPER=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_ENABLETESTING=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_ALLOWCORSFROM=http://localhost:8065" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SERVICESETTINGS_ENABLEONBOARDINGFLOW=false" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_PLUGINSETTINGS_ENABLE=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_PLUGINSETTINGS_ENABLEUPLOADS=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_PLUGINSETTINGS_AUTOMATICPREPACKAGEDPLUGINS=false" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_CLUSTERSETTINGS_ENABLE=true" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_CLUSTERSETTINGS_CLUSTERNAME=mm_server_e2e" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_LOGSETTINGS_CONSOLELEVEL=DEBUG" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_LOGSETTINGS_FILELEVEL=DEBUG" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_SQLSETTINGS_DATASOURCE=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_EXPERIMENTALSETTINGS_DISABLEAPPBAR=false" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_ANNOUNCEMENTSETTINGS_USERNOTICESENABLED=false" >>${WORKSPACE}/dotenv/app.private.env
+echo "MM_ANNOUNCEMENTSETTINGS_ADMINNOTICESENABLED=false" >>${WORKSPACE}/dotenv/app.private.env
+
+sudo cp -r ${WORKSPACE}/logs ${WORKSPACE}/logs1
+sudo cp -r ${WORKSPACE}/config ${WORKSPACE}/config1
+sudo chown -R 2000:2000 ${WORKSPACE}/logs1
+sudo chown -R 2000:2000 ${WORKSPACE}/config1
+
+sudo cp -r ${WORKSPACE}/logs ${WORKSPACE}/logs2
+sudo cp -r ${WORKSPACE}/config ${WORKSPACE}/config2
+sudo chown -R 2000:2000 ${WORKSPACE}/logs2
+sudo chown -R 2000:2000 ${WORKSPACE}/config2
+
+mkdir -p ${WORKSPACE}/mmdata
+sudo chown -R 2000:2000 ${WORKSPACE}/mmdata
+
+# Spawn mattermost server
+echo "Spawning mattermost server 1 ... "
+docker run -d --quiet --name ${CONTAINER_SERVER}1 \
+	--net ${DOCKER_NETWORK} \
+	--net-alias mm-server1 \
+	--user mattermost:mattermost \
+	--env-file="${WORKSPACE}/dotenv/app.private.env" \
+	-v ${WORKSPACE}/config1:/mattermost/config:rw \
+	-v ${WORKSPACE}/logs1:/mattermost/logs:rw \
+	-v ${WORKSPACE}/mmdata:/mattermost/data:rw \
+	${IMAGE_SERVER} \
+	sh -c "/mattermost/bin/mattermost server"
+
+echo "Checking node 1 is up and running"
+timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_mm1 --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server1:8065/api/v4/system/ping; do echo Waiting for mm-server1; sleep 2; done; echo mm-server1 is up"
+
+echo "Spawning mattermost server 2 ... "
+docker run -d --quiet --name ${CONTAINER_SERVER}2 \
+	--net ${DOCKER_NETWORK} \
+	--net-alias mm-server2 \
+	--user mattermost:mattermost \
+	--env-file="${WORKSPACE}/dotenv/app.private.env" \
+	-v ${WORKSPACE}/config2:/mattermost/config:rw \
+	-v ${WORKSPACE}/logs2:/mattermost/logs:rw \
+	-v ${WORKSPACE}/mmdata:/mattermost/data:rw \
+	${IMAGE_SERVER} \
+	sh -c "/mattermost/bin/mattermost server"
+
+echo "Checking node 2 is up and running"
+timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_mm2 --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server2:8065/api/v4/system/ping; do echo Waiting for mm-server2; sleep 2; done; echo mm-server2 is up"
+
+echo "Checking proxy is up and running"
+timeout --foreground 90s bash -c "until docker run --rm --quiet --name ${COMPOSE_PROJECT_NAME}_curl_proxy --net ${DOCKER_NETWORK} ${IMAGE_CURL} curl -fs http://mm-server:8065/api/v4/system/ping; do echo Waiting for proxy; sleep 2; done; echo proxy is up"