From 6f6bb4effbbe7aa80bf4d66ab752bb758d8bd0ab Mon Sep 17 00:00:00 2001 From: Sergio Date: Fri, 11 Oct 2024 14:31:42 +0200 Subject: [PATCH 1/9] Add build-any-ib GH action --- .github/workflows/build-any-ib.yml | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/build-any-ib.yml diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml new file mode 100644 index 00000000..27898171 --- /dev/null +++ b/.github/workflows/build-any-ib.yml @@ -0,0 +1,64 @@ +--- +# Launch the build-any-ib job in Jenkins +name: Build any IB + +'on': + workflow_dispatch: + inputs: + package_name: + type: string + description: Name of the package to build + default: 'O2' + alidist_slug: + type: string + description: Alidist version to use for the package (group/repo[@branch]) + default: 'alisw/alidist@master' + architecture: + type: choice + description: Architecture to build the package for + default: 'latest' + options: + +permissions: {} + +jobs: + build-any-ib: + runs-on: ubuntu-latest + + env: + # --- Jenkins and SSO params --- + JENKINS_URL: ${{ secrets.JENKINS_URL }} + SSO_AUTH_URL: ${{ secrets.SSO_AUTH_URL }} + CLIENT_ID: ${{ secrets.SSO_JENKINS_API_CLIENT_ID }} + CLIENT_SECRET: ${{ secrets.SSO_JENKINS_API_CLIENT_SECRET }} + TARGET_APP: ${{ secrets.SSO_JENKINS_API_TARGET_APP }} + JOB_NAME: 'build-any-ib' + + # --- build-any-ib build params --- + # ALIBUILD_SLUG: ${{ inputs.alibuild_slug }} + ALIDIST_SLUG: ${{ inputs.alidist_slug }} + ARCHITECTURE: ${{ inputs.architecture }} + PACKAGE_NAME: ${{ inputs.package_name }} + # OVERRIDE_TAGS: ${{ inputs.override_tags }} + # OVERRIDE_VERSIONS: ${{ inputs.override_versions }} + # DEFAULTS: "o2" + # PUBLISH_BUILDS: "true" + # USE_REMOTE_STORE: "true" + + steps: + - name: Launch the build-any-ib job in Jenkins + run: | + # Login against SSO + TOKEN="$(curl --location -X POST "$SSO_AUTH_URL" \ + --header 'Content-Type: application/x-www-form-urlencoded' \ + --data-urlencode 'grant_type=client_credentials' \ + --data-urlencode "client_id=$CLIENT_ID" \ + --data-urlencode "client_secret=$CLIENT_SECRET" \ + --data-urlencode "audience=$TARGET_APP" | jq -r '.access_token')" + + # Trigger the Jenkins job + curl "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" \ + -H "Authorization: Bearer $TOKEN" \ + --data "PACKAGE_NAME=$PACKAGE_NAME" \ + --data "ALIDIST_SLUG=$ALIDIST_SLUG" \ + --data "ARCHITECTURE=$ARCHITECTURE" From a33ae91098b214568be3aa8ad85c4a0409f4d5d6 Mon Sep 17 00:00:00 2001 From: Sergio Date: Fri, 11 Oct 2024 14:40:29 +0200 Subject: [PATCH 2/9] Add current archs --- .github/workflows/build-any-ib.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index 27898171..de547f9e 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -16,8 +16,16 @@ name: Build any IB architecture: type: choice description: Architecture to build the package for - default: 'latest' + default: 'slc9_x86-64' options: + - "slc9_x86-64" + - "slc8_x86-64" + - "slc7_x86-64" + - "slc6_x86-64" + - "ubuntu2004_x86-64" + - "ubuntu2204_x86-64" + - "osx_x86-64" + - "slc7_aarch64" permissions: {} From df79da5eca0e3ce954f2951945c7c34745675dcc Mon Sep 17 00:00:00 2001 From: Sergio Date: Tue, 5 Nov 2024 15:58:23 +0100 Subject: [PATCH 3/9] Update list --- .github/workflows/build-any-ib.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index de547f9e..2e899b14 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -19,13 +19,12 @@ name: Build any IB default: 'slc9_x86-64' options: - "slc9_x86-64" + - "slc9_aarch64" - "slc8_x86-64" - "slc7_x86-64" - - "slc6_x86-64" - "ubuntu2004_x86-64" - "ubuntu2204_x86-64" - "osx_x86-64" - - "slc7_aarch64" permissions: {} From a04144a1b0f3105ff5cfec608eaca9e13878128d Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 12:48:25 +0100 Subject: [PATCH 4/9] Fix CI --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa4b5bf3..00907609 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pyyaml boto3==1.23.10; python_version >= '3.6' and python_version < '3.8' # Newer Pythons need a newer boto3. boto3; python_version >= '3.8' -Twisted[services]==18.9.0 +Twisted[services] klein[services] python-ldap[services] # For gql; by default it pulls in a typing-extensions version that isn't From 77913e7f70bfe428c283001efda5cf58390614c3 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 12:54:22 +0100 Subject: [PATCH 5/9] Another --- .github/workflows/build-any-ib.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index 2e899b14..72ab6bfb 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -2,21 +2,21 @@ # Launch the build-any-ib job in Jenkins name: Build any IB -'on': +"on": workflow_dispatch: inputs: package_name: type: string description: Name of the package to build - default: 'O2' + default: "O2" alidist_slug: type: string description: Alidist version to use for the package (group/repo[@branch]) - default: 'alisw/alidist@master' + default: "alisw/alidist@master" architecture: type: choice description: Architecture to build the package for - default: 'slc9_x86-64' + default: "slc9_x86-64" options: - "slc9_x86-64" - "slc9_aarch64" @@ -24,6 +24,7 @@ name: Build any IB - "slc7_x86-64" - "ubuntu2004_x86-64" - "ubuntu2204_x86-64" + - "ubuntu2404_x86-64" - "osx_x86-64" permissions: {} @@ -39,7 +40,7 @@ jobs: CLIENT_ID: ${{ secrets.SSO_JENKINS_API_CLIENT_ID }} CLIENT_SECRET: ${{ secrets.SSO_JENKINS_API_CLIENT_SECRET }} TARGET_APP: ${{ secrets.SSO_JENKINS_API_TARGET_APP }} - JOB_NAME: 'build-any-ib' + JOB_NAME: "build-any-ib" # --- build-any-ib build params --- # ALIBUILD_SLUG: ${{ inputs.alibuild_slug }} From 1d63721458447be0fb6703db16de9f6a1056093e Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 13:58:01 +0100 Subject: [PATCH 6/9] Wait for completion --- .github/workflows/build-any-ib.yml | 38 ++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index 72ab6bfb..ed721090 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -54,7 +54,7 @@ jobs: # USE_REMOTE_STORE: "true" steps: - - name: Launch the build-any-ib job in Jenkins + - name: Launch the build-any-ib job in Jenkins and wait for completion run: | # Login against SSO TOKEN="$(curl --location -X POST "$SSO_AUTH_URL" \ @@ -64,9 +64,39 @@ jobs: --data-urlencode "client_secret=$CLIENT_SECRET" \ --data-urlencode "audience=$TARGET_APP" | jq -r '.access_token')" - # Trigger the Jenkins job - curl "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" \ + # Trigger the Jenkins job and get the queue item location + QUEUE_URL=$(curl -w "%{redirect_url}" -s -o /dev/null "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" \ -H "Authorization: Bearer $TOKEN" \ --data "PACKAGE_NAME=$PACKAGE_NAME" \ --data "ALIDIST_SLUG=$ALIDIST_SLUG" \ - --data "ARCHITECTURE=$ARCHITECTURE" + --data "ARCHITECTURE=$ARCHITECTURE") + + # Poll the queue item until we get the actual job URL + while true; do + QUEUE_RESPONSE=$(curl -s "$QUEUE_URL/api/json" -H "Authorization: Bearer $TOKEN") + if echo "$QUEUE_RESPONSE" | jq -e '.executable.url' > /dev/null; then + BUILD_URL=$(echo "$QUEUE_RESPONSE" | jq -r '.executable.url') + break + fi + echo "Waiting for job to start..." + sleep 10 + done + + echo "Job started at: $BUILD_URL" + + while true; do + JOB_STATUS=$(curl -s "$BUILD_URL/api/json" -H "Authorization: Bearer $TOKEN" | jq -r '.result') + if [ "$JOB_STATUS" = "SUCCESS" ]; then + echo "Job completed successfully!" + exit 0 + elif [ "$JOB_STATUS" = "FAILURE" ] || [ "$JOB_STATUS" = "ABORTED" ]; then + echo "::error::Jenkins job failed with status: $JOB_STATUS" + exit 1 + elif [ "$JOB_STATUS" = "null" ]; then + echo "Job is still running..." + sleep 30 + else + echo "::error::Unknown Jenkins job status: $JOB_STATUS" + exit 1 + fi + done From eabaadd4b33ce199e97dd99cd66710b1858fd469 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 14:27:03 +0100 Subject: [PATCH 7/9] Some bugs --- .github/workflows/build-any-ib.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index ed721090..83286ea8 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -56,6 +56,7 @@ jobs: steps: - name: Launch the build-any-ib job in Jenkins and wait for completion run: | + set -euo pipefail # Login against SSO TOKEN="$(curl --location -X POST "$SSO_AUTH_URL" \ --header 'Content-Type: application/x-www-form-urlencoded' \ @@ -69,11 +70,20 @@ jobs: -H "Authorization: Bearer $TOKEN" \ --data "PACKAGE_NAME=$PACKAGE_NAME" \ --data "ALIDIST_SLUG=$ALIDIST_SLUG" \ - --data "ARCHITECTURE=$ARCHITECTURE") + --data "ARCHITECTURE=$ARCHITECTURE") | grep -i "Location:" | awk '{print $2}' | tr -d '\r') + + # Remove any trailing whitespace, change http to https + QUEUE_URL=$(echo "$QUEUE_URL" | xargs) + QUEUE_URL=$(echo "$QUEUE_URL" | sed 's/http/https/') + + if [ -z "$QUEUE_URL" ]; then + echo "::error::Failed to get queue URL from Jenkins" + exit 1 + fi # Poll the queue item until we get the actual job URL while true; do - QUEUE_RESPONSE=$(curl -s "$QUEUE_URL/api/json" -H "Authorization: Bearer $TOKEN") + QUEUE_RESPONSE=$(curl -L -s "$QUEUE_URL/api/json" -H "Authorization: Bearer $TOKEN") if echo "$QUEUE_RESPONSE" | jq -e '.executable.url' > /dev/null; then BUILD_URL=$(echo "$QUEUE_RESPONSE" | jq -r '.executable.url') break @@ -84,6 +94,8 @@ jobs: echo "Job started at: $BUILD_URL" + BUILD_URL=$(echo "$BUILD_URL" | sed 's/alijenkins/alijenkins-api/') + while true; do JOB_STATUS=$(curl -s "$BUILD_URL/api/json" -H "Authorization: Bearer $TOKEN" | jq -r '.result') if [ "$JOB_STATUS" = "SUCCESS" ]; then From b0fc7fafb95e8e7518187bbd657640b9a7800c00 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 14:30:37 +0100 Subject: [PATCH 8/9] Fix --- .github/workflows/build-any-ib.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index 83286ea8..cd3251c1 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -70,11 +70,8 @@ jobs: -H "Authorization: Bearer $TOKEN" \ --data "PACKAGE_NAME=$PACKAGE_NAME" \ --data "ALIDIST_SLUG=$ALIDIST_SLUG" \ - --data "ARCHITECTURE=$ARCHITECTURE") | grep -i "Location:" | awk '{print $2}' | tr -d '\r') - - # Remove any trailing whitespace, change http to https - QUEUE_URL=$(echo "$QUEUE_URL" | xargs) - QUEUE_URL=$(echo "$QUEUE_URL" | sed 's/http/https/') + --data "ARCHITECTURE=$ARCHITECTURE" | grep -i "Location:" | awk '{print $2}' | tr -d '\r' |\ + xargs | sed 's/http/https/') # Remove any trailing whitespace, change http to https if [ -z "$QUEUE_URL" ]; then echo "::error::Failed to get queue URL from Jenkins" From eff3dddc39bc05fa2d7ddc173e20a9c184c16167 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 13 Jan 2025 14:31:59 +0100 Subject: [PATCH 9/9] ... --- .github/workflows/build-any-ib.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-any-ib.yml b/.github/workflows/build-any-ib.yml index cd3251c1..1db868fa 100644 --- a/.github/workflows/build-any-ib.yml +++ b/.github/workflows/build-any-ib.yml @@ -91,7 +91,7 @@ jobs: echo "Job started at: $BUILD_URL" - BUILD_URL=$(echo "$BUILD_URL" | sed 's/alijenkins/alijenkins-api/') + BUILD_URL=${BUILD_URL//alijenkins/alijenkins-api} while true; do JOB_STATUS=$(curl -s "$BUILD_URL/api/json" -H "Authorization: Bearer $TOKEN" | jq -r '.result')