From 500de30ffe6538d563231093924b3cbaed7d6853 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 8 Mar 2023 16:06:51 -0500 Subject: [PATCH] Refactor image publishing and PR validation workflows (#43) * Add ansible-builder configuration and Github Action workflows for image generation * Add label and reset PR workflows * Update sub-workflow to streamline build contexts * Create explicit workflow matrix for publishing images * Update builder.sh script to segment context folders * Update and add execution profiles * Update .gitignore for pytest caches * Disable legacy publishing workflow and add deprecation note Signed-off-by: Webster Mudge --- .github/workflows/build_push_image.yml | 121 ++++++++++++++++++ .github/workflows/label_pr.yml | 54 ++++++++ .github/workflows/publish_image.yml | 28 ++++ .../publish_image_github_packages.yml | 119 ----------------- .github/workflows/publish_images.yml | 22 ++++ .github/workflows/reset_pr.yml | 26 ++++ .github/workflows/validate_pr.yml | 82 ++++++++++++ .gitignore | 2 + NOTES.md | 40 ++++++ build.sh | 2 + builder/BUILDING.md | 43 +++++++ {payload => builder}/bashrc | 0 builder/build.sh | 49 +++++++ builder/devel/bindep.txt | 0 .../ansible.yml => builder/devel/galaxy.yml | 15 +-- builder/devel/requirements.txt | 9 ++ builder/ee-aws.yml | 40 ++++++ builder/ee-azure.yml | 39 ++++++ builder/ee-base.yml | 34 +++++ builder/ee-full.yml | 48 +++++++ builder/ee-gcp.yml | 34 +++++ {payload => builder}/env/envvars | 12 +- {payload => builder}/env/settings | 0 {payload => builder}/inventory/hosts | 0 builder/main/bindep.txt | 0 builder/main/galaxy.yml | 19 +++ builder/main/requirements.txt | 9 ++ {payload/deps => builder/repo}/azure-cli.repo | 0 .../repo}/google-cloud-sdk.repo | 0 {payload/deps => builder/repo}/hashicorp.repo | 0 .../deps => builder/repo}/kubernetes.repo | 0 payload/deps/python_aws.txt | 7 - payload/deps/python_azure.txt | 7 - payload/deps/python_base.txt | 26 ---- payload/deps/python_gcp.txt | 2 - payload/deps/python_secondary.txt | 2 - 36 files changed, 713 insertions(+), 178 deletions(-) create mode 100644 .github/workflows/build_push_image.yml create mode 100644 .github/workflows/label_pr.yml create mode 100644 .github/workflows/publish_image.yml delete mode 100644 .github/workflows/publish_image_github_packages.yml create mode 100644 .github/workflows/publish_images.yml create mode 100644 .github/workflows/reset_pr.yml create mode 100644 .github/workflows/validate_pr.yml create mode 100644 .gitignore create mode 100644 NOTES.md create mode 100644 builder/BUILDING.md rename {payload => builder}/bashrc (100%) create mode 100755 builder/build.sh create mode 100644 builder/devel/bindep.txt rename payload/deps/ansible.yml => builder/devel/galaxy.yml (57%) create mode 100644 builder/devel/requirements.txt create mode 100644 builder/ee-aws.yml create mode 100644 builder/ee-azure.yml create mode 100644 builder/ee-base.yml create mode 100644 builder/ee-full.yml create mode 100644 builder/ee-gcp.yml rename {payload => builder}/env/envvars (53%) rename {payload => builder}/env/settings (100%) rename {payload => builder}/inventory/hosts (100%) create mode 100644 builder/main/bindep.txt create mode 100644 builder/main/galaxy.yml create mode 100644 builder/main/requirements.txt rename {payload/deps => builder/repo}/azure-cli.repo (100%) rename {payload/deps => builder/repo}/google-cloud-sdk.repo (100%) rename {payload/deps => builder/repo}/hashicorp.repo (100%) rename {payload/deps => builder/repo}/kubernetes.repo (100%) delete mode 100644 payload/deps/python_aws.txt delete mode 100644 payload/deps/python_azure.txt delete mode 100644 payload/deps/python_base.txt delete mode 100644 payload/deps/python_gcp.txt delete mode 100644 payload/deps/python_secondary.txt diff --git a/.github/workflows/build_push_image.yml b/.github/workflows/build_push_image.yml new file mode 100644 index 0000000..2f4df89 --- /dev/null +++ b/.github/workflows/build_push_image.yml @@ -0,0 +1,121 @@ +--- +# See https://github.com/cloin/ee-builds + +name: Build and push Execution Environment image to Packages + +on: + workflow_call: + inputs: + BRANCH: + description: 'Upstream collections branch' + required: true + type: string + PROFILE: + description: 'Execution profile' + required: true + type: string + outputs: + image-paths: + description: 'all the constructed and pushed image paths' + value: ${{ jobs.build-push.outputs.registry-paths }} + image-sha: + description: 'the SHA tagged image path' + value: ghcr.io/${{ github.repository }}:sha-${{ jobs.build-push.outputs.sha-short }} + +permissions: + contents: read + packages: write + +jobs: + build-push: + runs-on: ubuntu-latest + outputs: + sha-short: ${{ steps.sha.outputs.sha-short }} + registry-paths: ${{ steps.push-image.outputs.registry-paths }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install ansible-builder + run: pip install ansible-builder==1.2.0 + + - name: Construct image metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + flavor: | + latest=auto + prefix=${{ inputs.PROFILE }}-,onlatest=true + tags: | + type=sha,prefix=${{ inputs.PROFILE }}-sha- + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Construct short SHA + id: sha + run: echo "sha-short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Set up context + run: | + mkdir -p ${{ inputs.PROFILE }} + cp -R builder/${{ inputs.BRANCH }}/* ${{ inputs.PROFILE }} + cp builder/ee-${{ inputs.PROFILE }}.yml ${{ inputs.PROFILE }}/execution-environment.yml + cp -R builder/env builder/inventory builder/repo builder/bashrc ${{ inputs.PROFILE }} + + - name: Create Containerfile + run: | + ansible-builder create --context ${{ inputs.PROFILE }} --file ${{ inputs.PROFILE }}/execution-environment.yml + + - name: Build image from context + uses: redhat-actions/buildah-build@v2 + with: + context: ${{ inputs.PROFILE }} + image: ${{ github.repository }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + containerfiles: | + ${{ inputs.PROFILE }}/Containerfile + + - name: Push image + id: push-image + uses: redhat-actions/push-to-registry@v2 + with: + tags: ${{ steps.meta.outputs.tags }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Print summary + working-directory: ${{ inputs.PROFILE }} + run: | + echo "## :rocket: Usage" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Image pushed to repository: [${{ steps.push-image.outputs.registry-path }}](https://${{ steps.push-image.outputs.registry-path }})" >> $GITHUB_STEP_SUMMARY + echo "> \`podman pull ${{ steps.push-image.outputs.registry-path }}\`" >> $GITHUB_STEP_SUMMARY + echo "## :pushpin: Ansible info:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "`podman run ${{ steps.push-image.outputs.registry-path }} ansible --version`" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "## :pushpin: Installed Python libraries:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "`podman run ${{ steps.push-image.outputs.registry-path }} pip freeze | sort`" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "## :pushpin: Installed collections:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "`podman run ${{ steps.push-image.outputs.registry-path }} ansible-galaxy collection list`" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "## :pushpin: Installed roles:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "`podman run ${{ steps.push-image.outputs.registry-path }} ansible-galaxy role list`" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "## :pushpin: Containerfile:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "`cat Containerfile`" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/label_pr.yml b/.github/workflows/label_pr.yml new file mode 100644 index 0000000..b69bf74 --- /dev/null +++ b/.github/workflows/label_pr.yml @@ -0,0 +1,54 @@ +--- +# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Label validated Pull Request + +on: + workflow_run: + workflows: ["Validate Pull Request"] + types: + - completed + +jobs: + label: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: Download the PR number artifact + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr_number" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); + + - name: 'Unzip artifact' + run: unzip pr_number.zip + + - name: Read the PR number + id: read + run: echo "pr_number=$(cat pr_number)" >> $GITHUB_OUTPUT + + - name: Label the PR + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: validated + number: ${{ steps.read.outputs.pr_number }} \ No newline at end of file diff --git a/.github/workflows/publish_image.yml b/.github/workflows/publish_image.yml new file mode 100644 index 0000000..f93767d --- /dev/null +++ b/.github/workflows/publish_image.yml @@ -0,0 +1,28 @@ +--- + +name: Publish a single Execution Environment image + +on: + workflow_dispatch: + inputs: + BRANCH: + description: 'Upstream collections branch' + default: 'main' + required: false + type: string + PROFILE: + description: 'Execution profile' + default: 'base' + required: false + type: string + +permissions: + contents: read + packages: write + +jobs: + publish: + uses: ./.github/workflows/build_push_image.yml + with: + BRANCH: ${{ inputs.BRANCH }} + PROFILE: ${{ inputs.PROFILE }} diff --git a/.github/workflows/publish_image_github_packages.yml b/.github/workflows/publish_image_github_packages.yml deleted file mode 100644 index 2f458c1..0000000 --- a/.github/workflows/publish_image_github_packages.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Publish Docker image -on: - release: - types: [published] -jobs: - push_to_registry: - name: Push Docker image to GitHub Packages - runs-on: ubuntu-latest - steps: - - name: Login to Github Container Registry - uses: docker/login-action@v1.8.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.PKG_PAT }} - - - name: Get Date of Base Build - id: date - uses: nanzm/get-time-action@v1.1 - with: - timeZone: 0 - format: 'YYYY-MM-DD' - - - name: Prepare Environment Variables - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - - - name: Push to GitHub Packages for Base Build - uses: docker/build-push-action@v2.3.0 - with: - tags: | - ghcr.io/${{ env.IMAGE_REPOSITORY }}:base-${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}:base-latest - push: true - build-args: | - BUILD_DATE=${{ steps.date.outputs.time }} - BUILD_TAG=base-${{ env.RELEASE_VERSION }} - CDPY=true - - - name: Get Time for Full Build - id: fulltime - uses: nanzm/get-time-action@v1.1 - - - name: Push to GitHub Packages for Full Build - uses: docker/build-push-action@v2.3.0 - with: - push: true - build-args: | - BASE_IMAGE_URI=ghcr.io/${{ env.IMAGE_REPOSITORY }} - BASE_IMAGE_TAG=base-${{ env.RELEASE_VERSION }} - BUILD_TAG=full-${{ env.RELEASE_VERSION }} - AWS=true - GCLOUD=true - AZURE=true - KUBECTL=true - CDPY=true - CACHE_TIME=${{ steps.fulltime.outputs.time }} - tags: | - ghcr.io/${{ env.IMAGE_REPOSITORY }}:full-${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}:full-latest - - - name: Get Time for AWS Build - id: awstime - uses: nanzm/get-time-action@v1.1 - - - name: Push to GitHub Packages for AWS only - uses: docker/build-push-action@v2.3.0 - with: - push: true - build-args: | - BASE_IMAGE_URI=ghcr.io/${{ env.IMAGE_REPOSITORY }} - BASE_IMAGE_TAG=base-${{ env.RELEASE_VERSION }} - BUILD_TAG=aws-${{ env.RELEASE_VERSION }} - AWS=true - CDPY=true - CACHE_TIME=${{ steps.awstime.outputs.time }} - tags: | - ghcr.io/${{ env.IMAGE_REPOSITORY }}:aws-${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}:aws-latest - - - - name: Get Time for GCP Build - id: gcptime - uses: nanzm/get-time-action@v1.1 - - - name: Push to GitHub Packages for GCP only - uses: docker/build-push-action@v2.3.0 - with: - push: true - build-args: | - BASE_IMAGE_URI=ghcr.io/${{ env.IMAGE_REPOSITORY }} - BASE_IMAGE_TAG=base-${{ env.RELEASE_VERSION }} - BUILD_TAG=gcp-${{ env.RELEASE_VERSION }} - GCLOUD=true - CDPY=true - CACHE_TIME=${{ steps.gcptime.outputs.time }} - tags: | - ghcr.io/${{ env.IMAGE_REPOSITORY }}:gcp-${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}:gcp-latest - - - name: Get Time for Azure Build - id: azuretime - uses: nanzm/get-time-action@v1.1 - - - name: Push to GitHub Packages for Azure only - uses: docker/build-push-action@v2.3.0 - with: - push: true - build-args: | - BASE_IMAGE_URI=ghcr.io/${{ env.IMAGE_REPOSITORY }} - BASE_IMAGE_TAG=base-${{ env.RELEASE_VERSION }} - BUILD_TAG=azure-${{ env.RELEASE_VERSION }} - AZURE=true - CDPY=true - CACHE_TIME=${{ steps.azuretime.outputs.time }} - tags: | - ghcr.io/${{ env.IMAGE_REPOSITORY }}:azure-${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}:azure-latest diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml new file mode 100644 index 0000000..9f5d8dd --- /dev/null +++ b/.github/workflows/publish_images.yml @@ -0,0 +1,22 @@ +--- + +name: Publish Execution Environment images + +on: + release: + types: [published] + +permissions: + contents: read + packages: write + +jobs: + publish: + uses: ./.github/workflows/build_push_image.yml + strategy: + fail-fast: false + matrix: + ee_profile: ["full", "base", "aws", "azure", "gcp"] + with: + BRANCH: main + PROFILE: ${{ matrix.ee_profile }} diff --git a/.github/workflows/reset_pr.yml b/.github/workflows/reset_pr.yml new file mode 100644 index 0000000..2663ea7 --- /dev/null +++ b/.github/workflows/reset_pr.yml @@ -0,0 +1,26 @@ +--- + +name: Reset Pull Request validation label + +on: + pull_request_target: + types: + - reopened + - synchronize + - ready_for_review + branches: + - 'release/**' + - 'devel' + - 'devel-pvc-base' + +jobs: + reset: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Reset the PR label + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: validated \ No newline at end of file diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml new file mode 100644 index 0000000..5a3e19e --- /dev/null +++ b/.github/workflows/validate_pr.yml @@ -0,0 +1,82 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate Pull Request + +on: + pull_request: + branches: + - 'release/**' + - 'devel' + - 'devel-pvc-update' + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ee_profile: ["full", "base", "aws", "azure", "gcp"] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install ansible-builder + run: pip install ansible-builder==1.2.0 + + - name: Set up context + run: | + mkdir -p ${{ matrix.ee_profile }} + cp -R builder/devel/* ${{ matrix.ee_profile }} + cp builder/ee-${{ matrix.ee_profile }}.yml ${{ matrix.ee_profile }}/execution-environment.yml + cp -R builder/env builder/inventory builder/repo builder/bashrc ${{ matrix.ee_profile }} + + - name: Create Containerfile + run: | + ansible-builder create --context ${{ matrix.ee_profile }} --file ${{ matrix.ee_profile }}/execution-environment.yml + + - name: Upload Containerfile + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.ee_profile }}-Containerfile + path: ${{ matrix.ee_profile }}/Containerfile + + - name: Build image from context + uses: redhat-actions/buildah-build@v2 + with: + context: ${{ matrix.ee_profile }} + image: ${{ github.repository }} + containerfiles: | + ${{ matrix.ee_profile }}/Containerfile + + record: + needs: validate + runs-on: ubuntu-latest + steps: + # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + mkdir -p ./pr + echo $PR_NUMBER > ./pr/pr_number + + - name: Upload the PR number + uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e84814e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +builder/contexts +.pytest_cache diff --git a/NOTES.md b/NOTES.md new file mode 100644 index 0000000..ba722e4 --- /dev/null +++ b/NOTES.md @@ -0,0 +1,40 @@ +# Ansible Builder - Execution Environment + +# Ansible Builder + +Rebuild the `ansible-builder` image to get to the latest -- `1.2.0` -- version; the current image is *10 months old* and is release `v1.0.0`. + +1. Clone cloudera-labs/ansible-builder +1. Check out release 1.2.0 branch +1. Build the image + ```bash + podman build --rm \ + -t ghcr.io/cloudera-labs/ansible-builder:1.2.0 \ + -f Containerfile \ + . + ``` +1. (Push the image to a remote registry if not using locally) + ```bash + podman login ghcr.io + podman push ghcr.io/cloudera-labs/ansible-builder:1.2.0 + podman push ghcr.io/cloudera-labs/ansible-builder:latest + ``` + +# Build cldr-runner + +(Assumes a minimal Ansible setup with `ansible-builder` installed in the `venv`.) + +1. Build `cldr-runner` using the instructions in [BUILDING](builder/BUILDING.md) document. +1. (Optionally) Push the image to a remote registry if not using locally + +# Run the playbooks + +These images are [Execution Environments](https://ansible-builder.readthedocs.io/en/stable/definition/#) and can be used as the `provision-isolation` container for `ansible-runner`. You can run commands directly from within the container itself, such as: + +```bash +# Run inside the container ala cloudera-deploy V1 +podman run -it ghcr.io/cloudera-labs/cldr-runner:latest /bin/bash + +# Or locally with a specific tagged image +podman run -it localhost/cldr-runner:base-main /bin/bash +``` diff --git a/build.sh b/build.sh index d44188e..560d811 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "DEPRECATED! Please build using the instructions in builder/BUILDING.md" + source $(cd $(dirname $0); pwd -L)/common.sh display_usage() { diff --git a/builder/BUILDING.md b/builder/BUILDING.md new file mode 100644 index 0000000..4661917 --- /dev/null +++ b/builder/BUILDING.md @@ -0,0 +1,43 @@ +# Building a Local cldr-runner Image + +`cldr-runner` is an [`ansible-runner`](https://github.com/ansible/ansible-runner) image constructed using the [`ansible-builder`](https://github.com/ansible/ansible-builder) application. + +`cldr-runner` has five core profiles: + +* `base` - the Cloudera collections and their dependencies. +* `aws` - builds on `base` and adds the AWS dependencies. +* `azure` - builds on `base` and adds the Azure dependencies. +* `gcp` - builds on `base` and adds the Google Cloud dependencies. +* `full` - includes all of the above plus additional applications for using the image for development and general execution, e.g. `nano`. + +Each profile is designed to draw from either the `main` or `devel` branches of the various Cloudera collections; the `ansible-builder` _context_ is a constructed with either `builder/main` or `builder/devel`. It then runs with the appropriate _Execution Environment_ configuration file, e.g. `builder/ee-base.yml` within this _context_. + +To coordinate this process at the CLI, you can run `builder/build.sh` with the appropriate parameters, and the script will copy the proper resources and return next step instructions. + +```bash +$> builder/build.sh main base + +Context created! Please run the following to build the Execution Environment image: + + ansible-builder build -c builder/contexts/base-main -f builder/contexts/base-main/execution-environment.yml -t cldr-runner:base-main --build-arg BUILD_VER="base-main" --prune-images + +Add the '-v 3' flag if you wish to see verbose logging. +``` + +Which when run, for example: + +```bash +$> ansible-builder build -c builder/contexts/base-main -f builder/contexts/base-main/execution-environment.yml -t cldr-runner:base-main --build-arg BUILD_VER="base-main" --prune-images +``` + +Resulting in the following: + +```bash +Running command: + podman build -f builder/contexts/base-main/Containerfile -t cldr-runner:base-main --build-arg=BUILD_VER=base-main builder/contexts/base-main +Running command: + podman image prune --force +Complete! The build context can be found at: /Users/wmudge/tmp/cldr-runner/builder/contexts/base-main +``` + +Based on the `-t` (tag) parameter specified above, the resulting image is set in the local repository as `localhost/cldr-runner:base-main`. \ No newline at end of file diff --git a/payload/bashrc b/builder/bashrc similarity index 100% rename from payload/bashrc rename to builder/bashrc diff --git a/builder/build.sh b/builder/build.sh new file mode 100755 index 0000000..3efa881 --- /dev/null +++ b/builder/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +display_usage() { + echo "cldr-runner/builder/builder.sh +A tool for constructing ansible-runner based Execution Environments using ansible-builder. + +Usage: + $(basename "$0") [main|devel] [full|aws|azure|gcp|base] [--help or -h] + +Description: + Constructs the ansible-builder context for cldr-runner. + +Arguments: + main - Include the 'main' branch collections + devel - Include the 'devel' branch collections + + base - Core collections only + full - All CSPs + aws - Amazon Web Services + azure - Microsoft Azure + gcp - Google Cloud" +} +if [[ ( $1 == "--help") || $1 == "-h" ]] +then + display_usage + exit 0 +fi + +ROOT_DIR="${0%/*}" + +if [[ "$#" == 2 ]]; then + BRANCH_TYPE="${1}" + CONTEXT_TYPE="${2}" + CONTEXT_DIR="contexts/${2}-${1}" # [base|full|aws|gcp|azure]-[devel|main] + cd "${ROOT_DIR}" + mkdir -p "${CONTEXT_DIR}" + cp -R env inventory repo bashrc "${CONTEXT_DIR}" + cp -R "${BRANCH_TYPE}/" "${CONTEXT_DIR}" + cp "ee-${CONTEXT_TYPE}.yml" "${CONTEXT_DIR}/execution-environment.yml" + echo "Context created! Please run the following to build the Execution Environment image:" + echo "" + echo " ansible-builder build -c ${ROOT_DIR}/${CONTEXT_DIR} -f ${ROOT_DIR}/${CONTEXT_DIR}/execution-environment.yml -t cldr-runner:${2}-${1} --build-arg BUILD_VER=\"${2}-${1}\" --prune-images" + echo "" + echo "Add the '-v 3' flag if you wish to see verbose logging." +else + echo "Invalid number of arguments." + display_usage + exit 1 +fi diff --git a/builder/devel/bindep.txt b/builder/devel/bindep.txt new file mode 100644 index 0000000..e69de29 diff --git a/payload/deps/ansible.yml b/builder/devel/galaxy.yml similarity index 57% rename from payload/deps/ansible.yml rename to builder/devel/galaxy.yml index 47d5034..bc74e9d 100644 --- a/payload/deps/ansible.yml +++ b/builder/devel/galaxy.yml @@ -1,22 +1,19 @@ --- -# Append other Ansible Collections to this list, e.g. -# - name: https://github.com/myuser/myrepo.git -# type: git -# version: mybranch collections: - - # Cloudera collections - name: https://github.com/cloudera-labs/cloudera.cluster.git type: git + version: devel - name: https://github.com/cloudera-labs/cloudera.exe.git type: git - -# Append other roles to this list. + version: devel + roles: - - src: geerlingguy.postgresql + # Required by cloudera.cluster + - name: geerlingguy.postgresql version: 2.2.0 + # Required by cloudera.cluster # geerlingguy.mysql with fix for issue #332 - src: https://github.com/dbeech/ansible-role-mysql version: master diff --git a/builder/devel/requirements.txt b/builder/devel/requirements.txt new file mode 100644 index 0000000..9d6cac4 --- /dev/null +++ b/builder/devel/requirements.txt @@ -0,0 +1,9 @@ +# TODO Track down where these two libraries are used + +# JSON manipulation +jsonmerge +deepdiff>=5 + +# OpenStack and Kubernetes +openstacksdk +openshift diff --git a/builder/ee-aws.yml b/builder/ee-aws.yml new file mode 100644 index 0000000..aa2475c --- /dev/null +++ b/builder/ee-aws.yml @@ -0,0 +1,40 @@ +--- + +# AWS cldr-runner + +version: 1 + +build_arg_defaults: + EE_BASE_IMAGE: "quay.io/ansible/ansible-runner:stable-2.12-latest" + EE_BUILDER_IMAGE: "ghcr.io/cloudera-labs/ansible-builder:${BUILD_TAG:-latest}" + +dependencies: + galaxy: galaxy.yml + python: requirements.txt + system: bindep.txt + +additional_build_steps: + prepend: | + ARG BUILD_VER="latest" + ENV CLDR_BUILD_VER=$BUILD_VER + ENV PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" + COPY . /tmp/src + RUN mv /tmp/src/bashrc /home/runner/.bashrc && \ + mv /tmp/src/repo/*.repo /etc/yum.repos.d/ && \ + mv /tmp/src/env /tmp/src/inventory /runner && \ + /usr/bin/python3 -m pip install --upgrade pip && \ + dnf install -y terraform unzip --nodocs --setopt install_weak_deps=0 && \ + curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x /usr/local/bin/aws-iam-authenticator && \ + curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \ + unzip /tmp/awscliv2.zip -d /tmp && \ + /tmp/aws/install && \ + rm /tmp/awscliv2.zip && rm -rf /tmp/aws + append: | + ENV PATH "$PATH:/home/runner/.local/bin" + RUN alternatives --set python /usr/bin/python3 && \ + pip cache purge || echo "No Pip cache to purge" && \ + dnf clean all -y && \ + rm -rf /var/cache/yum && \ + rm -rf /var/cache/dnf && \ + rm -rf /tmp/src diff --git a/builder/ee-azure.yml b/builder/ee-azure.yml new file mode 100644 index 0000000..d2c8863 --- /dev/null +++ b/builder/ee-azure.yml @@ -0,0 +1,39 @@ +--- + +# Azure cldr-runner + +version: 1 + +build_arg_defaults: + EE_BASE_IMAGE: "quay.io/ansible/ansible-runner:stable-2.12-latest" + EE_BUILDER_IMAGE: "ghcr.io/cloudera-labs/ansible-builder:${BUILD_TAG:-latest}" + +dependencies: + galaxy: galaxy.yml + python: requirements.txt + system: bindep.txt + +additional_build_steps: + prepend: | + ARG BUILD_VER="latest" + ENV CLDR_BUILD_VER=$BUILD_VER + ENV PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" + COPY . /tmp/src + RUN mv /tmp/src/bashrc /home/runner/.bashrc && \ + mv /tmp/src/repo/*.repo /etc/yum.repos.d/ && \ + mv /tmp/src/env /tmp/src/inventory /runner && \ + /usr/bin/python3 -m pip install --upgrade pip && \ + dnf install -y terraform --nodocs --setopt install_weak_deps=0 && \ + rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + dnf download azure-cli && \ + rpm -ivh --nodeps azure-cli-*.rpm && \ + rm -f azure-cli-*.rpm && \ + pip install azure-cli-core==2.30.0 --upgrade + append: | + ENV PATH "$PATH:/home/runner/.local/bin" + RUN alternatives --set python /usr/bin/python3 && \ + pip cache purge || echo "No Pip cache to purge" && \ + dnf clean all -y && \ + rm -rf /var/cache/yum && \ + rm -rf /var/cache/dnf && \ + rm -rf /tmp/src diff --git a/builder/ee-base.yml b/builder/ee-base.yml new file mode 100644 index 0000000..81e91ef --- /dev/null +++ b/builder/ee-base.yml @@ -0,0 +1,34 @@ +--- + +# Base cldr-runner + +version: 1 + +build_arg_defaults: + EE_BASE_IMAGE: "quay.io/ansible/ansible-runner:stable-2.12-latest" + EE_BUILDER_IMAGE: "ghcr.io/cloudera-labs/ansible-builder:latest" + +dependencies: + galaxy: galaxy.yml + python: requirements.txt + system: bindep.txt + +additional_build_steps: + prepend: | + ARG BUILD_VER="latest" + ENV CLDR_BUILD_VER=$BUILD_VER + ENV PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" + COPY . /tmp/src + RUN mv /tmp/src/bashrc /home/runner/.bashrc && \ + mv /tmp/src/repo/*.repo /etc/yum.repos.d/ && \ + mv /tmp/src/env /tmp/src/inventory /runner && \ + dnf clean all && \ + /usr/bin/python3 -m pip install --upgrade pip && \ + /usr/bin/python3 -m pip cache purge + append: | + RUN alternatives --set python /usr/bin/python3 && \ + pip cache purge || echo "No Pip cache to purge" && \ + dnf clean all -y && \ + rm -rf /var/cache/yum && \ + rm -rf /var/cache/dnf && \ + rm -rf /tmp/src diff --git a/builder/ee-full.yml b/builder/ee-full.yml new file mode 100644 index 0000000..67833e5 --- /dev/null +++ b/builder/ee-full.yml @@ -0,0 +1,48 @@ +--- + +# Full cldr-runner (aka kitchen sink) + +version: 1 + +build_arg_defaults: + EE_BASE_IMAGE: "quay.io/ansible/ansible-runner:stable-2.12-latest" + EE_BUILDER_IMAGE: "ghcr.io/cloudera-labs/ansible-builder:${BUILD_TAG:-latest}" + +dependencies: + galaxy: galaxy.yml + python: requirements.txt + system: bindep.txt + +additional_build_steps: + prepend: | + ARG BUILD_VER="latest" + ENV CLDR_BUILD_VER=$BUILD_VER + ENV PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" + COPY . /tmp/src + RUN mv /tmp/src/bashrc /home/runner/.bashrc && \ + mv /tmp/src/repo/*.repo /etc/yum.repos.d/ && \ + mv /tmp/src/env /tmp/src/inventory /runner && \ + /usr/bin/python3 -m pip install --upgrade pip && \ + rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + dnf install -y terraform kubectl vim nano git unzip --nodocs --setopt install_weak_deps=0 && \ + curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x /usr/local/bin/aws-iam-authenticator && \ + curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \ + unzip /tmp/awscliv2.zip -d /tmp && \ + /tmp/aws/install && \ + rm /tmp/awscliv2.zip && rm -rf /tmp/aws \ + dnf clean all && \ + dnf install -y google-cloud-sdk && \ + dnf download azure-cli && \ + rpm -ivh --nodeps azure-cli-*.rpm && \ + rm -f azure-cli-*.rpm && \ + pip install azure-cli-core==2.30.0 --upgrade && \ + /usr/bin/python3 -m pip cache purge + append: | + ENV PATH "$PATH:/home/runner/.local/bin" + RUN alternatives --set python /usr/bin/python3 && \ + pip cache purge || echo "No Pip cache to purge" && \ + dnf clean all -y && \ + rm -rf /var/cache/yum && \ + rm -rf /var/cache/dnf && \ + rm -rf /tmp/src diff --git a/builder/ee-gcp.yml b/builder/ee-gcp.yml new file mode 100644 index 0000000..badb57f --- /dev/null +++ b/builder/ee-gcp.yml @@ -0,0 +1,34 @@ +--- + +# Google Cloud cldr-runner + +version: 1 + +build_arg_defaults: + EE_BASE_IMAGE: "quay.io/ansible/ansible-runner:stable-2.12-latest" + EE_BUILDER_IMAGE: "ghcr.io/cloudera-labs/ansible-builder:${BUILD_TAG:-latest}" + +dependencies: + galaxy: galaxy.yml + python: requirements.txt + system: bindep.txt + +additional_build_steps: + prepend: | + ARG BUILD_VER="latest" + ENV CLDR_BUILD_VER=$BUILD_VER + ENV PKGMGR_OPTS="--nodocs --setopt install_weak_deps=0" + COPY . /tmp/src + RUN mv /tmp/src/bashrc /home/runner/.bashrc && \ + mv /tmp/src/repo/*.repo /etc/yum.repos.d/ && \ + mv /tmp/src/env /tmp/src/inventory /runner && \ + /usr/bin/python3 -m pip install --upgrade pip && \ + dnf install -y terraform google-cloud-sdk --nodocs --setopt install_weak_deps=0 + append: | + ENV PATH "$PATH:/home/runner/.local/bin" + RUN alternatives --set python /usr/bin/python3 && \ + pip cache purge || echo "No Pip cache to purge" && \ + dnf clean all -y && \ + rm -rf /var/cache/yum && \ + rm -rf /var/cache/dnf && \ + rm -rf /tmp/src diff --git a/payload/env/envvars b/builder/env/envvars similarity index 53% rename from payload/env/envvars rename to builder/env/envvars index 3dfaf1a..4d7417a 100644 --- a/payload/env/envvars +++ b/builder/env/envvars @@ -1,11 +1,11 @@ --- -ANSIBLE_LOG_PATH=/home/runner/.ansible/log/ansible-runner.log -ANSIBLE_INVENTORY="inventory" -ANSIBLE_CALLBACK_WHITELIST="ansible.posix.profile_tasks" -ANSIBLE_GATHERING="smart" -ANSIBLE_DEPRECATION_WARNINGS=false -ANSIBLE_SSH_RETRIES=10 +#ANSIBLE_LOG_PATH: /home/runner/.ansible/log/ansible-runner.log +ANSIBLE_INVENTORY: "inventory" +ANSIBLE_CALLBACKS_ENABLED: "ansible.posix.profile_tasks" +ANSIBLE_GATHERING: "smart" +# ANSIBLE_DEPRECATION_WARNINGS: false +ANSIBLE_SSH_RETRIES: 10 # Users and downstream images are expected to set the collections and role paths # The defaults are set to a non-standard location to keep the runner neutral, diff --git a/payload/env/settings b/builder/env/settings similarity index 100% rename from payload/env/settings rename to builder/env/settings diff --git a/payload/inventory/hosts b/builder/inventory/hosts similarity index 100% rename from payload/inventory/hosts rename to builder/inventory/hosts diff --git a/builder/main/bindep.txt b/builder/main/bindep.txt new file mode 100644 index 0000000..e69de29 diff --git a/builder/main/galaxy.yml b/builder/main/galaxy.yml new file mode 100644 index 0000000..6912e24 --- /dev/null +++ b/builder/main/galaxy.yml @@ -0,0 +1,19 @@ +--- + +collections: + - name: https://github.com/cloudera-labs/cloudera.cluster.git + type: git + version: main + - name: https://github.com/cloudera-labs/cloudera.exe.git + type: git + version: main + +roles: + # Required by cloudera.cluster + - name: geerlingguy.postgresql + version: 2.2.0 + + # Required by cloudera.cluster + # geerlingguy.mysql with fix for issue #332 + - src: https://github.com/dbeech/ansible-role-mysql + version: master diff --git a/builder/main/requirements.txt b/builder/main/requirements.txt new file mode 100644 index 0000000..9d6cac4 --- /dev/null +++ b/builder/main/requirements.txt @@ -0,0 +1,9 @@ +# TODO Track down where these two libraries are used + +# JSON manipulation +jsonmerge +deepdiff>=5 + +# OpenStack and Kubernetes +openstacksdk +openshift diff --git a/payload/deps/azure-cli.repo b/builder/repo/azure-cli.repo similarity index 100% rename from payload/deps/azure-cli.repo rename to builder/repo/azure-cli.repo diff --git a/payload/deps/google-cloud-sdk.repo b/builder/repo/google-cloud-sdk.repo similarity index 100% rename from payload/deps/google-cloud-sdk.repo rename to builder/repo/google-cloud-sdk.repo diff --git a/payload/deps/hashicorp.repo b/builder/repo/hashicorp.repo similarity index 100% rename from payload/deps/hashicorp.repo rename to builder/repo/hashicorp.repo diff --git a/payload/deps/kubernetes.repo b/builder/repo/kubernetes.repo similarity index 100% rename from payload/deps/kubernetes.repo rename to builder/repo/kubernetes.repo diff --git a/payload/deps/python_aws.txt b/payload/deps/python_aws.txt deleted file mode 100644 index c64f4a3..0000000 --- a/payload/deps/python_aws.txt +++ /dev/null @@ -1,7 +0,0 @@ -# AWS collection requirements - -# Pinned to community.aws==3.0.1 (set in cloudera.exe) --r https://raw.githubusercontent.com/ansible-collections/community.aws/3.0.1/requirements.txt - -# Pinned to amazon.aws==3.0.0 (set in cloudera.exe) --r https://raw.githubusercontent.com/ansible-collections/amazon.aws/3.0.0/requirements.txt \ No newline at end of file diff --git a/payload/deps/python_azure.txt b/payload/deps/python_azure.txt deleted file mode 100644 index c1488cc..0000000 --- a/payload/deps/python_azure.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Azure Collection requirements, as distinct from Azure CLI -# Pinned to az.collection==1.11.0 (set in cloudera.exe) --r https://raw.githubusercontent.com/ansible-collections/azure/v1.11.0/requirements-azure.txt - -# NetApp Collection requirements -# Pinned to netapp.azure==21.10.0 (set in cloudera.exe) --r https://raw.githubusercontent.com/ansible-collections/netapp.azure/21.10.0/requirements.txt diff --git a/payload/deps/python_base.txt b/payload/deps/python_base.txt deleted file mode 100644 index 54168ba..0000000 --- a/payload/deps/python_base.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Python internals -wheel - -# CVE avoidance -requests[security]>=2.27.1 -urllib3>1.24.2 -cryptography>=2.3.1 - -# Ansible -jmespath -netaddr - -# Json manipulation -jsonmerge -deepdiff>=5 - -# OpenStack and Kubernetes -openstacksdk -openshift - -# CDP Runtime -apache-ranger==0.0.5 -nipyapi==0.18.0 -kafka-python==2.0.2 - -## Append your additional Python requirements here diff --git a/payload/deps/python_gcp.txt b/payload/deps/python_gcp.txt deleted file mode 100644 index 8af44e6..0000000 --- a/payload/deps/python_gcp.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Google Collection requirements -google-auth \ No newline at end of file diff --git a/payload/deps/python_secondary.txt b/payload/deps/python_secondary.txt deleted file mode 100644 index e1a77a1..0000000 --- a/payload/deps/python_secondary.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Packages that are less well behaved and should be installed secondarily -apache-atlas