From 91a1141aa93e3abfbb6982be6602ec6b025cc0b9 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 30 Nov 2023 12:03:25 +0100 Subject: [PATCH 1/3] build: move ci tests to reusable partial Moves all jobs from ci.yml to _test.yml and from where they are included in ci.yml via the `uses` directive. You can check that the jobs itself did not change by diffing the old ci.yml with the new _test.yml: ``` diff \ <(curl -s https://raw.githubusercontent.com/secure-systems-lab/securesystemslib/a3651a1b321b8ad05ddb6e85aaceb402acec9671/.github/workflows/ci.yml) \ .github/workflows/_test.yml ``` Move jobs from ci.yml to _test.yml, so we can reuse them in different workflows. Signed-off-by: Lukas Puehringer --- .github/workflows/_test.yml | 73 +++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 70 ++--------------------------------- 2 files changed, 77 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/_test.yml diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 00000000..ebff3360 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,73 @@ +on: + workflow_call: + # Permissions inherited from caller workflow + +permissions: {} + +jobs: + test: + strategy: + fail-fast: false + # Run tests on each OS/Python combination + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-latest, macos-latest, windows-latest] + toxenv: [py] + + include: + - python-version: "3.11" + os: ubuntu-latest + toxenv: purepy311 + - python-version: "3.11" + os: ubuntu-latest + toxenv: py311-no-gpg + - python-version: "3.11" + os: ubuntu-latest + toxenv: py311-test-gpg-fails + - python-version: "3.11" + os: ubuntu-latest + toxenv: lint + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout securesystemslib + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "requirements*.txt" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + + - name: Install system dependencies + shell: bash + run: | + # NOTE: HSM tests are skipped silently, if PYKCS11LIB is unset. + + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y softhsm2 + echo "PYKCS11LIB=/usr/lib/softhsm/libsofthsm2.so" >> $GITHUB_ENV + + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install softhsm + echo "PYKCS11LIB=$(brew --prefix softhsm)/lib/softhsm/libsofthsm2.so" >> $GITHUB_ENV + + elif [ "$RUNNER_OS" == "Windows" ]; then + echo "Skipping HSM tests on Windows" + # see https://github.com/secure-systems-lab/securesystemslib/issues/520 + + + else + echo "$RUNNER_OS not supported" + exit 1 + fi + + - name: Run tox + run: tox -e ${{ matrix.toxenv }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c956c70..e109ccc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,75 +4,13 @@ on: push: branches: - main + pull_request: workflow_dispatch: permissions: {} jobs: - build: - strategy: - fail-fast: false - # Run tests on each OS/Python combination - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - os: [ubuntu-latest, macos-latest, windows-latest] - toxenv: [py] - - include: - - python-version: "3.11" - os: ubuntu-latest - toxenv: purepy311 - - python-version: "3.11" - os: ubuntu-latest - toxenv: py311-no-gpg - - python-version: "3.11" - os: ubuntu-latest - toxenv: py311-test-gpg-fails - - python-version: "3.11" - os: ubuntu-latest - toxenv: lint - - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout securesystemslib - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "requirements*.txt" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade tox - - - name: Install system dependencies - shell: bash - run: | - # NOTE: HSM tests are skipped silently, if PYKCS11LIB is unset. - - if [ "$RUNNER_OS" == "Linux" ]; then - sudo apt-get install -y softhsm2 - echo "PYKCS11LIB=/usr/lib/softhsm/libsofthsm2.so" >> $GITHUB_ENV - - elif [ "$RUNNER_OS" == "macOS" ]; then - brew install softhsm - echo "PYKCS11LIB=$(brew --prefix softhsm)/lib/softhsm/libsofthsm2.so" >> $GITHUB_ENV - - elif [ "$RUNNER_OS" == "Windows" ]; then - echo "Skipping HSM tests on Windows" - # see https://github.com/secure-systems-lab/securesystemslib/issues/520 - - - else - echo "$RUNNER_OS not supported" - exit 1 - fi - - - name: Run tox - run: tox -e ${{ matrix.toxenv }} + test: + name: Test + uses: ./.github/workflows/_test.yml From 3e9de2f9a1deae5e2c47e76c8081a3e5db005735 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 30 Nov 2023 12:12:11 +0100 Subject: [PATCH 2/3] build: pin build dependencies - add build requirements file with pinned build dependency - pin build backend in pyproject.yml (needs manual updating) Signed-off-by: Lukas Puehringer --- pyproject.toml | 3 ++- requirements-build.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 requirements-build.txt diff --git a/pyproject.toml b/pyproject.toml index 6922602a..6774eb1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [build-system] -requires = ["hatchling"] +# Version needs manual updates (dependabot/dependabot-core#8465) +requires = ["hatchling==1.18.0"] build-backend = "hatchling.build" [project] diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 00000000..e780e73e --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1 @@ +build==1.0.3 From 2c5a170338e6aa2fbba6d9c09d0a55391c0e6d0f Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 30 Nov 2023 12:13:46 +0100 Subject: [PATCH 3/3] build: add cd GitHub workflow Add GitHub workflow to build securesystemslib and release on GitHub and PyPI. The workflow is copied from python-tuf with minimal changes, see: ``` diff \ <(curl -s https://raw.githubusercontent.com/theupdateframework/python-tuf/c92cd28b38d3af5ee24411a3c2082fc2d6c37f4b/.github/workflows/cd.yml) \ .github/workflows/cd.yml ``` Prerequisites and usage details are described at https://github.com/theupdateframework/python-tuf/blob/v3.1.0/docs/RELEASE.md. (a verify_release script does not exist for securesystemslib) Signed-off-by: Lukas Puehringer --- .github/workflows/cd.yml | 118 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..9ccdf736 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,118 @@ +name: CD +concurrency: cd + +on: + push: + tags: + - v* + +permissions: {} + +jobs: + test: + name: Test + uses: ./.github/workflows/_test.yml + + build: + name: Build + runs-on: ubuntu-latest + needs: test + steps: + - name: Checkout release tag + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.workflow_run.head_branch }} + + - name: Set up Python + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: '3.x' + + - name: Install build dependency + run: python3 -m pip install -r requirements-build.txt + + - name: Build binary wheel and source tarball + run: python3 -m build --sdist --wheel --outdir dist/ . + + - name: Store build artifacts + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + # NOTE: The GitHub release page contains the release artifacts too, but using + # GitHub upload/download actions seems robuster: there is no need to compute + # download URLs and tampering with artifacts between jobs is more limited. + with: + name: build-artifacts + path: dist + + candidate_release: + name: Release candidate on Github for review + runs-on: ubuntu-latest + needs: build + permissions: + contents: write # to modify GitHub releases + outputs: + release_id: ${{ steps.gh-release.outputs.result }} + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: build-artifacts + path: dist + + - id: gh-release + name: Publish GitHub release draft + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + fs = require('fs') + res = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + name: '${{ github.ref_name }}-rc', + tag_name: '${{ github.ref }}', + body: 'Release waiting for review...', + }); + + fs.readdirSync('dist/').forEach(file => { + github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: res.data.id, + name: file, + data: fs.readFileSync('dist/' + file), + }); + }); + return res.data.id + + release: + name: Release + runs-on: ubuntu-latest + needs: candidate_release + environment: release + permissions: + contents: write # to modify GitHub releases + id-token: write # to authenticate as Trusted Publisher to pypi.org + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: build-artifacts + path: dist + + - name: Publish binary wheel and source tarball on PyPI + # Only attempt pypi upload in upstream repository + if: github.repository == 'secure-systems-lab/securesystemslib' + uses: pypa/gh-action-pypi-publish@b7f401de30cb6434a1e19f805ff006643653240e # v1.8.10 + + - name: Finalize GitHub release + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: '${{ needs.candidate_release.outputs.release_id }}', + name: '${{ github.ref_name }}', + body: 'See [CHANGELOG.md](https://github.com/' + + context.repo.owner + '/' + context.repo.repo + + '/blob/${{ github.ref_name }}/CHANGELOG.md) for details.' + })