pypi-cd.yml

# The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/).

name: PyPI - Continuous Deployment

on:
    pull_request:
        paths:
            - .github/workflows/pypi-cd.yml
            - .github/workflows/build-python-wrapper/action.yml
            - .github/workflows/start-self-hosted-runner/action.yml
            - .github/workflows/install-shared-dependencies/action.yml
            - .github/workflows/install-engine/action.yml
            - .github/json_matrices/**
            - .github/workflows/create-test-matrices/action.yml
    push:
        tags:
            - "v*.*"
    workflow_dispatch:
        inputs:
            version:
                description: "The release version of GLIDE, formatted as *.*.* or *.*.*-rc*"
                required: true

concurrency:
    group: pypi-${{ github.head_ref || github.ref }}
    cancel-in-progress: true

permissions:
    id-token: write

jobs:
    load-platform-matrix:
        runs-on: ubuntu-latest
        outputs:
            PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }}
        steps:
            - name: Checkout
              uses: actions/checkout@v4

            - name: load-platform-matrix
              id: load-platform-matrix
              shell: bash
              run: |
                  # Filter entries with pypi in PACKAGE_MANAGERS and replace "ephemeral" with "persistent" in RUNNER
                  export PLATFORM_MATRIX=$(jq 'map(
                      select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["pypi"])))
                      | .RUNNER = (
                          if (.RUNNER | type == "array") 
                          then (.RUNNER | map(if . == "ephemeral" then "persistent" else . end)) 
                          else (if .RUNNER == "ephemeral" then "persistent" else .RUNNER end) 
                          end
                      )
                  )' < .github/json_matrices/build-matrix.json | jq -c .)
                  echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT

    start-self-hosted-runner:
        if: github.repository_owner == 'valkey-io'
        runs-on: ubuntu-latest
        environment: AWS_ACTIONS
        steps:
            - name: Checkout
              uses: actions/checkout@v4
              with:
                  fetch-depth: 0

            - name: Start self hosted EC2 runner
              uses: ./.github/workflows/start-self-hosted-runner
              with:
                  role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
                  aws-region: ${{ secrets.AWS_REGION }}
                  ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}

    publish-binaries:
        needs: [start-self-hosted-runner, load-platform-matrix]
        if: github.repository_owner == 'valkey-io'
        name: Publish packages to PyPi
        runs-on: ${{ matrix.build.RUNNER }}
        timeout-minutes: 35
        strategy:
            fail-fast: false
            matrix:
                build: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }}
        steps:
            - name: Setup self-hosted runner access
              if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }}
              run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide

            - name: Checkout
              uses: actions/checkout@v4
              with:
                  submodules: "true"

            - name: Set the release version
              shell: bash
              run: |
                  if ${{ env.EVENT_NAME == 'pull_request' }}; then
                    R_VERSION="255.255.255"
                  elif ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then
                    R_VERSION="${{ env.INPUT_VERSION }}"
                  else
                    R_VERSION=${GITHUB_REF:11}
                  fi
                  echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV
              env:
                  EVENT_NAME: ${{ github.event_name }}
                  INPUT_VERSION: ${{ github.event.inputs.version }}

            - name: Check that the release version dont have typo init
              if: ${{ github.event_name != 'pull_request' && contains(env.RELEASE_VERSION, '-') && !contains(env.RELEASE_VERSION, 'rc') }}
              run: |
                  echo "The release version "${{ env.RELEASE_VERSION }}" contains a typo, please fix it"
                  echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release."
                  exit 1

            - name: Set the package version for Python
              working-directory: ./python
              run: |
                  SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".*  ]]; then echo "''"; fi`
                  sed -i $SED_FOR_MACOS "s|255.255.255|${{ env.RELEASE_VERSION }}|g" ./Cargo.toml
                  # Log the edited Cargo.toml file
                  cat Cargo.toml

            - name: Set up Python
              if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }}
              uses: actions/setup-python@v5
              with:
                  python-version: "3.13"

            - name: Update package version in config.toml
              uses: ./.github/workflows/update-glide-version
              with:
                  folder_path: "${{ github.workspace }}/python/.cargo"
                  named_os: ${{ matrix.build.NAMED_OS }}

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.build.OS }}
                  target: ${{ matrix.build.TARGET }}
                  publish: "true"
                  github-token: ${{ secrets.GITHUB_TOKEN }}

            - name: Include protobuf files in the package
              working-directory: ./python
              run: |
                  SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".*  ]]; then echo "''"; fi`
                  sed -i $SED_FOR_MACOS '/pb2.py/d' .gitignore
                  # Log the edited .gitignore file
                  cat .gitignore

            - name: Build Python wheels (linux)
              if: startsWith(matrix.build.NAMED_OS, 'linux')
              uses: PyO3/maturin-action@v1
              with:
                  working-directory: ./python
                  target: ${{ matrix.build.TARGET }}
                  args: --release --strip --out wheels -i ${{ github.event_name != 'pull_request' && 'python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }}
                  manylinux: auto
                  container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }}
                  before-script-linux: |
                      # Install protobuf compiler
                      if [[ $(`which apt`) != '' ]]
                      then
                        echo "installing unzip and curl"
                        apt-get update
                        apt install unzip curl python3.13 -y
                      fi
                      PB_REL="https://github.com/protocolbuffers/protobuf/releases"
                      ARCH=`uname -p`
                      if [[ $ARCH == 'x86_64' ]]; then 
                        PROTOC_ARCH="x86_64"
                      elif [[ $ARCH == 'aarch64' ]]; then 
                        PROTOC_ARCH="aarch_64"
                        export CC_aarch64_unknown_linux_gnu=gcc
                        export CFLAGS_aarch64_unknown_linux_gnu="-march=armv8-a"
                      else 
                        echo "Running on unsupported architecture: $ARCH. Expected one of: ['x86_64', 'aarch64']"
                        exit 1 
                      fi
                      curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-${PROTOC_ARCH}.zip
                      unzip protoc-3.20.3-linux-${PROTOC_ARCH}.zip -d $HOME/.local
                      export PATH="$PATH:$HOME/.local/bin"

            - name: Build Python wheels (macos)
              if: startsWith(matrix.build.NAMED_OS, 'darwin')
              uses: PyO3/maturin-action@v1
              with:
                  maturin-version: 0.14.17
                  working-directory: ./python
                  target: ${{ matrix.build.TARGET }}
                  args: --release --strip --out wheels -i  ${{ github.event_name != 'pull_request' && 'python3.9 python3.10 python3.11 python3.12 python3.13' || 'python3.12' }}

            - name: Upload Python wheels
              if: github.event_name != 'pull_request'
              uses: actions/upload-artifact@v4
              with:
                  name: wheels-${{ matrix.build.TARGET }}
                  path: python/wheels
                  if-no-files-found: error

    build-source-dist-and-publish-to-pypi:
        if: github.event_name != 'pull_request'
        name: Publish the base PyPi package
        runs-on: ubuntu-latest
        needs: publish-binaries
        steps:
            - name: Checkout
              uses: actions/checkout@v4
              with:
                  submodules: "true"

            - name: Set the release version
              shell: bash
              run: |
                  if ${{ env.EVENT_NAME == 'pull_request' }}; then
                    R_VERSION="255.255.255"
                  elif ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then
                    R_VERSION="${{ env.INPUT_VERSION }}"
                  else
                    R_VERSION=${GITHUB_REF:11}
                  fi
                  echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV
              env:
                  EVENT_NAME: ${{ github.event_name }}
                  INPUT_VERSION: ${{ github.event.inputs.version }}

            - name: Set the package version for Python
              working-directory: ./python
              run: |
                  sed -i "s|255.255.255|${{ env.RELEASE_VERSION }}|g" ./Cargo.toml
          
            - name: Build source distributions
              uses: PyO3/maturin-action@v1
              with:
                  working-directory: ./python
                  command: sdist
                  args: --out sdist
            
            - name: Download binaries 
              uses: actions/download-artifact@v4
              with:
                  path: python/wheels
                  merge-multiple: true

            - name: Publish binaries and source to PyPI 
              uses: PyO3/maturin-action@v1
              env:
                  MATURIN_PYPI_TOKEN: ${{ secrets.LIVEPYPI_API_TOKEN }}
                  MATURIN_REPOSITORY: pypi
              with:
                  command: upload
                  args: --skip-existing python/wheels/* python/sdist/*

    test-release:
        if: github.event_name != 'pull_request'
        name: Test the release
        runs-on: ${{ matrix.build.RUNNER }}
        needs: [build-source-dist-and-publish-to-pypi, load-platform-matrix]
        strategy:
            fail-fast: false
            matrix:
                build: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }}
        steps:
            - name: Setup self-hosted runner access
              if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }}
              run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide

            - name: checkout
              uses: actions/checkout@v4

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.12

            - name: Install engine
              uses: ./.github/workflows/install-engine
              with:
                  engine-version: "8.0"
                  target: ${{ matrix.build.target }}

            - name: Check if RC and set a distribution tag for the package
              shell: bash
              run: |
                  if [[ "${GITHUB_REF:11}" == *"rc"* ]]
                  then
                    echo "This is a release candidate"
                    echo "PIP_PRE=true" >> $GITHUB_ENV
                  else
                    echo "This is a stable release"
                    echo "PIP_PRE=false" >> $GITHUB_ENV
                  fi

            - name: Run the tests
              shell: bash
              working-directory: ./utils/release-candidate-testing/python
              run: |
                  python -m venv venv
                  source venv/bin/activate
                  pip install -U pip
                  if [[ "${{ env.PIP_PRE }}" == "true" ]]; then
                    pip install --pre valkey-glide
                  else
                    pip install valkey-glide
                  fi
                  python rc_test.py

            # Reset the repository to make sure we get the clean checkout of the action later in other actions.
            # It is not required since in other actions we are cleaning before the action, but it is a good practice to do it here as well.
            - name: Reset repository
              if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }}
              shell: bash
              run: |
                  git reset --hard
                  git clean -xdf