Improve build and automate push to PyPI

.github/workflows/pypi.yml

@@ -0,0 +1,136 @@
+name: Build wheels and publish to PyPI
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+jobs:
+  configure:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Read Python versions from pyproject.toml
+      id: read-versions
+      # produces output like: python_versions=39,310,311,312,313
+      run: >-
+        echo "python_versions=$(
+        grep -oP '(?<=Language :: Python :: )\d.\d+' pyproject.toml
+        | sed 's/\.//'
+        | tr '\n' ','
+        | sed 's/,$//'
+        )" >> $GITHUB_OUTPUT
+    outputs:
+      python_versions: ${{ steps.read-versions.outputs.python_versions }}
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check version
+        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        run: python version.py check "${{ github.ref }}"
+      - name: Add untagged version suffix
+        if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
+        run: python version.py update
+      - name: Build sdist
+        run: pipx run build --sdist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+  choose_linux_wheel_types:
+    name: Decide which wheel types to build
+    runs-on: ubuntu-latest
+    steps:
+      - id: manylinux_x86_64
+        run: echo "wheel_types=manylinux_x86_64" >> $GITHUB_OUTPUT
+      - id: musllinux_x86_64
+        run: echo "wheel_types=musllinux_x86_64" >> $GITHUB_OUTPUT
+    outputs:
+      wheel_types: ${{ toJSON(steps.*.outputs.wheel_types) }}
+
+  build_linux_wheels:
+    needs: [configure, choose_linux_wheel_types, build_sdist]
+    name: ${{ matrix.wheel_type }} wheels
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        wheel_type: ${{ fromJSON(needs.choose_linux_wheel_types.outputs.wheel_types) }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+      - name: Extract sdist
+        run: |
+          tar zxvf dist/*.tar.gz --strip-components=1
+      - uses: docker/setup-qemu-action@v3
+        if: runner.os == 'Linux'
+        name: Set up QEMU
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: cp{${{ needs.configure.outputs.python_versions }}}-${{ matrix.wheel_type }}
+          CIBW_ARCHS_LINUX: auto
+          CIBW_BEFORE_BUILD: cd {project}; pip install -e .  # is there a better way to build the .so files?
+          CIBW_TEST_COMMAND: cd {project}; python -m unittest
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.wheel_type }}-wheels
+          path: ./wheelhouse/*.whl
+
+  pypi-publish:
+    name: Upload release to PyPI
+    needs: [build_sdist, build_linux_wheels]
+    runs-on: ubuntu-latest
+    #if: startsWith(github.ref, 'refs/tags/v')  # removed until pypi-test-publish is working
+    environment:
+      name: pypi
+      url: https://pypi.org/p/jsonobject
+    permissions:
+      id-token: write
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          # with no name set, it downloads all of the artifacts
+          path: dist/
+      - run: |
+          mv dist/sdist/*.tar.gz dist/
+          mv dist/*-wheels/*.whl dist/
+          rmdir dist/{sdist,*-wheels}
+          ls -R dist
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  # https://github.com/finpassbr/json-object/issues/1
+  #pypi-test-publish:
+  #  name: Upload release to test PyPI
+  #  needs: [build_sdist, build_linux_wheels]
+  #  runs-on: ubuntu-latest
+  #  environment:
+  #    name: testpypi
+  #    url: https://test.pypi.org/p/jsonobject
+  #  permissions:
+  #    id-token: write
+  #  steps:
+  #    - name: Download all the dists
+  #      uses: actions/download-artifact@v4
+  #      with:
+  #        # with no name set, it downloads all of the artifacts
+  #        path: dist/
+  #    - run: |
+  #        mv dist/sdist/*.tar.gz dist/
+  #        mv dist/*-wheels/*.whl dist/
+  #        rmdir dist/{sdist,*-wheels}
+  #        ls -R dist
+  #    - name: Publish package distributions to PyPI
+  #      uses: pypa/gh-action-pypi-publish@release/v1
+  #      with:
+  #        repository-url: https://test.pypi.org/legacy/

.github/workflows/tests.yml

@@ -9,24 +9,40 @@ on:
   pull_request:
 
 jobs:
-  build:
+  configure:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Read Python versions from pyproject.toml
+      id: read-versions
+      # produces output like: python_versions=[ "3.9", "3.10", "3.11", "3.12" ]
+      run: >-
+        echo "python_versions=$(
+        grep -oP '(?<=Language :: Python :: )\d\.\d+' pyproject.toml
+        | jq --raw-input .
+        | jq --slurp .
+        | tr '\n' ' '
+        )" >> $GITHUB_OUTPUT
+    outputs:
+      python_versions: ${{ steps.read-versions.outputs.python_versions }}
 
+  build:
+    needs: [configure]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ${{ fromJSON(needs.configure.outputs.python_versions) }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install setuptools
         python -m pip install -e .
     - name: Run tests
       run: |

CHANGES.md

@@ -5,6 +5,20 @@
 No significant changes since the last release
 
 
+## 2.3.0
+
+| Released on | Released by   |
+|-------------|---------------|
+| 2025-02-24  | @millerdev    |
+
+- Improve build and automate push to PyPI (https://github.com/dimagi/jsonobject/pull/236)
+  - Add pyproject.toml to replace most of setup.py
+  - Automate python version matrix on gitub actions
+  - Update github action versions
+  - Publish releases to pypi.org
+- Build C files with Cython 3.0.12 (https://github.com/dimagi/jsonobject/pull/235)
+  - Add support for Python 3.13
+
 ## 2.2.0
 
 | Released on | Released by   |

LIFECYCLE.md

@@ -38,7 +38,8 @@ This section contains instructions for the Dimagi team member performing the rel
 
 ## Bump version & update CHANGES.md
 
-In a single PR, bump the version number and update CHANGES.md to include release notes for this new version.
+In a single PR, bump the version number in `jsonobject/__init__.py` and update
+CHANGES.md to include release notes for this new version.
 
 ### Pick a version number
 
@@ -62,6 +63,9 @@ Once this PR is reviewed and merged, move on to the steps to release the update
 
 ## Release the new version
 
-To push the package to pypi, we follow Dimagi's internal documentation.
-Follow the steps in https://confluence.dimagi.com/display/saas/Python+Packaging+Crash+Course
-to release.
+To push the package to pypi, create a git tag named "vX.Y.Z" using  the version
+in `jsonobject/__init__.py` and push it to Github.
+
+A dev release is pushed to pypi.com/p/jsonobject/#history on each push/merge to
+master. A dev release may also be published on-demand for any branch with
+[workflow dispatch](https://github.com/dimagi/jsonobject/actions/workflows/pypi.yml).

jsonobject/__init__.py

@@ -2,6 +2,7 @@
 from .properties import *
 from .api import JsonObject
 
+__version__ = '2.3.0'
 __all__ = [
     'IntegerProperty', 'FloatProperty', 'DecimalProperty',
     'StringProperty', 'BooleanProperty',

pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "jsonobject"
+description = "A library for dealing with JSON as python objects"
+authors = [{name = "Danny Roberts", email = "[email protected]"}]
+license = {file = "LICENSE"}
+readme = {file = "README.md", content-type = "text/markdown"}
+dynamic = ["version"]
+requires-python = ">= 3.9"
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+
+    # The following classifiers are parsed by Github Actions workflows.
+    # Precise formatting is important (no extra spaces, etc.)
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+
+    "License :: OSI Approved :: BSD License",
+]
+
+[project.urls]
+Home = "https://github.com/dimagi/jsonobject"
+
+[build-system]
+requires = [
+    "setuptools>=75",
+    "Cython>=3.0.0,<4.0.0",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["jsonobject"]
+
+[tool.setuptools.dynamic]
+version = {attr = "jsonobject.__version__"}

scripts/install_cython.sh

@@ -1,6 +1,7 @@
 #! /bin/bash
 
-# grep setup.py for the pinned version of cython
-PINNED_CYTHON=$(grep -oE 'cython>=[0-9]+\.[0-9]+\.[0-9]+,<[0-9]+\.[0-9]+\.[0-9]+' setup.py)
+# grep pyproject.toml for the pinned version of cython
+PINNED_CYTHON=$(grep -oE 'Cython>?=[^"]+' pyproject.toml)
 
-pip install $PINNED_CYTHON
+echo "Installing $PINNED_CYTHON"
+pip install $PINNED_CYTHON setuptools

setup.py

@@ -1,5 +1,4 @@
 from setuptools import setup
-import io
 
 from setuptools.extension import Extension
 
@@ -20,40 +19,15 @@
     Extension('jsonobject.utils', ["jsonobject/utils" + ext],),
 ]
 
-CYTHON_REQUIRES = ['cython>=3.0.0,<4.0.0']
 if USE_CYTHON:
     from Cython.Build import cythonize
     extensions = cythonize(extensions, compiler_directives={"language_level" : "3str"})
 else:
     print("You are running without Cython installed. It is highly recommended to run\n"
-          "  pip install {}\n"
-          "before you continue".format(' '.join(CYTHON_REQUIRES)))
-
-
-with io.open('README.md', 'rt', encoding="utf-8") as readme_file:
-    long_description = readme_file.read()
-
+          "  ./scripts/install_cython.sh\n"
+          "before you continue")
 
 setup(
     name='jsonobject',
-    version='2.2.0',
-    author='Danny Roberts',
-    author_email='[email protected]',
-    description='A library for dealing with JSON as python objects',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    url='https://github.com/dimagi/jsonobject',
-    packages=['jsonobject'],
-    setup_requires=CYTHON_REQUIRES,
     ext_modules=extensions,
-    classifiers=[
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-        'Programming Language :: Python :: 3.13',
-        'License :: OSI Approved :: BSD License',
-    ],
 )