diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 000000000..e9e011f0f --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,185 @@ +name: hera cicd + +on: + push: + branches: + - main + pull_request: { } + +jobs: + test: + name: build + test py${{ matrix.python-version }} on ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest ] + python-version: [ '3.7', '3.8', '3.9', '3.10' ] + env: + PYVER: ${{ matrix.python-version }} + OS: ${{ matrix.os }} + + runs-on: ${{ matrix.os }} + + steps: + - name: setup python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: checkout + uses: actions/checkout@v2 + + - name: install base dependencies + run: | + python -m pip install --upgrade "tox<4.0" tox-wheel twine + + - name: lint + env: + TOXENV: lint + run: | + tox + + - name: typecheck + env: + TOXENV: typecheck + run: | + tox + + - name: tests + env: + ENABLE_BDIST_EXT_MODULE: 'yes' + TOXENV: python${{ matrix.python-version }} + run: | + tox + + - name: twine check + run: | + twine check .tox/dist/* + + - name: upload build artifacts + uses: actions/upload-artifact@v2 + with: + name: pypi-build-arts + path: .tox/dist + + - name: upload coverage files + uses: actions/upload-artifact@v2 + with: + name: coverage-data + path: ".tox/.coverage.*" + + build-py3-none-any: + name: build py3-none-any on linux + timeout-minutes: 10 + + runs-on: ubuntu-latest + + steps: + - name: setup python3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Checkout + uses: actions/checkout@v2 + + - name: Install base dependencies + run: | + python -m pip install --upgrade build twine + + - name: build sdist and wheel ( no test ) + run: | + python -m build --sdist --wheel --outdir .tox/dist/ + + - name: list artifacts + run: | + ls -l .tox/dist + + - name: twine check + run: | + twine check .tox/dist/* + + - name: upload build artifacts + uses: actions/upload-artifact@v2 + with: + name: pypi-build-arts + path: .tox/dist + + coverage: + needs: [ test ] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: setup python 3.7 + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: download coverage files + uses: actions/download-artifact@v2 + with: + name: coverage-data + path: .tox + + - name: list coverage files + run: | + find .tox -name ".coverage.*" + + - name: install tox + run: | + python -m pip install --upgrade tox + + - name: create coverage report + env: + TOXENV: coverage + run: | + tox + + - name: upload coverage HTML report + uses: actions/upload-artifact@v2 + with: + name: html-report + path: .tox/htmlcov + + publish: + needs: [ coverage, test, build-py3-none-any ] + if: success() && ( github.ref == 'refs/heads/main' ) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: setup python 3.7 + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: get distribution wheels + uses: actions/download-artifact@v2 + with: + name: pypi-build-arts + path: dist + + - name: list all pypi build arts + run: | + ls -la dist/ + + - name: install check dependencies + run: | + python -m pip install --upgrade twine + + - name: check pypi artifacts + run: | + twine check dist/* + + - name: publish package to pypi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + verbose: true + skip_existing: true diff --git a/.github/workflows/hera_build_and_publish.yaml b/.github/workflows/hera_build_and_publish.yaml deleted file mode 100644 index 7bb136cd2..000000000 --- a/.github/workflows/hera_build_and_publish.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: Hera `main` build and publish -on: - push: - branches: - - main - paths: - - 'VERSION' -jobs: - # First, build Hera and publish the package to the PyPi test index. This ensures that Hera's `setup.py` - # is structured correctly, includes the right references, files, etc, its dependencies are installable, - # and a wheel can be built and published successfully. - test-index-build-and-publish: - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install core dependencies - run: | - python -m pip install --upgrade pip wheel pipenv - - - name: Checkout - uses: actions/checkout@v2 - - # Install everything in a virtual environment for the initial publication process since this is the - # same as the official "prod" process. - - name: Sync virtual environment - run: | - pipenv sync --dev --pre - - - name: Run build - run: | - pipenv run make build - - # Publish to the Test PyPi index, with verbose logs in case anything needs to be reported in rich format. - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_TEST_API_TOKEN }} - verbose: true - repository_url: https://test.pypi.org/legacy/ - - # Second, install Hera from the Test PyPi index and run tests again. While it may seem redundant to re-run tests, - # this helps capture any failures that may occur as a failure of installation, not of running. Plus, tests run - # quickly, and the cost of publishing a broken package is much higher than running tests some small N number of times. - test-index-install-and-run-tests: - needs: - - test-index-build-and-publish - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout - uses: actions/checkout@v2 - - - name: Install core dependencies - run: | - python -m pip install --upgrade pip wheel pytest - - # Install Hera specifically from the Test index URL. Note that this script "sleeps" for 60 seconds. This hacky - # approach was introduced in order to wait for Test PyPI to update the index with the new Hera version so it's - # available for installation. - - name: Install published test package - run: | - sleep 60s - HERA_VERSION=$(cat VERSION) - python -m pip install \ - --index-url https://test.pypi.org/simple/ hera-workflows==$HERA_VERSION \ - --extra-index-url=https://pypi.org/simple/ - - # If this succeeds, Hera is ready for publication. - - name: Run tests with published package - run: | - python -m pytest --durations=5 tests - - # Last, run the official "production"/ready to publish workflow that builds Hera and releases it to the official - # PyPi main index. - main-build-and-publish: - needs: - - test-index-install-and-run-tests - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout - uses: actions/checkout@v2 - - - name: Install pipenv - run: | - python -m pip install --upgrade pipenv wheel - - # Cache dependencies based on the hash of the lockfile. When the lockfile changes it means the virtual - # environment needs to change, which is when the `Install dependencies` step has to re-sync a virtual environment. - - id: cache-pipenv - uses: actions/cache@v1 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} - - - name: Install dependencies - if: steps.cache-pipenv.outputs.cache-hit != 'true' - run: | - pipenv sync --dev --pre - - - name: Run build - run: | - pipenv run make build - - # Finally, publish an official version of Hera! - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - verbose: true - skip_existing: true diff --git a/.github/workflows/hera_pr.yaml b/.github/workflows/hera_pr.yaml deleted file mode 100644 index b80295560..000000000 --- a/.github/workflows/hera_pr.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Hera PR build - -on: pull_request - -jobs: - hera-pr-build: - - timeout-minutes: 10 - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - steps: - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Checkout - uses: actions/checkout@v2 - - - name: Install base dependencies - run: | - python -m pip install --upgrade "tox<4.0" - - - name: Run lint - env: - TOXENV: lint - run: | - tox - - - name: Run typecheck - env: - TOXENV: typecheck - run: | - tox - - - name: Run tests - env: - TOXENV: python${{ matrix.python-version }} - run: | - tox diff --git a/Makefile b/Makefile index 45031d2c7..c125f747e 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ typecheck: mypy --namespace-packages tests test: - pytest --durations=5 $(TEST_FOLDER) + pytest -c tox.ini $(TEST_FOLDER) verify: lint typecheck test echo "Lint, typecheck, test successfully completed!" diff --git a/README.md b/README.md index 5546b2a29..b51421940 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ and its crew were specially protected by the goddess Hera. (https://en.wikipedia.org/wiki/Argo) -[![Build](https://github.com/argoproj-labs/hera-workflows/actions/workflows/hera_build_and_publish.yaml/badge.svg)](https://github.com/argoproj-labs/hera-workflows/blob/main/.github/workflows/hera_build_and_publish.yaml) +[![Build](https://github.com/argoproj-labs/hera-workflows/actions/workflows/cicd.yaml/badge.svg)](https://github.com/argoproj-labs/hera-workflows/blob/main/.github/workflows/cicd.yaml) [![Pypi](https://img.shields.io/pypi/v/hera-workflows.svg)](https://pypi.python.org/pypi/hera-workflows) +[![CondaForge](https://img.shields.io/conda/v/conda-forge/hera-workflows.svg)](https://anaconda.org/conda-forge/hera-workflows) [![Downloads](https://pepy.tech/badge/hera-workflows)](https://pepy.tech/project/hera-workflows) [![Downloads/month](https://pepy.tech/badge/hera-workflows/month)](https://pepy.tech/project/hera-workflows) [![Downloads/week](https://pepy.tech/badge/hera-workflows/week)](https://pepy.tech/project/hera-workflows) @@ -94,13 +95,27 @@ In you activated `pipenv` shell, you can utilize the tasks found in `Makefile`, make test ``` -To run tests on all supported python versions run [tox](https://tox.wiki/en/latest/): +To run tests on all supported python versions with coverage run [tox](https://tox.wiki/en/latest/): ```shell tox ``` -> See project `tox.ini` for more details +To list all available `tox` envs run: + +```shell +tox -a +``` + +To run selected tox envs, e.g. for a specific python version with coverage run: + +```shell +tox -e python3.7,coverage +``` + +As `coverage` *dependes* on `python3.7`, it will run *after* `python3.7` + +See project `tox.ini` for more details Also, see the [contributing guide](https://github.com/argoproj-labs/hera-workflows/blob/main/CONTRIBUTING.md)! @@ -124,7 +139,7 @@ from hera.workflow_service import WorkflowService def say(message: str): """ - This can be anything as long as the Docker image satisfies the dependencies. You can import anything Python + This can be anything as long as the Docker image satisfies the dependencies. You can import anything Python that is in your container e.g torch, tensorflow, scipy, biopython, etc - just provide an image to the task! """ print(message) diff --git a/setup.cfg b/setup.cfg index e9201935b..ccd35843e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,3 @@ -[pytest] -addopts = --cov=hera -vv -# for now, keep this commented out, as it's used explicitly in Makefile -; testpaths = tests - [flake8] exclude = # do not look here for things to lint .git diff --git a/setup.py b/setup.py index e15ea9cfe..c3ab4f50b 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,27 @@ -from setuptools import find_namespace_packages, setup +import os +from setuptools import setup, Distribution VERSION = open('VERSION').read().strip() LONG_DESCRIPTION = open('README.md').read() + +class BinaryDistribution(Distribution): + """Distribution which enables a binary package with platform name + + Will build `none-any` wheels by default, e.g. - + + hera_workflows-1.8.0-py3-none-any.whl + + If the env var ENABLE_BDIST_EXT_MODULE=yes is defined built wheel will have platform and + python version info, e.g. - + Used to create wheels for specific OS and Python version, e.g. - + + hera_workflows-1.8.0-cp37-cp37m-macosx_10_9_x86_64.whl + """ + def has_ext_modules(self): + return os.environ.get('ENABLE_BDIST_EXT_MODULE') == 'yes' + + setup( name='hera-workflows', version=VERSION, @@ -45,6 +64,7 @@ "certifi", "pytz" ], - zip_safe=False + zip_safe=False, + distclass=BinaryDistribution ) diff --git a/tox.ini b/tox.ini index af71def2f..9abeabd8b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,24 +1,66 @@ [tox] -envlist = python{3.7,3.8,3.9,3.10} +envlist = python{3.7,3.8,3.9,3.10},coverage,lint,typecheck + +[pytest] +addopts = --durations=5 -vv + +[coverage:run] +branch = true +parallel = true + +[coverage:report] +skip_covered = True +show_missing = True + +[coverage:paths] +source = src/hera + */.tox/*/lib/python*/site-packages/hera + */.tox/pypy*/site-packages/hera + */.tox\*\Lib\site-packages\hera + */src/hera + *\src\hera [testenv] wheel = true -allowlist_externals = - make +setenv = + ENABLE_BDIST_EXT_MODULE = {env:ENABLE_BDIST_EXT_MODULE:no} + OS = {env:OS:} + COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{env:OS}.{envname}} deps = pytest + pytest-cov commands= - make test + pytest \ + --cov "{envsitepackagesdir}/hera" \ + --cov-config "{toxinidir}/tox.ini" + +[testenv:coverage] +setenv = + COVERAGE_FILE = {toxworkdir}/.coverage +skip_install = true +deps = + coverage +commands = + coverage combine + coverage report -m + coverage xml -o {toxworkdir}/coverage.xml + coverage html -d {toxworkdir}/htmlcov +depends = python3.7, python3.8, python3.9, python3.10 [testenv:lint] skipsdist = True +skip_install = true deps = flake8 +allowlist_externals = + make commands= make lint [testenv:typecheck] skipsdist = True +allowlist_externals = + make deps = mypy {[testenv]deps}