From b8f69063e1abba0e0649b0205e1e5645e5234c55 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 14 Oct 2024 03:15:06 +0100 Subject: [PATCH] feat:semver (#3) --- .github/workflows/build_tests.yml | 36 ++++++++ .github/workflows/conventional-label.yaml | 10 ++ .github/workflows/coverage.yml | 44 +++++++++ .github/workflows/install_tests.yml | 34 +++++++ .github/workflows/license_tests.yml | 44 +++++++++ .github/workflows/publish_stable.yml | 58 ++++++++++++ .github/workflows/python-package.yml | 48 ---------- .github/workflows/release_workflow.yml | 108 ++++++++++++++++++++++ .github/workflows/unit_tests.yml | 68 ++++++++++++++ demo.py | 34 ------- docs/.gitignore | 3 - docs/Makefile | 20 ---- docs/conf.py | 82 ---------------- docs/index.rst | 23 ----- ovos_padatious/version.py | 6 ++ requirements.txt | 2 +- setup.py | 54 +++++++++-- tests/requirements.txt | 2 + 18 files changed, 457 insertions(+), 219 deletions(-) create mode 100644 .github/workflows/build_tests.yml create mode 100644 .github/workflows/conventional-label.yaml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/install_tests.yml create mode 100644 .github/workflows/license_tests.yml create mode 100644 .github/workflows/publish_stable.yml delete mode 100644 .github/workflows/python-package.yml create mode 100644 .github/workflows/release_workflow.yml create mode 100644 .github/workflows/unit_tests.yml delete mode 100755 demo.py delete mode 100644 docs/.gitignore delete mode 100644 docs/Makefile delete mode 100644 docs/conf.py delete mode 100644 docs/index.rst create mode 100644 ovos_padatious/version.py create mode 100644 tests/requirements.txt diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml new file mode 100644 index 0000000..f204bb7 --- /dev/null +++ b/.github/workflows/build_tests.yml @@ -0,0 +1,36 @@ +name: Run Build Tests +on: + push: + workflow_dispatch: + +jobs: + build_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev swig libssl-dev libfann-dev portaudio19-dev libpulse-dev + - name: Build Source Packages + run: | + python setup.py sdist + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Install tflite_runtime workaround tflit bug + run: | + pip3 install numpy + pip3 install --extra-index-url https://google-coral.github.io/py-repo/ tflite_runtime + - name: Install core repo + run: | + pip install .[audio-backend,mark1,stt,tts,skills_minimal,skills,gui,bus,all] diff --git a/.github/workflows/conventional-label.yaml b/.github/workflows/conventional-label.yaml new file mode 100644 index 0000000..0a449cb --- /dev/null +++ b/.github/workflows/conventional-label.yaml @@ -0,0 +1,10 @@ +# auto add labels to PRs +on: + pull_request_target: + types: [ opened, edited ] +name: conventional-release-labels +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: bcoe/conventional-release-labels@v1 \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..7393b31 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,44 @@ +name: Run CodeCov +on: + push: + branches: + - dev + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + env: + PYTHON: '3.9' + steps: + - uses: actions/checkout@master + - name: Setup Python + uses: actions/setup-python@master + with: + python-version: 3.9 + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev + python -m pip install build wheel + - name: Install repo + run: | + pip install -e . + - name: Install test dependencies + run: | + pip install -r test/requirements.txt + - name: Generate coverage report + run: | + pip install pytest + pip install pytest-cov + pytest --cov=ovos_bus_client --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./coverage/reports/ + fail_ci_if_error: true + files: ./coverage.xml,!./cache + flags: unittests + name: codecov-umbrella + verbose: true \ No newline at end of file diff --git a/.github/workflows/install_tests.yml b/.github/workflows/install_tests.yml new file mode 100644 index 0000000..4aaabea --- /dev/null +++ b/.github/workflows/install_tests.yml @@ -0,0 +1,34 @@ +name: Run Install Tests +on: + push: + branches: + - master + - dev + workflow_dispatch: + +jobs: + install: + strategy: + max-parallel: 2 + matrix: + python-version: [ 3.7, 3.8, 3.9, "3.10" ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev swig libssl-dev + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Install package + run: | + pip install .[all] \ No newline at end of file diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 0000000..29f4063 --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,44 @@ +name: Run License Tests +on: + push: + branches: + - master + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + license_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev swig libssl-dev + - name: Install core repo + run: | + pip install . + - name: Get explicit and transitive dependencies + run: | + pip freeze > requirements-all.txt + - name: Check python + id: license_check_report + uses: pilosus/action-pip-license-checker@v0.5.0 + with: + requirements: 'requirements-all.txt' + fail: 'Copyleft,Other,Error' + fails-only: true + exclude: '^(tqdm).*' + exclude-license: '^(Mozilla).*$' + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" \ No newline at end of file diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml new file mode 100644 index 0000000..38ebc4b --- /dev/null +++ b/.github/workflows/publish_stable.yml @@ -0,0 +1,58 @@ +name: Stable Release +on: + push: + branches: [master] + workflow_dispatch: + +jobs: + publish_stable: + uses: TigreGotico/gh-automations/.github/workflows/publish-stable.yml@master + secrets: inherit + with: + branch: 'master' + version_file: 'ovos_padatious/version.py' + setup_py: 'setup.py' + publish_release: true + + publish_pypi: + needs: publish_stable + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} + + + sync_dev: + needs: publish_stable + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + ref: master + - name: Push master -> dev + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: dev \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index a969f14..0000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,48 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: - push: - branches: [ dev ] - pull_request: - branches: [ dev ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install FANN2 from source - run: | - wget http://downloads.sourceforge.net/project/fann/fann/2.2.0/FANN-2.2.0-Source.zip - unzip FANN-2.2.0-Source.zip - pushd FANN-2.2.0-Source/ - cmake . - sudo make install - popd - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - python setup.py install - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - # Ensure pytest can find the FANN2 libraries - LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH python -m pytest diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml new file mode 100644 index 0000000..3281df8 --- /dev/null +++ b/.github/workflows/release_workflow.yml @@ -0,0 +1,108 @@ +name: Release Alpha and Propose Stable + +on: + pull_request: + types: [closed] + branches: [dev] + +jobs: + publish_alpha: + if: github.event.pull_request.merged == true + uses: TigreGotico/gh-automations/.github/workflows/publish-alpha.yml@master + secrets: inherit + with: + branch: 'dev' + version_file: 'ovos_padatious/version.py' + setup_py: 'setup.py' + update_changelog: true + publish_prerelease: true + changelog_max_issues: 100 + + notify: + if: github.event.pull_request.merged == true + needs: publish_alpha + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Send message to Matrix bots channel + id: matrix-chat-message + uses: fadenb/matrix-chat-message@v0.0.6 + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + new ${{ github.event.repository.name }} PR merged! https://github.com/${{ github.repository }}/pull/${{ github.event.number }} + + publish_pypi: + needs: publish_alpha + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} + + + propose_release: + needs: publish_alpha + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - name: Checkout dev branch + uses: actions/checkout@v3 + with: + ref: dev + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Get version from setup.py + id: get_version + run: | + VERSION=$(python setup.py --version) + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Create and push new branch + run: | + git checkout -b release-${{ env.VERSION }} + git push origin release-${{ env.VERSION }} + + - name: Open Pull Request from dev to master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Variables + BRANCH_NAME="release-${{ env.VERSION }}" + BASE_BRANCH="master" + HEAD_BRANCH="release-${{ env.VERSION }}" + PR_TITLE="Release ${{ env.VERSION }}" + PR_BODY="Human review requested!" + + # Create a PR using GitHub API + curl -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -d "{\"title\":\"$PR_TITLE\",\"body\":\"$PR_BODY\",\"head\":\"$HEAD_BRANCH\",\"base\":\"$BASE_BRANCH\"}" \ + https://api.github.com/repos/${{ github.repository }}/pulls + diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..72c4ce1 --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,68 @@ +name: Run UnitTests +on: + pull_request: + branches: + - dev + paths-ignore: + - 'ovos_bus_client/version.py' + - 'examples/**' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'README.md' + - 'scripts/**' + push: + branches: + - master + paths-ignore: + - 'ovos_bus_client/version.py' + - 'requirements/**' + - 'examples/**' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'README.md' + - 'scripts/**' + workflow_dispatch: + +jobs: + unit_tests: + strategy: + matrix: + python-version: [ 3.7, 3.8, 3.9, '3.10'] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v2 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev swig + python -m pip install build wheel + - name: Install repo + run: | + pip install -e . + - name: Install test dependencies + run: | + pip install -r tests/requirements.txt + - name: Run unittests + run: | + pytest --cov=ovos_bus_client --cov-report=xml tests/unittests + # NOTE: additional pytest invocations should also add the --cov-append flag + # or they will overwrite previous invocations' coverage reports + # (for an example, see OVOS Skill Manager's workflow) + - name: Upload coverage + if: "${{ matrix.python-version == '3.9' }}" + uses: codecov/codecov-action@v3 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: coverage.xml + verbose: true \ No newline at end of file diff --git a/demo.py b/demo.py deleted file mode 100755 index 6952e84..0000000 --- a/demo.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# Sample Padatious program used for testing - -import sys -from builtins import input -from glob import glob -from os.path import basename - -from ovos_padatious import IntentContainer - -reload_cache = len(sys.argv) > 1 and sys.argv[1] == '-r' -container = IntentContainer('intent_cache') - -for file_name in glob('data/*.intent'): - name = basename(file_name).replace('.intent', '') - container.load_file(name, file_name, reload_cache=reload_cache) - -for file_name in glob('data/*.entity'): - name = basename(file_name).replace('.entity', '') - container.load_entity(name, file_name, reload_cache=reload_cache) - -container.train() - -query = None -while query != 'q': - try: - query = input('> ') - except (KeyboardInterrupt, EOFError): - print() - break - data = container.calc_intent(query) - print(data.name + ': ' + str(data.conf)) - for key, val in data.matches.items(): - print('\t' + key + ': ' + val) diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 833cc5f..0000000 --- a/docs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -_build/ -_static/ -_templates/ diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 4adc931..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = python3 -msphinx -SPHINXPROJ = Padatious -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index be5599b..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Padatious documentation build configuration file -# - -import os -import sys - -sys.path.insert(0, os.path.abspath('../')) - -# General Configuration - -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon'] - -autodoc_mock_imports = ['fann2.libfann', 'xxhash'] - -templates_path = ['_templates'] -source_suffix = '.rst' -master_doc = 'index' - -# General Info -project = 'Padatious' -copyright = '2017, Mycroft AI' -author = 'Matthew Scholefield' - -version = '0.1.0' -release = '0.1.0' # Includes alpha/beta/rc tags. - -language = None -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# Syntax Highlighting -pygments_style = 'sphinx' - -todo_include_todos = False - -import sphinx_rtd_theme -html_theme = "sphinx_rtd_theme" -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -html_theme_options = { - 'navigation_depth': 4, -} - -html_static_path = [] -htmlhelp_basename = 'Padatiousdoc' - -# Options for LaTeX output - -latex_elements = {} -latex_documents = [ - (master_doc, 'Padatious.tex', 'Padatious Documentation', - 'Matthew Scholefield', 'manual'), -] - - -# Options for manual page output - -man_pages = [ - (master_doc, 'ovos_padatious', 'Padatious Documentation', - [author], 1) -] - - -# Options for Texinfo output - -texinfo_documents = [ - (master_doc, 'Padatious', 'Padatious Documentation', - author, 'Padatious', 'Neural Network Intent Parser.', - 'Miscellaneous'), -] - -# Options for Napoleon - -napoleon_google_docstring = True -napoleon_numpy_docstring = False - - diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index e0efc87..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Padatious documentation master file - -Padatious -========= - -*An efficient and agile neural network intent parser* - -.. toctree:: - :maxdepth: 4 - :caption: Contents: - -**IntentContainer** -=================== - -.. autoclass:: padatious.IntentContainer - :members: - :undoc-members: - -**MatchData** -============= - -.. autoclass:: padatious.MatchData - :members: diff --git a/ovos_padatious/version.py b/ovos_padatious/version.py new file mode 100644 index 0000000..be3eeb2 --- /dev/null +++ b/ovos_padatious/version.py @@ -0,0 +1,6 @@ +# START_VERSION_BLOCK +VERSION_MAJOR = 0 +VERSION_MINOR = 1 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 +# END_VERSION_BLOCK diff --git a/requirements.txt b/requirements.txt index 7a0c8ed..f885613 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ fann2>=1.0.7, < 1.1.0 xxhash -ovos-plugin-manager>=0.0.26a33 \ No newline at end of file +ovos-plugin-manager>=0.0.26 \ No newline at end of file diff --git a/setup.py b/setup.py index 4de0ce7..a083b87 100755 --- a/setup.py +++ b/setup.py @@ -1,19 +1,57 @@ #!/usr/bin/env python3 -from os.path import join, abspath, dirname + +import os +import os.path + from setuptools import setup -with open(join(dirname(abspath(__file__)), 'requirements.txt')) as f: - requirements = f.readlines() +BASEDIR = os.path.abspath(os.path.dirname(__file__)) -with open("README.md", "r") as fh: - long_description = fh.read() -PLUGIN_ENTRY_POINT = 'ovos-padatious-pipeline-plugin=ovos_padatious.opm:PadatiousPipeline' +def get_version(): + """ Find the version""" + version_file = os.path.join(BASEDIR, 'ovos_padatious', 'version.py') + major, minor, build, alpha = (None, None, None, None) + with open(version_file) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if int(alpha): + version += f"a{alpha}" + return version + +with open(os.path.join(BASEDIR, "README.md"), "r") as f: + long_description = f.read() + + +def required(requirements_file): + """ Read requirements file and remove comments and empty lines. """ + with open(os.path.join(BASEDIR, requirements_file), 'r') as f: + requirements = f.read().splitlines() + if 'MYCROFT_LOOSE_REQUIREMENTS' in os.environ: + print('USING LOOSE REQUIREMENTS!') + requirements = [r.replace('==', '>=') for r in requirements] + return [pkg for pkg in requirements + if pkg.strip() and not pkg.startswith("#")] + + +PLUGIN_ENTRY_POINT = 'ovos-padatious-pipeline-plugin=ovos_padatious.opm:PadatiousPipeline' setup( name='ovos-padatious', - version='0.4.8', # Also change in ovos_padatious/__init__.py + version=get_version(), description='A neural network intent parser', long_description=long_description, long_description_content_type="text/markdown", @@ -23,7 +61,7 @@ packages=[ 'ovos_padatious' ], - install_requires=requirements, + install_requires=required('requirements.txt'), zip_safe=True, classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..aad120b --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +flake8 +pytest \ No newline at end of file