#99 coverage and security checking (#100)

* #99 add coverage and security checking

* #99 cleanup

* #99 cleanup

* #99 black reformatting

* #99 cleanup

* #99 reference links and mypy fixes

* #99 cleanup

* format

* #99 Add testdata directory

* #99 Patch template-python for mypy

* #99 Patch template-python for mypy 2

* #99 Patch template-python for mypy 3

* #99 Update Makefile 3

* add placeholder

* remove unsupported versions

* debug

* debug

* debug

* sanity check

* add coveragerc

* setup.cfg

* test

* test

* test

* fix coverage issue

* update installer version

* revert win and darwin var name

* fix linting

---------

Co-authored-by: docktermj <[email protected]>

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+relative_files = true

.github/workflows/add-to-project-g2-python-dependabot.yaml

@@ -4,6 +4,9 @@ on:
   pull_request:
     branches: [main]
 
+permissions:
+  repository-projects: write
+
 jobs:
   add-to-project-dependabot:
     secrets:

.github/workflows/bandit.yaml

@@ -0,0 +1,29 @@
+name: bandit
+
+on:
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '.github/**'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  bandit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Bandit Scan
+        uses: lukehinds/bandit-action@new-action
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          path: " src"
+          recursive: "true"

.github/workflows/black.yaml

@@ -0,0 +1,34 @@
+name: black
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  black:
+    name: black Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: update pip
+        run: python3 -m pip install --upgrade pip
+
+      - name: pip install python packages for testing
+        # run: pip install black pylint mypy
+        run: pip install black
+
+      - name: black testing
+        run: |
+          # shellcheck disable=SC2046
+          black --diff --check $(git ls-files '*.py' ':!:docs/source/*')

.github/workflows/dependabot-approve-and-merge.yaml

@@ -4,11 +4,12 @@ on:
   pull_request:
     branches: [main]
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   dependabot-approve-and-merge:
-    permissions:
-      contents: write
-      pull-requests: write
     secrets:
       SENZING_GITHUB_CODEOWNER_PR_RW_TOKEN: ${{ secrets.SENZING_GITHUB_CODEOWNER_PR_RW_TOKEN }}
     uses: senzing-factory/build-resources/.github/workflows/dependabot-approve-and-merge.yaml@v1

.github/workflows/dependency-scan.yaml

@@ -0,0 +1,54 @@
+name: dependency scan
+
+on:
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '.github/**'
+
+env:
+  DEP_PATH: requirements.txt
+
+permissions:
+  contents: read
+
+jobs:
+  safety:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Security vulnerabilities scan
+        uses: aufdenpunkt/[email protected]
+        with:
+          safety_args: '-i 62044'
+
+  pip-audit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - name: pip install
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install .
+
+      - uses: pypa/[email protected]
+        with:
+          inputs: requirements.txt

.github/workflows/docker-build-container.yaml

@@ -6,6 +6,9 @@ on:
       - main
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   docker-build-container:
     runs-on: ubuntu-latest

.github/workflows/flake8.yaml

@@ -0,0 +1,32 @@
+name: flake8
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  flake8:
+    name: flake8 Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - run: pwd; ls -la
+
+      - name: flake8 Lint
+        uses: py-actions/flake8@v2
+        with:
+          max-line-length: 88
+          path: src
+          plugins: flake8-black

.github/workflows/isort.yaml

@@ -0,0 +1,19 @@
+name: isort
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  isort:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: isort/isort-action@v1
+        with:
+          configuration:
+          requirements-files: requirements.txt
+          sort-paths: "src"

.github/workflows/lint-workflows.yaml

@@ -9,6 +9,7 @@ on:
 permissions:
   contents: read
   packages: read
+  pull-requests: read
   statuses: write
 
 jobs:

.github/workflows/mypy.yaml

@@ -0,0 +1,35 @@
+name: mypy
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  mypy:
+    name: mypy Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: update pip
+        run: python3 -m pip install --upgrade pip
+
+      - name: pip install mypy
+        # run: pip install black pylint mypy
+        run: pip install mypy
+
+      - name: mypy testing
+        #run: mypy --strict $(git ls-files '*.py' ':!:docs/source/*')
+        run: |
+          # shellcheck disable=SC2046
+          mypy --strict $(git ls-files '*.py' ':!:docs/source/*' ':!:tests/*')

.github/workflows/pylint.yaml

@@ -2,12 +2,15 @@ name: pylint
 
 on: [push]
 
+permissions:
+  contents: read
+
 jobs:
   pylint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/pytest-darwin.yaml

@@ -0,0 +1,73 @@
+name: pytest darwin
+
+on: [pull_request, workflow_dispatch]
+
+env:
+  PYTHONPATH: /Users/runner/work/g2-sdk-python-next/g2-sdk-python-next/src
+  SENZING_TOOLS_DATABASE_URL: 'sqlite3://na:na@/tmp/sqlite/G2C.db'
+
+jobs:
+  pytest-darwin:
+    name: "pytest Senzing: ${{ matrix.senzingapi-version }}; OS: ${{ matrix.os }}; Python ${{ matrix.python-version }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        # 3.8 and 3.9 are unsupported on the darwin runner
+        python-version: ["3.10", "3.11"]
+        senzingapi-version: [production-v3]
+
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install psutil pytest pytest-cov pytest-schema
+
+      - if: matrix.python-version == '3.8' || matrix.python-version == '3.9'
+        name: install python libraries for python 3.8 and 3.9
+        run: pip install typing_extensions
+
+      - name: install Senzing API
+        uses: senzing-factory/github-action-install-senzing-api@v3
+        with:
+          senzingapi-version: ${{ matrix.senzingapi-version }}
+
+      - name: copy /etc files
+        run: sudo cp testdata/senzing-license/g2.lic /opt/senzing/g2/etc/g2.lic
+
+      - name: copy test database files
+        run: mkdir /tmp/sqlite && cp testdata/sqlite/G2C.db /tmp/sqlite/G2C.db
+
+      - name: run pytest on tests
+        run: |
+          export DYLD_LIBRARY_PATH=/opt/senzing/g2/lib:/opt/senzing/g2/lib/macos
+          pytest tests/ --verbose --capture=no --cov=src
+
+      - name: rename coverage file
+        env:
+          COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
+        run: |
+          mv .coverage "$COVERAGE_FILE"
+
+      - name: Store coverage file
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: .coverage.${{ matrix.python-version }}
+
+  coverage:
+    name: coverage
+    needs: pytest-darwin
+    permissions:
+      pull-requests: write
+      contents: write
+    uses: senzing-factory/build-resources/.github/workflows/python-coverage-comment.yaml@v1