Added a first cut of Dev/make commands (#446)

* Github actions local act helper files

- Add default actrc configuration for common arguments to act.
- Add event file to set global env var so test that act is present
can be used.
- Add env check to requirements update action.
- Add some possible input files and doc for limiting scope of
integration tests.

* Add a Makefile with shortcuts to common dev tasks

Make args are verbs for common development tasks that are
run locally.
Includes:
- creating independent virtualenvs for python tooling
- running tox lint, test, dep separately
- generating documentation pdf
- updating repo from upstream changes
- updating requirements files (with pip-compile) (calls act)
- running integration tests
- running black

* Setting version of python to default to 3.6 for PyCharm

.actrc

@@ -0,0 +1,2 @@
+-P self-hosted=nektos/act-environments-ubuntu:18.04
+--eventpath .github/act-events/default.json

.github/act-events/default.json

@@ -0,0 +1,3 @@
+{
+  "act": true
+}

.github/actions/update-requirements/action.yml

@@ -45,9 +45,16 @@ runs:
       run: |
         pip-compile --generate-hashes requirements/testing.in -o requirements/testing.txt
 
+    - name: Run pip-compile on documentation dependency requirements/document.in
+      shell: bash
+      run: |
+        pip-compile --generate-hashes requirements/document.in -o requirements/document.txt
+
     # commit and push changes (if any) on the same branch than the current commit.
     - name: Commit and push changes
       uses: devops-infra/[email protected]
       with:
         github_token: ${{ env.GITHUB_TOKEN }}
         commit_message: "Update python requirements files"
+      if: >
+        !env.ACT

.github/examples/README.md

@@ -0,0 +1,10 @@
+# Running act with workflows
+
+## Single workflow events
+
+If you want to run a single integration test with inputs use one of the event files in this directory, or create
+your own.
+
+```shell
+act -W .github/workflows/single_integration_test.yml --eventpath .github/examples/bdr-simple-event.json workflow_dispatch
+```

.github/examples/bdr-always-on-event.json

@@ -0,0 +1,9 @@
+{
+  "act": true,
+  "action": "workflow_dispatch",
+  "inputs": {
+      "tpa_architecture": "BDR-Always-ON",
+      "tpa_os_image": "tpa/debian:10",
+      "tpa_postgres_flavour": "2q"
+  }
+}

.github/examples/bdr-simple-event.json

@@ -0,0 +1,9 @@
+{
+  "act": true,
+  "action": "workflow_dispatch",
+  "inputs": {
+      "tpa_architecture": "BDR-Simple",
+      "tpa_os_image": "tpa/debian:10",
+      "tpa_postgres_flavour": "postgresql"
+  }
+}

.github/workflows/requirements.yml

@@ -31,4 +31,4 @@ jobs:
 
       - name: Run dependence tree with tox
         run: |
-          TOX_PARALLEL_NO_SPINNER=1 tox -e dep -p auto
+          TOX_PARALLEL_NO_SPINNER=1 tox -e dep

.github/workflows/single_integration_test.yml

@@ -0,0 +1,41 @@
+name: Single Integration Test
+
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      tpa_architecture:
+        description: "TPA architecture"
+        required: true
+        default: "BDR-Always-ON"
+      tpa_os_image:
+        description: "TPA OS image"
+        required: true
+        default: "tpa/redhat"
+      tpa_postgres_flavour:
+        description: "Postgres flavour"
+        required: true
+        default: "epas"
+
+env:
+  LANG: "C.UTF-8"
+  TPA_2Q_SUBSCRIPTION_TOKEN: ${{ secrets.TPA_2Q_SUBSCRIPTION_TOKEN }}
+  EDB_REPO_CREDENTIALS: ${{ secrets.EDB_REPO_CREDENTIALS }}
+  EDB_REPO_CREDENTIALS_FILE: /tmp/edb_repo.conf
+  ANSIBLE_SKIP_TAGS: pgbench
+
+jobs:
+
+  integration-test:
+    name: Integration test
+
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/integration-test
+        with:
+          tpa_architecture: ${{ github.event.inputs.tpa_architecture }}
+          tpa_os_image: ${{ github.event.inputs.tpa_os_image }}
+          tpa_postgres_flavour: ${{ github.event.inputs.tpa_postgres_flavour }}

.prospector.yaml

@@ -0,0 +1,9 @@
+pep257:
+  disable:
+    - D202
+    - D203
+    - D212
+    - D406
+    - D407
+ignore-patterns:
+  - platforms/.*/inventory/.*.py

Makefile

@@ -0,0 +1,77 @@
+# Makefile to assist with common tasks that would be executed by a developer
+#
+# Goal is to have `all` as
+# all: prep lint test document
+#
+# Other commands implemented and possible future commands to extend:
+# make  # create a venv, run lint, test, build, document generation/test
+# make <verb>  # Ran one of lint, test, clean, tag, etc.
+# make update  # git pull, merge changes from template, updated hooks
+# make tag  # bumped version for release, created git tag, pushed
+# make minor/major/patch  # like above for semver versioning
+
+WORKON_HOME ?= ~/.virtualenvs
+NAME = $(shell basename $(PWD))
+OPEN = $(shell command -v open || command -v xdg-open)
+
+all: prep dep_test test document
+.PHONY: all prep update
+
+# Note venv will not interfere with tpaexec tpa-venv if created so we can be clean in these operations
+venv:
+	mkdir -p $(WORKON_HOME)
+	python3.6 -m venv $(WORKON_HOME)/$(NAME)
+
+install_requirements: venv
+	$(WORKON_HOME)/$(NAME)/bin/pip install -r requirements.txt -r requirements/ansible.txt
+
+tox: venv
+	$(WORKON_HOME)/$(NAME)/bin/pip install tox
+
+lint: tox
+	$(WORKON_HOME)/$(NAME)/bin/tox -e py36-lint
+
+test: tox
+	$(WORKON_HOME)/$(NAME)/bin/tox -e py36-test
+	$(OPEN) test-output/tests.html
+	sleep 1
+	$(OPEN) test-output/coverage/index.html
+
+dep_test: tox
+	$(WORKON_HOME)/$(NAME)/bin/tox -e dep
+
+integ_test:
+	act -W .github/workflows/simple_integration_tests.yml workflow_dispatch
+
+document: venv
+	$(WORKON_HOME)/$(NAME)/bin/pip install -r requirements/document.txt
+	. $(WORKON_HOME)/$(NAME)/bin/activate && $(MAKE) -C docs all
+	$(OPEN) docs/pdf/tpaexec.pdf
+
+pull:
+	git pull
+
+update: pull install_requirements
+
+install_update_reqs: venv
+	$(WORKON_HOME)/$(NAME)/bin/pip install pip-tools
+
+%.txt: %.in
+	pip-compile --generate-hashes -o $@ $<
+
+REQUIREMENTS_IN = $(wildcard *.in requirements/*in)
+REQUIREMENTS_TXT = $(REQUIREMENTS_IN:.in=.txt)
+.PHONY: $(REQUIREMENTS_IN)
+update_reqs: install_update_reqs $(REQUIREMENTS_TXT)
+
+black:
+	$(WORKON_HOME)/$(NAME)/bin/pip install black
+	$(WORKON_HOME)/$(NAME)/bin/black lib library
+
+prep: update_reqs black
+
+clean:
+	rm -rf $(WORKON_HOME)/$(NAME) .tox .pytest_cache .coverage \
+	test-output coverage-reports workflow docs/pdf/*.pdf ansible.log
+	find . -name \*.pyc -delete
+	find . -depth -name __pycache__ -delete

requirements/document.in

@@ -0,0 +1,4 @@
+mkdocs
+mkdocs-material
+mkdocs-exclude
+mkdocs-with-pdf