Adjusted CI pipeline for build stage

Removed dependency on Cloud Foundry for testing
Re-wrote startup logic and moved relevant code to other files
Added pylint error linting to M2EE; fixed or disabled errors
Merge lint and test requirements to dev requirements
Added CI configuration generator
Add mxbuild to gitignore
Externalize test host; add correct host lookup in Travis configuration
Test and documentation optimizations
Allow for no routes (PR mendix#213)

.gitignore

@@ -142,7 +142,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+# lib/
 lib64/
 parts/
 sdist/
@@ -238,3 +238,4 @@ requirements-**.txt
 
 .vscode
 .local
+mxbuild/

.travis.yml

@@ -1,52 +1,94 @@
+# This file is autogenerated from .travis.yml.j2
+#
+#   To generate, run dev/generate_travis_ci_config.py
+#
+
 if: tag is blank
 
 os: linux
-
 dist: bionic
-
 language: python
+python: 3.6
 
-python: 3.6.9
+services:
+  - docker
 
 env:
   global:
     - TEST_PREFIX=travis-$TRAVIS_JOB_ID
-    - CF_DIAL_TIMEOUT=30
 
-before_install:
-  - wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
-  - echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
-  - sudo apt-get update && sudo apt-get install -y cf-cli
-  - sudo curl -s https://download.mendix.com/Mendix-CA-G2.crt -o /usr/local/share/ca-certificates/ca.crt && sudo update-ca-certificates
+cache:
+  directories:
+  - $HOME/.cache/pip
+  - $HOME/.cf
+  - dist
 
 install:
-  - make install_build_requirements install_test_requirements
-
-before_script:
-  - curl ifconfig.co/ip
-  - make login
+  - make install_requirements
 
-script:
-  - make test &
-  - MAKE_PID=$!
-  - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
-  - sleep 15
-  - wait $MAKE_PID
-  - sleep 5
+_install_cf_cli_and_local: &install_cf_cli_and_local |-
+  wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+  echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+  sudo apt-get update && sudo apt-get install -y cf-cli
+  sudo curl -s https://download.mendix.com/Mendix-CA-G2.crt -o /usr/local/share/ca-certificates/ca.crt && sudo update-ca-certificates
+  cf install-plugin cflocal -f
 
-after_script:
-  - make clean_cf
+_get_host: &get_host |-
+  export TEST_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
+  echo $TEST_HOST
 
 jobs:
   include:
-    - stage: lint
-      before_install: skip
-      install: make install_lint_requirements install_build_requirements install_test_requirements
-      before_script: skip
-      script: make lint
-      after_script: skip
+    - stage: build
+      script: make lint test_unit build
+    - stage: test
+      before_install:
+        - *install_cf_cli_and_local
+      before_script:
+        - *get_host
+      script:
+        - make test_integration TEST_FILES='tests/integration/test_access_restrictions.py tests/integration/test_appdynamics.py tests/integration/test_client_certificates.py tests/integration/test_constants.py tests/integration/test_custom_headers.py tests/integration/test_custom_runtime_settings.py' &
+        - MAKE_PID=$!
+        - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
+        - sleep 15
+        - wait $MAKE_PID
+        - sleep 5
+    - stage: test
+      before_install:
+        - *install_cf_cli_and_local
+      before_script:
+        - *get_host
+      script:
+        - make test_integration TEST_FILES='tests/integration/test_datadog.py tests/integration/test_debug_container.py tests/integration/test_deprecations.py tests/integration/test_emit_metrics.py tests/integration/test_instadeploy.py tests/integration/test_jdbc_parameters.py' &
+        - MAKE_PID=$!
+        - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
+        - sleep 15
+        - wait $MAKE_PID
+        - sleep 5
+    - stage: test
+      before_install:
+        - *install_cf_cli_and_local
+      before_script:
+        - *get_host
+      script:
+        - make test_integration TEST_FILES='tests/integration/test_jdk_versions.py tests/integration/test_jetty_config.py tests/integration/test_logging.py tests/integration/test_mda_app_deployed.py tests/integration/test_mono.py' &
+        - MAKE_PID=$!
+        - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
+        - sleep 15
+        - wait $MAKE_PID
+        - sleep 5
     - stage: test
-      env: CF_STACK=cflinuxfs3
+      before_install:
+        - *install_cf_cli_and_local
+      before_script:
+        - *get_host
+      script:
+        - make test_integration TEST_FILES='tests/integration/test_mpk_app_deployed.py tests/integration/test_mxadmin_exposed_authorized.py tests/integration/test_telegraf.py tests/integration/test_telegraf_datadog.py tests/integration/test_termination.py' &
+        - MAKE_PID=$!
+        - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
+        - sleep 15
+        - wait $MAKE_PID
+        - sleep 5
 
 notifications:
-  email: false
+  email: false

.travis.yml.j2

@@ -0,0 +1,56 @@
+{{ generated_notice }}
+if: tag is blank
+
+os: linux
+dist: bionic
+language: python
+python: 3.6
+
+services:
+  - docker
+
+env:
+  global:
+    - TEST_PREFIX=travis-$TRAVIS_JOB_ID
+
+cache:
+  directories:
+  - $HOME/.cache/pip
+  - $HOME/.cf
+  - dist
+
+install:
+  - make install_requirements
+
+_install_cf_cli_and_local: &install_cf_cli_and_local |-
+  wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+  echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+  sudo apt-get update && sudo apt-get install -y cf-cli
+  sudo curl -s https://download.mendix.com/Mendix-CA-G2.crt -o /usr/local/share/ca-certificates/ca.crt && sudo update-ca-certificates
+  cf install-plugin cflocal -f
+
+_get_host: &get_host |-
+  export TEST_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')
+  echo $TEST_HOST
+
+jobs:
+  include:
+    - stage: build
+      script: make lint test_unit build
+{%- for tests in testset %}
+    - stage: test
+      before_install:
+        - *install_cf_cli_and_local
+      before_script:
+        - *get_host
+      script:
+        - make test_integration TEST_FILES='{{ tests|join(' ') }}' &
+        - MAKE_PID=$!
+        - while [ -e /proc/$MAKE_PID ]; do sleep 60 && echo -n "."; done
+        - sleep 15
+        - wait $MAKE_PID
+        - sleep 5
+{%- endfor %}
+
+notifications:
+  email: false

.version

@@ -1 +1 @@
-v4.7.0
+v4.8.0

CONTRIBUTING.md

@@ -1,26 +1,30 @@
 # Contributing to the Mendix Buildpack
+
 This document contains guidelines for contributing to the Mendix Buildpack (issues, PRs). Use in conjunction with [`DEVELOPING.md`](DEVELOPING.md) and [`LICENSE`](LICENSE).
 
 We welcome your contributions! Please reach out to a maintainer if you need help or if you have any questions after reading this document.
 
 ## General Rules and Guidelines
+
 The following rules and guidelines apply when contributing to the buildpack:
 
 * For general Mendix support, please reach out to our regular channels instead of opening an issue specific to this buildpack.
 * We require that you accept the license.
 * If you're not a maintainer, use your own fork to develop and submit your PR on.
-* New releases are always tagged `vX.X.X`. We make liberal use of semantic versioning.
-* Our code complies to PEP8, is formatted in [`black`]((https://github.com/psf/black)) and linted in `pylint`.
+* New releases are always tagged `vX.X.X` . We make liberal use of semantic versioning.
+* Our code complies to PEP8, is formatted in [`black`]((https://github.com/psf/black)) and linted in `pylint` .
 
 ## Issues
+
 We have no formal issue template. If you submit an issue, please make sure that it contains all context we need to get started on it. Think logs, reproduction scenarios, etc. Any issue which is not complete will not be picked up, and please respect that we may ask you for any information we need to get started on an issue if it is not complete.
 
 ## PRs
-The following guidelines must be respected to get your PR merged to `master`:
+
+The following guidelines must be respected to get your PR merged to `master` :
 
 * Rebase your git history in such a way that each commit makes one consistent change. Don't include separate "fixup" commits later on.
 * Submit your PR with all the information we need to review it. The same applies to PRs as it does to issues.
 * The code must pass linting and all integration tests (in Travis). Code which does not pass will not be considered / reviewed. We have to be this strict since all deployments to the Mendix public cloud use the `master` branch.
-* Always bump the version number in `buildpack/start.py` to an appropriate value.
+* Always bump the version number in `.version` to an appropriate value.
 * There is a minimum of one reviewer per PR.
 * After a PR is merged, a release must be created with the appropriate version.

DEVELOPING.md

@@ -1,107 +1,108 @@
 # Mendix Buildpack Development
+
 This document describes best practices of developing the Mendix Buildpack. Use in conjunction with [`CONTRIBUTING.md`](CONTRIBUTING.md).
 
 ## Buildpack Structure
+
 The buildpack features the following directories:
-* `bin`: Cloud Foundry buildpack lifecycle scripts, utility scripts and binaries live here
-* `buildpack`: All Python code lives here, this is the home of the main buildpack module. Entry points are `compile.py` and `start.py`
-* `docs`: Extra documentation in Markdown format lives here
-* `etc`: Configuration templates for e.g. nginx and M2EE live here
-* `lib`: A forked version of M2EE Tools suited for the cloud lives here, used for working with the Mendix Runtime
-* `tests`: All test code lives here
-* `vendor`: All vendorized dependencies live here
+
+* `bin` : Cloud Foundry buildpack lifecycle scripts, utility scripts and binaries live here
+* `buildpack` : All Python code lives here, this is the home of the main buildpack module. Entry points are `compile.py` and `start.py`
+* `dev` : Code for local development and CI lives here
+* `docs` : Extra documentation in Markdown format lives here
+* `etc` : Configuration templates for e.g. nginx and M2EE live here
+* `lib` : A forked version of M2EE Tools suited for the cloud lives here, used for working with the Mendix Runtime
+* `tests` : All test code lives here
+* `vendor` : All vendorized dependencies live here
 
 ## Prerequisites
-The buildpack is a Python project, and you must have some experience in Python, bash and the `make` system to develop.
 
-For developing the buildpack, `pyenv` must be set up, and the dependencies for testing and linting must be installed. Additionally, you must have a Cloud Foundry cluster available for testing with all the required service brokers installed, and configure your environment to point to that cluster.
+The buildpack is a Python project, and you must have some experience in Python, `bash` and the `make` system to develop.
+
+For developing the buildpack, you must set up the following:
+
+* A shell / terminal of your choice
+* [Docker](https://www.docker.com/)
+* The [Cloud Foundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html) and the [`cf-local`](https://github.com/cloudfoundry-incubator/cflocal) plugin
+* A [`pyenv virtualenv`](https://github.com/pyenv/pyenv-virtualenv) virtual environment (or you can choose to develop in a Docker container)
 
 ### Installing `pyenv`
-[pyenv](https://github.com/pyenv/pyenv) in combination with [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv) can be used to create a local sandbox to develop in. Note that you'll have to create an environment with the latest version of Python 3.6 - the default in the Cloud Foundry root filesystem (`cflinuxfs3`) we use to deploy applications on.
+
+[`pyenv`](https://github.com/pyenv/pyenv) in combination with [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) can be used to create a local Python virtual environment to develop in. Note that you'll have to create an environment with the latest version of Python 3.6 - the default in the Cloud Foundry root filesystem (`cflinuxfs3`) we use to deploy applications on.
 
 ### Developing in `Docker`
-As an alternative to running Python on your host you can run it in Docker container.
-* set up required environment variables
-* go to `./dev` folder
-* execute `bash run-locally.sh`
 
-This will start the Docker container with preinstalled Python and "ssh" you into it.
-The project folder will be mapped to the current folder in the Docker container, so if you edit files on you host 
-the changes will be immediately available in the container.
+As an alternative to running Python on your host you can run it in a Docker container. To do this:
+
+* Set up required environment variables
+* Go to the `dev/` directory
+* Run `./run-locally.sh`
+This will start the Docker container with preinstalled Python and provide you with an interactive shell.
+The project folder will be mapped to the current folder in the Docker container, so if you edit files on your host, the changes will be immediately available in the container.
 
 ### Installing testing and linting requirements
-The buildpack makes use of the `make` system. For dependency management, `pip-compile` is used under the hood.
 
-If you're using a Mac, be sure to use Brew to install `coreutils` and `findutils`.
+The buildpack makes use of the `make` system. For dependency management, `pip-compile` is used under the hood.
 
 A few `make` targets to use are:
 
-* `vendor`: downloads the Python runtime dependencies as wheels into `vendor/wheels`
-* `install_build_requirements`: installs the build / runtime requirements
-* `install_test_requirements`: installs the test requirements
-* `install_lint_requirements`: installs the linting / formatting requirements
+* `vendor` : downloads the Python runtime dependencies as wheels into `vendor/wheels`
+* `install_requirements` : installs all requirements and generates requirements.txt
 
 **Never change the `requirements*.txt` files directly!** Use `requirements*.in` to that.
 
 ### Setting up your environment
-For integration tests, you need to use a fully-featured Cloud Foundry cluster containing the Mendix S3, RDS and backup (Schnapps) service brokers. Once you have that in place, you'll need to set up the following environment variables:
 
-```
-export BUILDPACK="<your custom buildpack name>"
-export TEST_PREFIX="<prefix identifying your test run app for others>
-export CF_ENDPOINT="<your testing cluster API endpoint>"
-export CF_USER="<your username>"
-export CF_PASSWORD="<your password>"
-export CF_ORG="<test org>"
-export CF_SPACE="<test space>"
-export CF_DOMAIN="<your test domain>"
-export MX_PASSWORD="<sample password for Mendix apps deployed during testing>"
+For integration tests, you need to have installed the prerequisites. Once you have those in place, you can set up the following environment variables:
+
+``` shell
+export TEST_PREFIX="<prefix identifying your test run; default=test>"
+export TEST_PROCESSES="<amount of simultaneous tests to run; default=2>"
+export TEST_HOST="<custom host the tests will use to connect to dependencies; default=host.docker.internal>"
+export TEST_MEMORY="<memory a test app container gets; default=1G>"
+export TEST_DISK="<disk space a test app container gets; default=1G>"
 ```
 
 ## Building
+
 To ensure that your CF cluster has the buildpack you're developing available, use the following `make` targets:
 
-* `clean`: removes all the nasties, including leftover Mendix files, from your working directory
-* `build`: builds the buildpack, i.e. updates / fetches all dependencies that need to be in source control, including all runtime Python dependencies as wheels
-* `upload`: uploads your working directory as `$BUILDPACK` to your Cloud Foundry testing cluster
-* `install`: same as `build upload`
+* `clean` : removes all the nasties, including leftover Mendix files, from your working directory
+* `lint` : ensures that code adheres to our standards
+* `build` : builds the buildpack, i.e. updates / fetches all dependencies that need to be in source control, including all runtime Python dependencies as wheels, and compresses it to `dist/`
 
 ## Testing
-We have split up tests into unit tests, which do not need a CF cluster to run, and integration tests, which do.
-
-All tests applications deployed have a prefix so that the test script can clean up before and after any applications that are left behind. However, this limits the number or runs into the same org and space to one (1).
 
-To overcome this, you can specify `TEST_PREFIX` as environment variable to change the naming of the test apps. `TEST_PREFIX` defaults to `ops`. 
+We have split up tests into unit tests, which do not need to fully start a Mendix application, and integration tests, which do.
 
 ## Running Tests
+
 To run all the tests locally you need to do is to go to the root folder and run the following command:
 
-```
+``` shell
 make test
 ```
 
 ### Running Unit Tests
+
 To run the unit (offline) tests only, run the following command:
 
-```
+``` shell
 make test_unit
 ```
 
 ### Running Integration Tests
+
 To run the integration (online) tests only, run the following command:
 
-```
+``` shell
 make test_integration
 ```
 
-If your tests fail, be sure to clean up the Cloud Foundry environment with:
-
-```
-make clean_cf
-```
+You can keep watch on the tests with regular Docker commands such as `docker ps`.
 
-To run a one separate test do:
+To run one or more separate tests do:
 
+``` shell
+make test_integration TEST_FILES='file1, file2'
 ```
-nosetests --nocapture --verbosity=3 --process-timeout=3600 --with-timer --timer-no-color tests/integration/test-file-name.py
-```