From 6b92294c2f53520f6fe3d2b2392fbeb5c39902ba Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Thu, 21 Nov 2024 15:29:37 +0000 Subject: [PATCH 1/9] DES-6613 Create destination dir if it doesn't exist. --- build.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index a44dd69..3428ed8 100755 --- a/build.py +++ b/build.py @@ -189,12 +189,15 @@ def parse_version(version): return tuple([ int(n) for n in version.split('.') ]) def prepare_destination(destination_path): - with os.scandir(destination_path) as entries: - for entry in entries: - if entry.is_dir() and not entry.is_symlink(): - shutil.rmtree(entry.path) - else: - os.remove(entry.path) + if os.path.exists(destination_path): + with os.scandir(destination_path) as entries: + for entry in entries: + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry.path) + else: + os.remove(entry.path) + else: + os.makedirs(destination_path, 0o755) project_path = os.path.join(destination_path, 'project') os.mkdir(project_path, 0o755) shutil.copytree('scripts', os.path.join(destination_path, 'scripts')) From c76e6aaaa60e88ae057c63e0e28d3f133838adcb Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Fri, 22 Nov 2024 10:00:55 +0000 Subject: [PATCH 2/9] DES-6665 Fixed Java version detection. * Check existence of java-version file instead of just the directory. * Use show-java-version as a cleaner way to detect Java version. * When using dump-mpr, read the entire output & wait for exit code - so that errors are not streamed to jq. * Log how exactly a Mendix version was detected. --- mxbuild/build | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mxbuild/build b/mxbuild/build index 9fad08b..7b1b2a7 100755 --- a/mxbuild/build +++ b/mxbuild/build @@ -15,18 +15,28 @@ fi cd /workdir -if [ -f /workdir/project ]; then - JAVA_VERSION=$(cat java-version) +if [ -f /workdir/project/java-version ]; then + echo "Using java-version to select Java version" + JAVA_VERSION=$(cat /workdir/project/java-version) elif [ -f /opt/mendix/modeler/mx ]; then - JAVA_VERSION=$(/opt/mendix/modeler/mx dump-mpr --unit-type 'Settings$ProjectSettings' /workdir/project/"${MPR_FILENAME}" | \ - jq -r '.units[] | select(.["$Type"]=="Settings$ProjectSettings") | .["settingsParts"][] | select(.["$Type"]=="Settings$RuntimeSettings").javaVersion | if (. == null or . == "null") then "Java11" else . end') + if JAVA_VERSION=$(/opt/mendix/modeler/mx show-java-version /workdir/project/"${MPR_FILENAME}"); then + echo "Using mx show-java-version to select Java version" + elif JAVA_VERSION=$(PROJECT_SETTINGS=`/opt/mendix/modeler/mx dump-mpr --unit-type 'Settings$ProjectSettings' /workdir/project/"${MPR_FILENAME}"` && \ + echo -n $PROJECT_SETTINGS | \ + jq -r '.units[] | select(.["$Type"]=="Settings$ProjectSettings") | .["settingsParts"][] | select(.["$Type"]=="Settings$RuntimeSettings").javaVersion | if (. == null or . == "null") then "Java11" else . end'); then + echo "Using mx dump-mpr to select Java version" + else + echo "Unable to detect Java version using mx tool" + JAVA_VERSION=11 + fi else + echo "mx tool is not available" JAVA_VERSION=11 fi JAVA_VERSION=$(echo -n $JAVA_VERSION| sed s/\^Java// | head) -echo "Detected Java $JAVA_VERSION" +echo "Using Java $JAVA_VERSION" export JDK_HOME=/etc/alternatives/java_sdk_${JAVA_VERSION} $MXBUILD_COMMAND \ From 50bf803c7fc92a7c0c351ff37ece94d6f77fcdd3 Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Fri, 22 Nov 2024 15:14:13 +0000 Subject: [PATCH 3/9] DES-6613 Bumped up CF Buildpack to v5.0.23. --- README.md | 2 +- rootfs-builder.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1e219b8..dea56aa 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ docker push When building the the `rootfs-builder.dockerfile` file, you can provide the following additional arguments: -- **CF_BUILDPACK** is a version of CloudFoundry buildpack. Defaults to `v5.0.16`. For stable pipelines, it's recommended to use a fixed **v5.0.16** version. Other Cloud Foundry buildpacks might not work with this version of Docker Buildpack. +- **CF_BUILDPACK** is a version of CloudFoundry buildpack. Defaults to `v5.0.23`. For stable pipelines, it's recommended to use a fixed **v5.0.23** version. Other Cloud Foundry buildpacks might not work with this version of Docker Buildpack. - **CF_BUILDPACK_URL** specifies the URL where the CF buildpack should be downloaded from (for example, a local mirror). Defaults to `https://github.com/mendix/cf-mendix-buildpack/releases/download/${CF_BUILDPACK}/cf-mendix-buildpack.zip`. Specifying **CF_BUILDPACK_URL** will override the version from **CF_BUILDPACK**. - **BUILDPACK_XTRACE** can be used to enable CF Buildpack [debug logging](https://github.com/mendix/cf-mendix-buildpack#logging-and-debugging). Set this variable to `true` to enable debug logging. diff --git a/rootfs-builder.dockerfile b/rootfs-builder.dockerfile index 215583b..5498c31 100644 --- a/rootfs-builder.dockerfile +++ b/rootfs-builder.dockerfile @@ -10,7 +10,7 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 # CF buildpack version -ARG CF_BUILDPACK=v5.0.20 +ARG CF_BUILDPACK=v5.0.23 # CF buildpack download URL ARG CF_BUILDPACK_URL=https://github.com/mendix/cf-mendix-buildpack/releases/download/${CF_BUILDPACK}/cf-mendix-buildpack.zip From 62482c25eca366721ede74c5b7f355f2877d8c4e Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Mon, 25 Nov 2024 11:55:44 +0100 Subject: [PATCH 4/9] Removed unused build_image code. It could potentially be restored later, if build.py would also run the Docker build. --- build.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.py b/build.py index 3428ed8..1f19b12 100755 --- a/build.py +++ b/build.py @@ -227,14 +227,6 @@ def prepare_mda(source_path, destination_path, artifacts_repository=None): else: raise Exception('No supported files found in source path') -def build_image(mda_dir): - # TODO: build the full image, or just copy MDA into destination? - mda_path = mda_dir.name if isinstance(mda_dir, tempfile.TemporaryDirectory) else mda_dir - mda_metadata = get_metadata_value(mda_path) - mx_version = mda_metadata['RuntimeVersion'] - java_version = mda_metadata.get('JavaVersion', 11) - logging.debug("Detected Mendix {} Java {}".format(mx_version, java_version)) - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Build a Mendix app') parser.add_argument('--source', metavar='source', required=True, nargs='?', type=pathlib.Path, help='Path to source Mendix app (MDA file, MPK file, MPR directory or extracted MDA directory)') @@ -250,4 +242,3 @@ def build_image(mda_dir): except KeyboardInterrupt: stop_processes() raise - # build_image(args.destination) From 5f6ae009910761a215b65af342f7041d6963f620 Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Mon, 25 Nov 2024 11:59:10 +0100 Subject: [PATCH 5/9] DES-6613 Updated documentation. * Clarified requirements and limitations * Added an upgrade guide explaining how to use build.py in existing pipelines * Updated versions --- README.md | 17 +++++++------ upgrading-from-v5.md | 59 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 upgrading-from-v5.md diff --git a/README.md b/README.md index dea56aa..dadc986 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The Mendix Buildpack for Docker (aka docker-mendix-buildpack) is an **example project** you can use to build and run your Mendix Application in a [Docker](https://www.docker.com/) container. -**⚠️ Warning** If your pipeline is based on Docker Buildpack V4 or an earlier version, see the [upgrading from Docker Buildpack v4](upgrading-from-v4.md) document. To use Docker Buildpack v5, some changes will be required in your build process. +**⚠️ Warning** If your pipeline is based on Docker Buildpack v5 or an earlier version, see the [upgrading from Docker Buildpack v5](upgrading-from-v6.md) document. To use Docker Buildpack v6, some changes will be required in your build process. For a Kubernetes native solution to run Mendix apps, see [Mendix for Private Cloud](https://www.mendix.com/evaluation-guide/app-lifecycle/mendix-for-private-cloud/). @@ -42,8 +42,13 @@ This project is a goto reference for the following scenarios : * Docker 20.10 (Installation [here](https://docs.docker.com/engine/installation/)) * Earlier Docker versions are no longer compatible because they don't support multistage builds. To use Docker versions below 20.10, download an earlier Mendix Docker Buildpack release, such as [v2.3.2](https://github.com/mendix/docker-mendix-buildpack/releases/tag/v2.3.2) + * Alternatively, Podman version 5 or later * Python 3.8 -* For preparing, a local installation of `curl` + * No additional dependencies are needed +* A UNIX-like operating system, such as Linux or macOS, or Windows Subsystem for Linux +* An x86-64 (AMD64) based CPU + * ARM64 CPUs are not fully supported +* For running the example tests, a local installation of `curl` * For local testing, make sure you can run the [docker-compose command](https://docs.docker.com/compose/install/) * A Mendix app based on Mendix 8 or a later version @@ -52,7 +57,7 @@ This project is a goto reference for the following scenarios : ### Preparation: rootfs To save build time, the build pack needs a prebuilt rootfs containing the base OS and additional packages. -This rootfs is based on [Red Hat Universal Base Image 8 minimal](https://developers.redhat.com/articles/ubi-faq) image. +This rootfs is based on [Red Hat Universal Base Image 9 minimal](https://developers.redhat.com/articles/ubi-faq) image. To build the rootfs, run the following commands @@ -127,7 +132,7 @@ docker build \ For build you can provide next arguments: -- **BUILD_PATH** indicates where the application model is located. It is a root directory of an unzipped .MDA or .MPK file. In the latter case, this is the directory where your .MPR file is located. Must be within [build context](https://docs.docker.com/engine/reference/commandline/build/#extended-description). Defaults to `./project`. +- **BUILD_PATH** indicates where the application model is located. It is a root directory of an unzipped .MDA or .MPK file. In the latter case, this is the directory where your .MPR file is located. Must be within [build context](https://docs.docker.com/engine/reference/commandline/build/#extended-description). Should not be used when using the `build.py` script. Defaults to `./project`. - **ROOTFS_IMAGE** is a type of rootfs image. Defaults to `mendix-rootfs:app` (a locally prebuilt image). - **BUILDER_ROOTFS_IMAGE** is a type of rootfs image used for downloading the Mendix app dependencies and compiling the Mendix app from source. Defaults to `mendix-rootfs:builder` (a locally prebuilt image). - **EXCLUDE_LOGFILTER** will exclude the `mendix-logfilter` binary from the resulting Docker image if set to `true`. Defaults to `true`. Excluding `mendix-logfilter` will reduce the image size and remove a component that's not commonly used; the `LOG_RATELIMIT` environment variable option will be disabled. @@ -398,9 +403,7 @@ Contributions are welcomed: ### Build Details -This was built with the following: - -* Docker version 20.10 +Docker Buildpack is tested by running a Github Actions pipeline. ### Versioning diff --git a/upgrading-from-v5.md b/upgrading-from-v5.md new file mode 100644 index 0000000..8ed228f --- /dev/null +++ b/upgrading-from-v5.md @@ -0,0 +1,59 @@ +# Upgrading from Docker Buildpack v5 + +Docker Buildpack v6 contains a breaking change and might require some changes in your CI/CD pipeline: + +Building Mendix projects from source (\*.mpr or \*.mpk files) is now done using a build.py script. +If your CI/CD pipeline uses Docker Buildpack to build \*.mda files (compiled Mendix apps), no further changes are needed. + +If you're upgrading from Docker Buildpack v4 (or an older version), you'll also need to follow the [upgrading from Docker Buildpack v4](upgrading-from-v4.md) instructions. + +## Using the build.py script + +Docker Buildpack v6 no longer uses CF Buildpack to compile MPR (or MPK) files - to continue supporting newer versions of Mendix, Java and the base OS. +Instead, a custom `build.py` script will: + +1. Prepare a clean [Docker context](https://docs.docker.com/build/concepts/context/) in the path specified by `--destination`. All files required to build the app image will be copied to this destination. +2. Detect the file type of the source path specified by the `--source` arg (an MPK file, an MPR file, an MDA file or an unpacked MDA directory). +3. If necessary (`--source` specifies project that needs to be compiled) + 1. Create an image containing [mxbuild](https://docs.mendix.com/refguide/mxbuild/) and its dependencies. + 2. Run an `mxbuild` in a container, and copy the resulting MDA contents to the destination path specified by `--destination`. +4. Otherwise (`--source` specifies a path to an MDA file or unpacked MDA directory), `build.sh` will just copy the MDA contents to the destination path specified by `--destination`. + +Once the `build.py` script runs successfully, the path specified by `--destination` will contain a Docker context and everything needed to run a `docker build` command. + +### Updating an existing pipeline to use build.py + +There instructions are provided as a reference, based on a typical pipeline. Your CI/CD pipeline might be different - for support with updating a custom pipeline, please check the [Mendix Support Policy](https://www.mendix.com/evaluation-guide/evaluation-learning/support/). + +1. Verify your pipeline image or runner has Python 3.8 available, and uses a UNIX-like operating system (Linux, macOS or Windows Subsystem for Linux). +2. Locate the `docker build` step in your CI/CD pipeline that builds the app image. This should be the step that builds the Mendix app, and not the rootfs or its dependencies. Any `docker build` commands that build the rootfs should not be changed. +3. Before the `docker build` step, add the following lines (replacing `` with the path to the project source, and `` with an empty/temporary writable path): + ```shell + ./build.py --source --destination build-mda-dir + ``` +4. In the `docker build` step: + * Remove `--build-arg BUILD_PATH` args. + * Remove `-f` and `--file` args specifying a Dockerfile, if they exist. + * Update the [Docker context](https://docs.docker.com/build/concepts/context/) path to the ``. + +After the update, your pipeline might look like this: + +```shell +# Preparation steps +# Downloag Docker Buildpack +DOCKER_BUILDPACK_VERSION=v6.0.0 +curl -LJ -o - https://github.com/mendix/docker-mendix-buildpack/archive/refs/tags/${DOCKER_BUILDPACK_VERSION}.tar.gz | tar --strip-components=1 -xvz +# Checkout the Mendix app source +git clone mendix-app-src +# Build the Mendix app from mendix-app-src to a temporary location +./build.py --source mendix-app-src --destination /tmp/docker-buildpack-context build-mda-dir +# Prepare and push the Docker image, using /tmp/docker-buildpack-context as the Docker context +docker build --tag example.com/mendix-app:latest /tmp/docker-buildpack-context +docker push example.com/mendix-app:latest +# Follow-up steps +``` + +# Other changes + +Docker Buildpack v6 switched from `ubi8` to `ubi9` images ([Red Hat Universal Base Images](https://developers.redhat.com/articles/ubi-faq) whenever possible. +Building Mendix 8 and 9 apps still uses `ubi8`, as those versions depend on an older version of Mono that doesn't work in newer operating systems. From 96982c28f390b808a54dc6a932ed6dfe9cbd4d4e Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Mon, 25 Nov 2024 12:11:43 +0100 Subject: [PATCH 6/9] Added a note about CF Buildpack builds. Any pipeline that isn't updated to use build.py will fail to run, as CF Buildpack only works with specific Ubuntu versions. --- upgrading-from-v5.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upgrading-from-v5.md b/upgrading-from-v5.md index 8ed228f..32fb8df 100644 --- a/upgrading-from-v5.md +++ b/upgrading-from-v5.md @@ -7,6 +7,8 @@ If your CI/CD pipeline uses Docker Buildpack to build \*.mda files (compiled Men If you're upgrading from Docker Buildpack v4 (or an older version), you'll also need to follow the [upgrading from Docker Buildpack v4](upgrading-from-v4.md) instructions. +⚠️ If your current pipeline is failing with an _Only Ubuntu is supported_ error, your pipeline depends on CF Buildpack to build Mendix MPR files, and needs to be updated as described in this document. + ## Using the build.py script Docker Buildpack v6 no longer uses CF Buildpack to compile MPR (or MPK) files - to continue supporting newer versions of Mendix, Java and the base OS. From 9b4c66d2194b84089748d5472766982b49d4df9b Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Mon, 25 Nov 2024 20:16:11 +0100 Subject: [PATCH 7/9] Removed confusing BUILD_PATH arg. Clarified that any Python >=3.8 <4 is supported. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index dadc986..87d5eb8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ This project is a goto reference for the following scenarios : * Earlier Docker versions are no longer compatible because they don't support multistage builds. To use Docker versions below 20.10, download an earlier Mendix Docker Buildpack release, such as [v2.3.2](https://github.com/mendix/docker-mendix-buildpack/releases/tag/v2.3.2) * Alternatively, Podman version 5 or later -* Python 3.8 +* Python 3, version 3.8 or later * No additional dependencies are needed * A UNIX-like operating system, such as Linux or macOS, or Windows Subsystem for Linux * An x86-64 (AMD64) based CPU @@ -126,7 +126,6 @@ Before running the container, it is necessary to build the image with your appli ``` docker build \ - --build-arg BUILD_PATH= \ --tag mendix/mendix-buildpack:v1.2 . ``` From 891c7061a719c8155b5098125a30ff89ccc27600 Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Thu, 28 Nov 2024 14:57:09 +0100 Subject: [PATCH 8/9] Added a note that only WSL Python works in Windows. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 87d5eb8..8cb1c08 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ This project is a goto reference for the following scenarios : * Alternatively, Podman version 5 or later * Python 3, version 3.8 or later * No additional dependencies are needed + * In Windows, only Python running in Windows Subsystem for Linux is supported * A UNIX-like operating system, such as Linux or macOS, or Windows Subsystem for Linux * An x86-64 (AMD64) based CPU * ARM64 CPUs are not fully supported From 3ff535a79a66e2c2e745e20e9f2f25383fa2c21b Mon Sep 17 00:00:00 2001 From: Dmitry Zolotukhin Date: Thu, 28 Nov 2024 15:23:37 +0100 Subject: [PATCH 9/9] Addressed internal review comments: * Clarify that the destination directory will be created/cleaned up. * Clarified that build.py should work with Mendix 8, 9 and 10 MPRs/MPKs. --- README.md | 2 +- upgrading-from-v5.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8cb1c08..e4c4922 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ When building the the `rootfs-builder.dockerfile` file, you can provide the foll If your app is a source MPK file, an MPR project directory or a compressed MDA file, it needs to be converted or compiled into a format supported by CF Buildpack - an extracted MDA file. -This feature is available in Docker Buildpack version v5.1.0 and later, and is intended to allow building Mendix 10 apps in custom CI/CD pipelines. +This feature is available in Docker Buildpack version v6.0.0 and later, and is intended to allow building Mendix 8, 9 and 10 apps in custom CI/CD pipelines. To do this, run: diff --git a/upgrading-from-v5.md b/upgrading-from-v5.md index 32fb8df..0b37f7d 100644 --- a/upgrading-from-v5.md +++ b/upgrading-from-v5.md @@ -14,7 +14,7 @@ If you're upgrading from Docker Buildpack v4 (or an older version), you'll also Docker Buildpack v6 no longer uses CF Buildpack to compile MPR (or MPK) files - to continue supporting newer versions of Mendix, Java and the base OS. Instead, a custom `build.py` script will: -1. Prepare a clean [Docker context](https://docs.docker.com/build/concepts/context/) in the path specified by `--destination`. All files required to build the app image will be copied to this destination. +1. Prepare a clean [Docker context](https://docs.docker.com/build/concepts/context/) in the path specified by `--destination`. All files required to build the app image will be copied to this destination. If the directory doesn't exist, the `build.py` script will create it; if the directory is not empty, `build.py` will delete its contents. 2. Detect the file type of the source path specified by the `--source` arg (an MPK file, an MPR file, an MDA file or an unpacked MDA directory). 3. If necessary (`--source` specifies project that needs to be compiled) 1. Create an image containing [mxbuild](https://docs.mendix.com/refguide/mxbuild/) and its dependencies.