From f8a47b0a5ed3d34f5fada10b7da8187d373f2f0e Mon Sep 17 00:00:00 2001 From: Angel Misevski Date: Mon, 7 Oct 2019 13:09:54 -0400 Subject: [PATCH] Enable building offline version of registry Enable users to build an offline version of the devfile registry, which contains .zip files for all projects. The offline registry should be launched with environment variable CHE_DEVFILE_HTTPS_ENDPOINT set to the public URL of the devfile registry for it to properly serve project zip files. If the environment variable is not set at startup, the registry will try to resolve its cluster IP and port from Kubernetes-provisioned environment variables. The regular (non-offline) build of the registry now requires docker builds to target 'registry', e.g. docker build [...] --target registry offline builds can instead target 'offline-registry' Signed-off-by: Angel Misevski --- README.md | 12 ++- build.sh | 6 +- build/dockerfiles/Dockerfile | 12 ++- build/dockerfiles/Dockerfile.rhel | 60 ++++++++++++++ build/dockerfiles/entrypoint.sh | 23 ++++++ build/dockerfiles/rhel.Dockerfile | 10 +++ build/scripts/cache_projects.sh | 133 ++++++++++++++++++++++++++++++ cico_functions.sh | 5 +- 8 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 build/dockerfiles/Dockerfile.rhel create mode 100755 build/scripts/cache_projects.sh diff --git a/README.md b/README.md index 6f2a95fbd..c5fa8b6a9 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ This repository holds ready-to-use Devfiles for different languages and technolo Execute ```shell -docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly . +docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly --target registry . # or to use & create a RHEL-based image -docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly -f build/dockerfiles/rhel.Dockerfile . +docker build --no-cache -t quay.io/eclipse/che-devfile-registry:nightly -f build/dockerfiles/rhel.Dockerfile --target registry. ``` Where `--no-cache` is needed to prevent usage of cached layers with devfile registry files. Useful when you change devfile files and rebuild the image. @@ -24,6 +24,14 @@ Though you may also just provide the image to the older versions of Docker (ex. `quay.io/eclipse/che-devfile-registry:nightly` image would be rebuilt after each commit in master. +### Offline registry + +The default docker build has multiple targets: +- `--target registry` is used to build the default devfile registry, where projects in devfiles refer to publically hosted git repos +- `--target offline-registry` is used to build a devfile registry which self-hosts projects as zip files. + +The offline registry build will, during the docker build, pull zips from all projects hosted on github and store them in the `/resources` path. This registry should be deployed with environment variable `CHE_DEVFILE_HTTPS_ENDPOINT` set to the URL of the route/endpoint that exposes the devfile registry, as devfiles need to be rewritten to point to internally hosted zip files. + ## OpenShift You can deploy Che devfile registry on Openshift with command. ``` diff --git a/build.sh b/build.sh index 0cf686eaf..da436653f 100755 --- a/build.sh +++ b/build.sh @@ -17,10 +17,12 @@ VERSION=$(head -n 1 VERSION) case $VERSION in *SNAPSHOT) echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry." - docker build -t "quay.io/eclipse/che-devfile-registry:nightly" -f ./build/dockerfiles/Dockerfile . + docker build -t "quay.io/eclipse/che-devfile-registry:nightly" -f ./build/dockerfiles/Dockerfile --target registry . ;; *) echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}." - docker build -t "quay.io/eclipse/che-devfile-registry:${VERSION}" -f ./build/dockerfiles/Dockerfile . --build-arg "PATCHED_IMAGES_TAG=${VERSION}" + docker build -t "quay.io/eclipse/che-devfile-registry:${VERSION}" -f ./build/dockerfiles/Dockerfile . \ + --build-arg "PATCHED_IMAGES_TAG=${VERSION}" \ + --target registry ;; esac diff --git a/build/dockerfiles/Dockerfile b/build/dockerfiles/Dockerfile index 168ed9031..e4bd1d300 100644 --- a/build/dockerfiles/Dockerfile +++ b/build/dockerfiles/Dockerfile @@ -26,10 +26,20 @@ RUN ./check_mandatory_fields.sh devfiles RUN ./index.sh > /build/devfiles/index.json RUN chmod -R g+rwX /build/devfiles -FROM registry.centos.org/centos/httpd-24-centos7 +FROM registry.centos.org/centos/httpd-24-centos7 AS registry RUN mkdir /var/www/html/devfiles COPY .htaccess README.md /var/www/html/ COPY --from=builder /build/devfiles /var/www/html/devfiles COPY ./build/dockerfiles/entrypoint.sh /usr/bin/ ENTRYPOINT ["/usr/bin/entrypoint.sh"] CMD ["/usr/bin/run-httpd"] + + +# Offline registry: download project zips and place them in /build/resources +FROM builder AS offline-builder +RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build + +# Offline registry: copy updated devfile.yamls and cached projects +FROM registry AS offline-registry +COPY --from=offline-builder /build/devfiles /var/www/html/devfiles +COPY --from=offline-builder /build/resources /var/www/html/resources diff --git a/build/dockerfiles/Dockerfile.rhel b/build/dockerfiles/Dockerfile.rhel new file mode 100644 index 000000000..0650e1a7c --- /dev/null +++ b/build/dockerfiles/Dockerfile.rhel @@ -0,0 +1,60 @@ +# +# Copyright (c) 2018-2019 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +FROM alpine:3.10 AS builder +RUN apk add --no-cache py-pip jq bash && pip install yq + +# Registry, organization, and tag to use for base images in dockerfiles. Devfiles +# will be rewritten during build to use these values for base images. +ARG PATCHED_IMAGES_REG="quay.io" +ARG PATCHED_IMAGES_ORG="eclipse" +ARG PATCHED_IMAGES_TAG="nightly" + +COPY ./build/scripts ./arbitrary-users-patch/base_images /build/ +COPY /devfiles /build/devfiles +WORKDIR /build/ +RUN TAG=${PATCHED_IMAGES_TAG} \ + ORGANIZATION=${PATCHED_IMAGES_ORG} \ + REGISTRY=${PATCHED_IMAGES_REG} \ + ./update_devfile_patched_image_tags.sh +RUN ./check_mandatory_fields.sh devfiles +RUN ./index.sh > /build/devfiles/index.json +RUN chmod -R g+rwX /build/devfiles + +FROM quay.io/openshiftio/rhel-base-httpd:latest AS registry + +RUN sed -i -e "s,Listen 80,Listen 8080," /etc/httpd/conf/httpd.conf +RUN sed -i -e "s,logs/error_log,/dev/stderr," /etc/httpd/conf/httpd.conf +RUN sed -i -e "s,logs/access_log,/dev/stdout," /etc/httpd/conf/httpd.conf +RUN sed -i -e "s,AllowOverride None,AllowOverride All," /etc/httpd/conf/httpd.conf + +EXPOSE 80 + +# the htpasswd file may be periodically replaced during run +RUN chmod a+rwX /etc/httpd/conf +RUN chmod a+rwX /run/httpd + +RUN mkdir /var/www/html/devfiles +COPY .htaccess README.md /var/www/html/ +COPY --from=builder /build/devfiles /var/www/html/devfiles + +STOPSIGNAL SIGWINCH + +COPY ./build/dockerfiles/rhel.entrypoint.sh ./build/dockerfiles/entrypoint.sh /usr/local/bin/ +RUN chmod g+rwX /usr/local/bin/entrypoint.sh /usr/local/bin/rhel.entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["/usr/local/bin/rhel.entrypoint.sh"] + + +# Offline devfile registry build +FROM builder AS offline-builder +RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build + +FROM registry AS offline-registry +COPY --from=offline-builder /build/devfiles /var/www/html/devfiles +COPY --from=offline-builder /build/resources /var/www/html/resources diff --git a/build/dockerfiles/entrypoint.sh b/build/dockerfiles/entrypoint.sh index b9a3e42b5..adacdb123 100755 --- a/build/dockerfiles/entrypoint.sh +++ b/build/dockerfiles/entrypoint.sh @@ -16,6 +16,12 @@ # By default, this script will operate on the `/var/www/html/devfiles` directory. # This can be overridden by the environment variable $DEVFILES_DIR # +# In addition, this script will perform the necessary set up for the offline +# devfile registry, replacing placeholders in all devfiles based off environment +# variable +# CHE_DEVFILE_HTTPS_ENDPOINT +# which should be set to the public endpoint for this registry. +# # Will execute any arguments on completion (`exec $@`) set -e @@ -23,6 +29,7 @@ set -e REGISTRY=${CHE_DEVFILE_IMAGES_REGISTRY_URL} ORGANIZATION=${CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION} TAG=${CHE_DEVFILE_IMAGES_REGISTRY_TAG} +PUBLIC_URL=${CHE_DEVFILE_HTTPS_ENDPOINT} DEFAULT_DEVFILES_DIR="/var/www/html/devfiles" DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}" @@ -58,4 +65,20 @@ for devfile in "${devfiles[@]}"; do fi done +if [ -n "$PUBLIC_URL" ]; then + echo "Updating devfiles to point at internal project zip files" + PUBLIC_URL=${PUBLIC_URL%/} + sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" +else + if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then + echo "WARNING: environment variable 'CHE_DEVFILE_HTTPS_ENDPOINT' not configured" \ + "for an offline build of this registry. This may cause issues with importing" \ + "projects in a workspace." + # Experimental workaround -- detect service IP for che-devfile-registry + # Depends on service used being named 'che-devfile-registry' + URL="http://${CHE_DEVFILE_REGISTRY_SERVICE_HOST}:${CHE_DEVFILE_REGISTRY_SERVICE_PORT}" + sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" + fi +fi + exec "${@}" diff --git a/build/dockerfiles/rhel.Dockerfile b/build/dockerfiles/rhel.Dockerfile index eadcc8b01..0655037ad 100644 --- a/build/dockerfiles/rhel.Dockerfile +++ b/build/dockerfiles/rhel.Dockerfile @@ -96,3 +96,13 @@ ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["/usr/local/bin/rhel.entrypoint.sh"] # append Brew metadata here + + + +# Offline devfile registry build +FROM builder AS offline-builder +RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build + +FROM registry AS offline-registry +COPY --from=offline-builder /build/devfiles /var/www/html/devfiles +COPY --from=offline-builder /build/resources /var/www/html/resources diff --git a/build/scripts/cache_projects.sh b/build/scripts/cache_projects.sh new file mode 100755 index 000000000..d0e7d25ce --- /dev/null +++ b/build/scripts/cache_projects.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright (c) 2012-2018 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Arguments +# $1 - devfiles directory +# $2 - resources directory, where project zips will be stored. +# +# Only supports downloading projecst from GitHub. + +set -e + +DEVFILES_DIR="${1}" +RESOURCES_DIR="${2}" +TEMP_DIR="${2}/devfiles_temp/" +TEMP_FILE="${TEMP_DIR}temp.yaml" + +# Builds the URL for downloading a GitHub project as a .zip +# Args: +# $1 - main repo URL; if it ends in '.git', this will be trimmed +# $2 - branch to download; if empty or 'null', 'master' is used +function build_project_zip_url() { + location="$1" + branch="$2" + + # Trim unwanted path portions + location="${location%/}" + location="${location%.git}" + + # set branch to "master" if undefined + if [ -z "$branch" ] || [ "$branch" = "null" ]; then + branch="master" + fi + + URL="${location}/archive/${branch}.zip" + echo "$URL" +} + +# Download a project's zip to specified directory. If file already exists, nothing +# is done. +# Args: +# $1 - URL to download from +# $2 - path + name of file to save +function download_project_zip() { + URL="$1" + destination="$2" + if [ -f "$destination" ]; then + echo " Project already cached" + else + echo " Downloading $URL to $destination" + wget -O "$destination" -nv "$URL" 2>&1 | sed "s/^/ /g" + fi +} + +# Update devfile to refer to a locally stored zip instead of a public git repo +# Args: +# $1 - path to devfile to update +# $2 - name of project to update within devfile +# $3 - path to downloaded project zip +function update_devfile() { + devfile="$1" + project_name="$2" + destination="$3" + echo " Updating devfile $devfile to point at cached project zip $destination" + # The yq script below will rewrite the project with $project_name to be a zip-type + # project with provided path. The location field contains a placeholder + # '{{ DEVFILE_REGISTRY_URL }}' which must be filled at runtime (see + # build/dockerfiles/entrypoint.sh script) + # shellcheck disable=SC2016 + yq -y \ + '(.projects | map(select(.name != $PROJECT_NAME))) as $projects | + . + { + "projects": ( + $projects + [{ + "name": $PROJECT_NAME, + "source": { + "type": "zip", + "location": "{{ DEVFILE_REGISTRY_URL }}/\($PROJECT_PATH)" + } + }] + ) + }' "$devfile" \ + --arg "PROJECT_NAME" "${project_name}" \ + --arg "PROJECT_PATH" "${destination}" \ + > "$TEMP_FILE" + # As a workaround since jq does not support in-place updates, we need to copy + # to a temp file and then overwrite the original. + echo " Copying $TEMP_FILE -> $devfile" + mv "$TEMP_FILE" "$devfile" + +} + +readarray -d '' devfiles < <(find "$DEVFILES_DIR" -name 'devfile.yaml' -print0) +mkdir -p "$TEMP_DIR" "$RESOURCES_DIR" +for devfile in "${devfiles[@]}"; do + echo "Caching project files for devfile $devfile" + for project in $(yq -c '.projects[]?' "$devfile"); do + project_name=$(echo "$project" | jq -r '.name') + echo " Caching project $project_name" + + type=$(echo "$project" | jq -r '.source.type') + if [ "$type" != "git" ]; then + echo " [WARN]: Project type is not 'git'; skipping." + continue + fi + + location=$(echo "$project" | jq -r '.source.location') + branch=$(echo "$project" | jq -r '.source.branch') + if ! echo "$location" | grep -q "github"; then + echo " [WARN]: Project is not hosted on GitHub; skipping." + continue + fi + + URL=$(build_project_zip_url "$location" "$branch") + + filename=$(basename "$URL") + target=${URL#*//} + destination="${RESOURCES_DIR%/}/${target}" + + mkdir -p "${destination%${filename}}" + download_project_zip "$URL" "$destination" + + update_devfile "$devfile" "$project_name" "$destination" + done +done + +rm -rf "$TEMP_DIR" + diff --git a/cico_functions.sh b/cico_functions.sh index 6c895967e..808182f4e 100644 --- a/cico_functions.sh +++ b/cico_functions.sh @@ -99,7 +99,7 @@ function setup_environment() { # Build, tag, and push devfile registry, tagged with ${TAG} and ${GIT_COMMIT_TAG} function build_and_push() { # Let's build and push image to 'quay.io' using git commit hash as tag first - docker build -t ${IMAGE} -f ${DOCKERFILE_PATH} . + docker build -t ${IMAGE} -f ${DOCKERFILE_PATH} --target registry . tag_push "${REGISTRY}/${ORGANIZATION}/${IMAGE}:${GIT_COMMIT_TAG}" echo "CICO: '${GIT_COMMIT_TAG}' version of images pushed to '${REGISTRY}/${ORGANIZATION}' organization" @@ -116,7 +116,8 @@ function build_and_push() { function build_and_push_release() { echo "CICO: building release '${TAG}' version of devfile registry" docker build -t ${IMAGE} -f ${DOCKERFILE_PATH} . \ - --build-arg PATCHED_IMAGES_TAG=${TAG} + --build-arg PATCHED_IMAGES_TAG=${TAG} \ + --target registry echo "CICO: release '${TAG}' version of devfile registry built" tag_push "${REGISTRY}/${ORGANIZATION}/${IMAGE}:${TAG}" echo "CICO: release '${TAG}' version of devfile registry pushed to '${REGISTRY}/${ORGANIZATION}' organization"