This repository has been archived by the owner on Aug 16, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable building offline version of registry
Enable users to build an offline version of the devfile registry, which
contains .zip files for all projects. The offline registry should be
launched with environment variable CHE_DEVFILE_HTTPS_ENDPOINT set to the
public URL of the devfile registry for it to properly serve project zip
files. If the environment variable is not set at startup, the registry
will try to resolve its cluster IP and port from Kubernetes-provisioned
environment variables.
The regular (non-offline) build of the registry now requires docker
builds to target 'registry', e.g.
docker build [...] --target registry
offline builds can instead target 'offline-registry'
Signed-off-by: Angel Misevski <[email protected]>- Loading branch information
Showing
9 changed files
with
254 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| # | ||
| # Copyright (c) 2018-2019 Red Hat, Inc. | ||
| # This program and the accompanying materials are made | ||
| # available under the terms of the Eclipse Public License 2.0 | ||
| # which is available at https://www.eclipse.org/legal/epl-2.0/ | ||
| # | ||
| # SPDX-License-Identifier: EPL-2.0 | ||
| # | ||
| FROM alpine:3.10 AS builder | ||
| RUN apk add --no-cache py-pip jq bash && pip install yq | ||
|
|
||
| # Registry, organization, and tag to use for base images in dockerfiles. Devfiles | ||
| # will be rewritten during build to use these values for base images. | ||
| ARG PATCHED_IMAGES_REG="quay.io" | ||
| ARG PATCHED_IMAGES_ORG="eclipse" | ||
| ARG PATCHED_IMAGES_TAG="nightly" | ||
|
|
||
| COPY ./build/scripts ./arbitrary-users-patch/base_images /build/ | ||
| COPY /devfiles /build/devfiles | ||
| WORKDIR /build/ | ||
| RUN TAG=${PATCHED_IMAGES_TAG} \ | ||
| ORGANIZATION=${PATCHED_IMAGES_ORG} \ | ||
| REGISTRY=${PATCHED_IMAGES_REG} \ | ||
| ./update_devfile_patched_image_tags.sh | ||
| RUN ./check_mandatory_fields.sh devfiles | ||
| RUN ./index.sh > /build/devfiles/index.json | ||
| RUN chmod -R g+rwX /build/devfiles | ||
|
|
||
| FROM quay.io/openshiftio/rhel-base-httpd:latest AS registry | ||
|
|
||
| RUN sed -i -e "s,Listen 80,Listen 8080," /etc/httpd/conf/httpd.conf | ||
| RUN sed -i -e "s,logs/error_log,/dev/stderr," /etc/httpd/conf/httpd.conf | ||
| RUN sed -i -e "s,logs/access_log,/dev/stdout," /etc/httpd/conf/httpd.conf | ||
| RUN sed -i -e "s,AllowOverride None,AllowOverride All," /etc/httpd/conf/httpd.conf | ||
|
|
||
| EXPOSE 80 | ||
|
|
||
| # the htpasswd file may be periodically replaced during run | ||
| RUN chmod a+rwX /etc/httpd/conf | ||
| RUN chmod a+rwX /run/httpd | ||
|
|
||
| RUN mkdir /var/www/html/devfiles | ||
| COPY .htaccess README.md /var/www/html/ | ||
| COPY --from=builder /build/devfiles /var/www/html/devfiles | ||
|
|
||
| STOPSIGNAL SIGWINCH | ||
|
|
||
| COPY ./build/dockerfiles/rhel.entrypoint.sh ./build/dockerfiles/entrypoint.sh /usr/local/bin/ | ||
| RUN chmod g+rwX /usr/local/bin/entrypoint.sh /usr/local/bin/rhel.entrypoint.sh | ||
| ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] | ||
| CMD ["/usr/local/bin/rhel.entrypoint.sh"] | ||
|
|
||
|
|
||
| # Offline devfile registry build | ||
| FROM builder AS offline-builder | ||
| RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build | ||
|
|
||
| FROM registry AS offline-registry | ||
| COPY --from=offline-builder /build/devfiles /var/www/html/devfiles | ||
| COPY --from=offline-builder /build/resources /var/www/html/resources |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright (c) 2012-2018 Red Hat, Inc. | ||
| # This program and the accompanying materials are made | ||
| # available under the terms of the Eclipse Public License 2.0 | ||
| # which is available at https://www.eclipse.org/legal/epl-2.0/ | ||
| # | ||
| # SPDX-License-Identifier: EPL-2.0 | ||
| # | ||
| # Arguments | ||
| # $1 - devfiles directory | ||
| # $2 - resources directory, where project zips will be stored. | ||
| # | ||
| # Only supports downloading projecst from GitHub. | ||
|
|
||
| set -e | ||
|
|
||
| DEVFILES_DIR="${1}" | ||
| RESOURCES_DIR="${2}" | ||
| TEMP_DIR="${2}/devfiles_temp/" | ||
| TEMP_FILE="${TEMP_DIR}temp.yaml" | ||
|
|
||
| # Builds the URL for downloading a GitHub project as a .zip | ||
| # Args: | ||
| # $1 - main repo URL; if it ends in '.git', this will be trimmed | ||
| # $2 - branch to download; if empty or 'null', 'master' is used | ||
| function build_project_zip_url() { | ||
| location="$1" | ||
| branch="$2" | ||
|
|
||
| # Trim unwanted path portions | ||
| location="${location%/}" | ||
| location="${location%.git}" | ||
|
|
||
| # set branch to "master" if undefined | ||
| if [ -z "$branch" ] || [ "$branch" = "null" ]; then | ||
| branch="master" | ||
| fi | ||
|
|
||
| URL="${location}/archive/${branch}.zip" | ||
| echo "$URL" | ||
| } | ||
|
|
||
| # Download a project's zip to specified directory. If file already exists, nothing | ||
| # is done. | ||
| # Args: | ||
| # $1 - URL to download from | ||
| # $2 - path + name of file to save | ||
| function download_project_zip() { | ||
| URL="$1" | ||
| destination="$2" | ||
| if [ -f "$destination" ]; then | ||
| echo " Project already cached" | ||
| else | ||
| echo " Downloading $URL to $destination" | ||
| wget -O "$destination" -nv "$URL" 2>&1 | sed "s/^/ /g" | ||
| fi | ||
| } | ||
|
|
||
| # Update devfile to refer to a locally stored zip instead of a public git repo | ||
| # Args: | ||
| # $1 - path to devfile to update | ||
| # $2 - name of project to update within devfile | ||
| # $3 - path to downloaded project zip | ||
| function update_devfile() { | ||
| devfile="$1" | ||
| project_name="$2" | ||
| destination="$3" | ||
| echo " Updating devfile $devfile to point at cached project zip $destination" | ||
| # The yq script below will rewrite the project with $project_name to be a zip-type | ||
| # project with provided path. The location field contains a placeholder | ||
| # '{{ DEVFILE_REGISTRY_URL }}' which must be filled at runtime (see | ||
| # build/dockerfiles/entrypoint.sh script) | ||
| # shellcheck disable=SC2016 | ||
| yq -y \ | ||
| '(.projects | map(select(.name != $PROJECT_NAME))) as $projects | | ||
| . + { | ||
| "projects": ( | ||
| $projects + [{ | ||
| "name": $PROJECT_NAME, | ||
| "source": { | ||
| "type": "zip", | ||
| "location": "{{ DEVFILE_REGISTRY_URL }}/\($PROJECT_PATH)" | ||
| } | ||
| }] | ||
| ) | ||
| }' "$devfile" \ | ||
| --arg "PROJECT_NAME" "${project_name}" \ | ||
| --arg "PROJECT_PATH" "${destination}" \ | ||
| > "$TEMP_FILE" | ||
| # As a workaround since jq does not support in-place updates, we need to copy | ||
| # to a temp file and then overwrite the original. | ||
| echo " Copying $TEMP_FILE -> $devfile" | ||
| mv "$TEMP_FILE" "$devfile" | ||
|
|
||
| } | ||
|
|
||
| readarray -d '' devfiles < <(find "$DEVFILES_DIR" -name 'devfile.yaml' -print0) | ||
| mkdir -p "$TEMP_DIR" "$RESOURCES_DIR" | ||
| for devfile in "${devfiles[@]}"; do | ||
| echo "Caching project files for devfile $devfile" | ||
| for project in $(yq -c '.projects[]?' "$devfile"); do | ||
| project_name=$(echo "$project" | jq -r '.name') | ||
| echo " Caching project $project_name" | ||
|
|
||
| type=$(echo "$project" | jq -r '.source.type') | ||
| if [ "$type" != "git" ]; then | ||
| echo " [WARN]: Project type is not 'git'; skipping." | ||
| continue | ||
| fi | ||
|
|
||
| location=$(echo "$project" | jq -r '.source.location') | ||
| branch=$(echo "$project" | jq -r '.source.branch') | ||
| if ! echo "$location" | grep -q "github"; then | ||
| echo " [WARN]: Project is not hosted on GitHub; skipping." | ||
| continue | ||
| fi | ||
|
|
||
| URL=$(build_project_zip_url "$location" "$branch") | ||
|
|
||
| filename=$(basename "$URL") | ||
| target=${URL#*//} | ||
| destination="${RESOURCES_DIR%/}/${target}" | ||
|
|
||
| mkdir -p "${destination%${filename}}" | ||
| download_project_zip "$URL" "$destination" | ||
|
|
||
| update_devfile "$devfile" "$project_name" "$destination" | ||
| done | ||
| done | ||
|
|
||
| rm -rf "$TEMP_DIR" | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters