From c036e88bc9c50ca65e1a4ab7da37c012478d1fa1 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Wed, 31 Jul 2024 10:46:48 +0100 Subject: [PATCH 1/6] Changes to containerise hyperion and enable deployment to kubernetes and podman * New build_docker_image.sh script to build an image on the cli * New run_in_podman.sh script to run the image from podman * Github CI workflow to build container image on release and manual execution and push to GHCR registry * Helm charts to deploy container images to kubernetes * Hyperion now has --version option to report the current version * The current version is now set automatically on pip install --- .dockerignore | 10 ++ .github/workflows/publish_docker_image.yml | 47 ++++++++ .gitignore | 3 + .pre-commit-config.yaml | 1 + Dockerfile | 6 +- docs/deploying.md | 45 ++++++++ helmchart/Chart.yaml | 6 + helmchart/templates/deployment.yaml | 105 ++++++++++++++++++ helmchart/templates/service.yaml | 13 +++ helmchart/values.yaml | 10 ++ pyproject.toml | 8 +- run_in_podman.sh | 122 +++++++++++++++++++++ src/hyperion/__main__.py | 4 +- src/hyperion/parameters/cli.py | 8 ++ utility_scripts/build_docker_image.sh | 53 +++++++++ utility_scripts/deploy/deploy_to_k8s.sh | 67 +++++++++++ utility_scripts/docker/entrypoint.sh | 72 ++++++++++++ utility_scripts/docker/healthcheck.sh | 3 + utility_scripts/docker/i03-compose.yml | 60 ++++++++++ 19 files changed, 638 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/publish_docker_image.yml create mode 100644 docs/deploying.md create mode 100644 helmchart/Chart.yaml create mode 100644 helmchart/templates/deployment.yaml create mode 100644 helmchart/templates/service.yaml create mode 100644 helmchart/values.yaml create mode 100755 run_in_podman.sh create mode 100755 utility_scripts/build_docker_image.sh create mode 100755 utility_scripts/deploy/deploy_to_k8s.sh create mode 100755 utility_scripts/docker/entrypoint.sh create mode 100644 utility_scripts/docker/healthcheck.sh create mode 100644 utility_scripts/docker/i03-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6766e07b8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +# List of folders and files to be excluded from the docker image +.github +.pytest_cache +.ruff_cache + +# virtualenv stuff - this gets built by the docker script +.venv +activate + +tmp diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml new file mode 100644 index 000000000..1373de7a7 --- /dev/null +++ b/.github/workflows/publish_docker_image.yml @@ -0,0 +1,47 @@ +name: Publish Docker Image +on: + release: + types: [published] + # Allow the workflow to be triggered manually + workflow_dispatch: +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} +jobs: + build_and_push_image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + # v4.1.7 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + - name: Log in to GHCR + # v3.3.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + # v5.5.1 + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + # v6.5.0 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Generate artifact attestation + # v1.4.0 + uses: actions/attest-build-provenance@210c1913531870065f03ce1f9440dd87bc0938cd + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 19bbd143c..e69d1d026 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,9 @@ MANIFEST *.manifest *.spec +# Package version +_version.py + # Installer logs pip-log.txt pip-delete-this-directory.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f3fde273..d59c174c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,6 +32,7 @@ repos: - id: check-ast - id: check-yaml args: ["--allow-multiple-documents"] + exclude: ^helmchart/ - id: check-merge-conflict - id: check-added-large-files args: ["--maxkb=500"] diff --git a/Dockerfile b/Dockerfile index 9bad33805..61b55ec99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,8 @@ FROM python:3.11 AS build ADD . /project/ WORKDIR "/project" RUN pip install -e .[dev] -RUN python -m build \ No newline at end of file +RUN python -m build + +ENTRYPOINT /project/utility_scripts/docker/entrypoint.sh + +EXPOSE 5005 \ No newline at end of file diff --git a/docs/deploying.md b/docs/deploying.md new file mode 100644 index 000000000..520db6853 --- /dev/null +++ b/docs/deploying.md @@ -0,0 +1,45 @@ +Building the Docker image +==== + +If you run into issues with `podman build .` failing with the error message +`io: read/write on closed pipe` then you may be running out of disk space - try setting TMPDIR environment variable + +https://github.com/containers/podman/issues/22342 + +### Building image on ubuntu + +If you run into issues such as +```commandline +potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument +``` + +* Ensure newuidmap is installed +`sudo apt-get install uidmap` +* Add appropriate entries to `/etc/subuid` and `/etc/subgid` +e.g. +``` +# subuid/subgid file +myuser:10000000:65536 + +# subuid/subgid file +myuser:10000000:65536 +``` +* kill any existing podman processes and retry + +For further information, see https://github.com/containers/podman/issues/2542 + +Pushing the docker image +=== + +If building a release image, the image should be pushed to github by the release CI scripts _TODO_ + +If building a test image, the image should be pushed to your personal GH account: + +`cat | podman login ghcr.io --username --password-stdin` + +`podman push ghcr.io//` + +Deploying to kubernetes +=== + +Once the docker image is built, the image can be deployed to kubernetes using the `deploy_to_k8s.sh` script \ No newline at end of file diff --git a/helmchart/Chart.yaml b/helmchart/Chart.yaml new file mode 100644 index 000000000..cea620e9b --- /dev/null +++ b/helmchart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: hyperion +description: Hyperion server +type: application +# version of the chart +version: 0.0.1 diff --git a/helmchart/templates/deployment.yaml b/helmchart/templates/deployment.yaml new file mode 100644 index 000000000..cd7090198 --- /dev/null +++ b/helmchart/templates/deployment.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hyperion-deployment +spec: + selector: + matchLabels: + app: hyperion + replicas: 1 + template: + metadata: + labels: + app: hyperion + spec: + securityContext: + # gda2 + runAsUser: {{ .Values.hyperion.runAsUser }} + runAsGroup: {{ .Values.hyperion.runAsGroup }} + supplementalGroups: {{ .Values.hyperion.supplementalGroups }} + volumes: + - name: dls-sw-bl + hostPath: + path: "/dls_sw/{{ .Values.hyperion.beamline }}" + type: Directory + - name: dls-sw-apps + hostPath: + path: "/dls_sw/apps" + type: Directory + - name: dls-sw-dasc + hostPath: + path: "/dls_sw/dasc" + type: Directory + {{- if .Values.hyperion.dev }} + # Bind some source folders for easier debugging + - name: src + hostPath: + path: "{{ .Values.hyperion.projectDir }}/src" + type: Directory + - name: tests + hostPath: + path: "{{ .Values.hyperion.projectDir }}/tests" + type: Directory + - name: utility-scripts + hostPath: + path: "{{ .Values.hyperion.projectDir }}/utility_scripts" + type: Directory + - name: devlogs + hostPath: + path: "{{ .Values.hyperion.projectDir }}/tmp" + type: Directory + {{- end }} + containers: + - name: hyperion + image: {{ .Values.hyperion.imageRepository}}/hyperion:latest + resources: + limits: + cpu: "1" + memory: "1Gi" + ports: + - name: hyperion-api + containerPort: 5005 + protocol: TCP + env: + - name: HYPERION_LOG_DIR + value: {{ .Values.hyperion.logDir }} + - name: BEAMLINE + value: "{{ .Values.hyperion.beamline }}" + {{- if not .Values.hyperion.dev }} + - name: ZOCALO_GO_USER + value: "gda2" + - name: ZOCALO_GO_HOSTNAME + value: "{{ .Values.hyperion.beamline }}-control" + - name: ZOCALO_CONFIG + value: "/dls_sw/apps/zocalo/live/configuration.yaml" + - name: ISPYB_CONFIG_PATH + value: "/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-{{ .Values.hyperion.beamline }}.cfg" + args: [ "--external-callbacks" ] + {{- end }} + readinessProbe: + exec: + command: [ "/project/docker/healthcheck.sh" ] + periodSeconds: 5 + volumeMounts: + - mountPath: "/dls_sw/{{ .Values.hyperion.beamline }}" + name: dls-sw-bl + readOnly: true + mountPropagation: HostToContainer + - mountPath: "/dls_sw/apps" + name: dls-sw-apps + readOnly: true + mountPropagation: HostToContainer + - mountPath: "/dls_sw/dasc" + name: dls-sw-dasc + readOnly: true + mountPropagation: HostToContainer + {{- if .Values.hyperion.dev }} + - mountPath: "/project/src" + name: src + - mountPath: "/project/tests" + name: tests + - mountPath: "/project/utility_scripts" + name: utility-scripts + - mountPath: "/project/tmp" + name: devlogs + {{ end }} diff --git a/helmchart/templates/service.yaml b/helmchart/templates/service.yaml new file mode 100644 index 000000000..635c6832e --- /dev/null +++ b/helmchart/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: hyperion-svc +spec: + type: LoadBalancer + ports: + - name: hyperion-api + port: 5005 + protocol: TCP + targetPort: 5005 + selector: + app: hyperion diff --git a/helmchart/values.yaml b/helmchart/values.yaml new file mode 100644 index 000000000..126805503 --- /dev/null +++ b/helmchart/values.yaml @@ -0,0 +1,10 @@ +hyperion: + imageRepository: ghcr.io/DiamondLightSource + runAsUser: 37500 + runAsGroup: 37500 + supplementalGroups: [] + beamline: i03 + dev: false + logDir: "/dls_sw/i03/logs/bluesky" +service: + type: NodePort \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4a23dbeb3..cc0ef3a58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,10 @@ [build-system] -requires = ["setuptools<57", "wheel==0.33.1"] +requires = ["setuptools<57", "wheel==0.33.1", "setuptools_scm[toml]>=6.2"] build-backend = "setuptools.build_meta" +[project] +dynamic = ["version"] + [tool.pytest.ini_options] asyncio_mode = "auto" markers = [ @@ -23,3 +26,6 @@ lint.select = [ "W", # pycodestyle warnings - https://beta.ruff.rs/docs/rules/#warning-w "I001", # isort ] + +[tool.setuptools_scm] +write_to = "src/hyperion/_version.py" \ No newline at end of file diff --git a/run_in_podman.sh b/run_in_podman.sh new file mode 100755 index 000000000..92a94b411 --- /dev/null +++ b/run_in_podman.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +STOP=0 +START=0 +UP=0 +IN_DEV=false + +for option in "$@"; do + case $option in + -b=*|--beamline=*) + BEAMLINE="${option#*=}" + shift + ;; + --stop) + STOP=1 + ;; + --start) + START=1 + ;; + --up) + UP=1 + START=0 + ;; + --logs) + LOGS=1 + ;; + --restart) + STOP=1 + START=1 + ;; + --dev) + IN_DEV=true + ;; + + --help|--info|--h) + echo " --dev Use dev options, such as local graylog instances and S03" + echo " -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline" + echo " " + echo "Operations" + echo " --stop Used to stop a currently running instance of Hyperion. Will override any other operations" + echo " options" + echo " --start Specify that the script should start the server" + echo " --up Create the container for the service but do not start" + echo " --restart Specify that the script should stop and then start the server." + exit 0 + ;; + -*|--*) + echo "Unknown option ${option}. Use --help for info on option usage." + exit 1 + ;; + esac +done + +kill_active_apps () { + echo "Killing active instances of hyperion and hyperion-callbacks..." + podman compose -f ${COMPOSE_YAML} stop ${SERVICE} + echo "done." +} + +check_user () { + if [[ $HOSTNAME != "${BEAMLINE}-control.diamond.ac.uk" || $USER != "gda2" ]]; then + echo "Must be run from beamline control machine as gda2" + echo "Current host is $HOSTNAME and user is $USER" + exit 1 + fi +} + +RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; ) + +COMPOSE_YAML=${RELATIVE_SCRIPT_DIR}/utility_scripts/docker/${BEAMLINE}-compose.yml +EXTRA_ARGS="" + +if [[ -z "${BEAMLINE}" ]]; then + echo "BEAMLINE parameter not set." + echo "If you would like to run on a dev machine set the option -b, --beamline=BEAMLNE to set it manually" + exit 1 +fi + +if [[ $IN_DEV == false ]]; then + SERVICE=hyperion +else + if [[ $UP == 1 ]]; then + echo "--up cannot be used with --dev" + exit 1 + fi + SERVICE=hyperion-dev +fi + +if [[ $LOGS == 1 ]]; then + podman compose -f ${COMPOSE_YAML} logs ${SERVICE} + exit 0 +fi + +if [[ $STOP == 1 ]]; then + if [[ $IN_DEV == false ]]; then + check_user + fi + kill_active_apps + + echo "Hyperion stopped" +fi + +if [[ $UP == 1 ]]; then + podman compose -f ${COMPOSE_YAML} down --remove-orphans + podman compose -f ${COMPOSE_YAML} up --no-start --no-build ${SERVICE} +fi + +if [[ $START == 1 ]]; then + if [ $IN_DEV == false ]; then + check_user + fi + + kill_active_apps + + if [[ $IN_DEV == true ]]; then + echo "Starting with podman compose -f ${COMPOSE_YAML} run -v ${HOME}/.zocalo:/root/.zocalo ${SERVICE}" + podman compose -f ${COMPOSE_YAML} run -v ${HOME}/.zocalo:/root/.zocalo ${SERVICE} + else + echo "Starting with podman compose -f ${COMPOSE_YAML} start ${SERVICE}" + podman compose -f ${COMPOSE_YAML} start ${SERVICE} + fi +fi \ No newline at end of file diff --git a/src/hyperion/__main__.py b/src/hyperion/__main__.py index c4a354f66..88982a773 100755 --- a/src/hyperion/__main__.py +++ b/src/hyperion/__main__.py @@ -275,9 +275,7 @@ def get(self, **kwargs): action = kwargs.get("action") status_and_message = StatusAndMessage(Status.FAILED, f"{action} not understood") if action == Actions.STATUS.value: - LOGGER.debug( - f"Runner recieved status request - state of the runner object is: {self.runner.__dict__} - state of the RE is: {self.runner.RE.__dict__}" - ) + LOGGER.debug("Runner recieved status request") status_and_message = self.runner.current_status return asdict(status_and_message) diff --git a/src/hyperion/parameters/cli.py b/src/hyperion/parameters/cli.py index 6aa3cebb2..275d7dc20 100644 --- a/src/hyperion/parameters/cli.py +++ b/src/hyperion/parameters/cli.py @@ -2,6 +2,8 @@ from pydantic.dataclasses import dataclass +from hyperion._version import version + @dataclass class HyperionArgs: @@ -51,6 +53,12 @@ def parse_cli_args() -> HyperionArgs: action="store_true", help="Run the external hyperion-callbacks service and publish events over ZMQ", ) + parser.add_argument( + "--version", + help="Print hyperion version string", + action="version", + version=version + ) args = parser.parse_args() return HyperionArgs( verbose_event_logging=args.verbose_event_logging or False, diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh new file mode 100755 index 000000000..42c717d2c --- /dev/null +++ b/utility_scripts/build_docker_image.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# builds the docker image +BUILD=1 +PUSH=1 +for option in "$@"; do + case $option in + --no-build) + BUILD=0 + shift + ;; + --no-push) + PUSH=0 + shift + ;; + --help|--info|--h) + CMD=`basename $0` + echo "$CMD [options" + echo "Builds and/or pushes the docker container image to the repository" + echo " --help This help" + echo " --no-build Do not build the image" + echo " --no-push Do not push the image" + exit 0 + ;; + -*|--*) + echo "Unknown option ${option}. Use --help for info on option usage." + exit 1 + ;; + esac +done + +PROJECTDIR=`dirname $0`/.. +VERSION=$1 +if [ -z $VERSION ]; then + VERSION=`hyperion --version | sed -e 's/[^a-zA-Z0-9._-]/_/g'` +fi +PROJECT=hyperion +TAG=$PROJECT:$VERSION +LATEST_TAG=$PROJECT:latest + +if [[ $BUILD == 1 ]]; then + echo "podman build --tag $TAG --tag $LATEST_TAG $PROJECTDIR" + TMPDIR=/tmp podman build --tag $TAG --tag $LATEST_TAG $PROJECTDIR +fi + +if [[ $PUSH == 1 ]]; then + NAMESPACE=`podman login --get-login ghcr.io` + if [[ $? != 0 ]]; then + echo "Not logged in to ghcr.io" + exit 1 + fi + echo "Pushing to ghcr.io/$NAMESPACE/$PROJECT:latest ..." + podman push $PROJECT:latest ghcr.io/$NAMESPACE/$PROJECT:latest +fi \ No newline at end of file diff --git a/utility_scripts/deploy/deploy_to_k8s.sh b/utility_scripts/deploy/deploy_to_k8s.sh new file mode 100755 index 000000000..8f1e06a6a --- /dev/null +++ b/utility_scripts/deploy/deploy_to_k8s.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Installs helm package to kubernetes +for option in "$@"; do + case $option in + -b=*|--beamline=*) + BEAMLINE="${option#*=}" + shift + ;; + --dev) + DEV=true + shift + ;; + --repository=*) + REPOSITORY="${option#*=}" + shift + ;; + --help|--info|--h) + CMD=`basename $0` + echo "$CMD [options] " + echo "Deploys hyperion to kubernetes" + echo " --help This help" + echo " --dev Install to a development kubernetes cluster (assumes project checked out under /home)" + echo " -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline" + echo " --repository=REPOSITORY Override the repository to fetch the image from" + exit 0 + ;; + -*|--*) + echo "Unknown option ${option}. Use --help for info on option usage." + exit 1 + ;; + esac +done + +if [[ -z $BEAMLINE ]]; then + echo "BEAMLINE not set and -b not specified" + exit 1 +fi + +RELEASE=$1 + +if [[ -z $RELEASE ]]; then + echo "Release must be specified" + exit 1 +fi + +HELM_OPTIONS="" +PROJECTDIR=$(readlink -e $(dirname $0)/../..) + +if [[ -n $REPOSITORY ]]; then + HELM_OPTIONS+="--set hyperion.imageRepository=$REPOSITORY " +fi + +if [[ -n $DEV ]]; then + GID=`id -g` + SUPPLEMENTAL_GIDS=37904 + HELM_OPTIONS+="--set \ +hyperion.dev=true,\ +hyperion.runAsUser=$EUID,\ +hyperion.runAsGroup=$GID,\ +hyperion.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ +hyperion.logDir=/project/tmp,\ +hyperion.projectDir=$PROJECTDIR \ +--replace " + mkdir -p $PROJECTDIR/tmp +fi + +helm install $HELM_OPTIONS $RELEASE $PROJECTDIR/helmchart diff --git a/utility_scripts/docker/entrypoint.sh b/utility_scripts/docker/entrypoint.sh new file mode 100755 index 000000000..e8f5dfd8b --- /dev/null +++ b/utility_scripts/docker/entrypoint.sh @@ -0,0 +1,72 @@ +#!/bin/bash -x +# Entry point for the production docker image that launches the external callbacks +# as well as the main server + +for option in "$@"; do + case $option in + --skip-startup-connection) + SKIP_STARTUP_CONNECTION=true + ;; + --dev) + IN_DEV=true + ;; + --verbose-event-logging) + VERBOSE_EVENT_LOGGING=true + ;; + --external-callbacks) + EXTERNAL_CALLBACK_SERVICE=true + ;; + + --help|--info|--h) + echo "Arguments:" + echo " --dev start in development mode without external callbacks" + exit 0 + ;; + -*|--*) + echo "Unknown option ${option}. Use --help for info on option usage." + exit 1 + ;; + esac +done + +RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; ) +cd ${RELATIVE_SCRIPT_DIR} + +echo "$(date) Logging to $HYPERION_LOG_DIR" +mkdir -p $HYPERION_LOG_DIR +start_log_path=$HYPERION_LOG_DIR/start_log.log +callback_start_log_path=$HYPERION_LOG_DIR/callback_start_log.log + +#Add future arguments here +declare -A h_only_args=( ["SKIP_STARTUP_CONNECTION"]="$SKIP_STARTUP_CONNECTION" + ["VERBOSE_EVENT_LOGGING"]="$VERBOSE_EVENT_LOGGING" + ["EXTERNAL_CALLBACK_SERVICE"]="$EXTERNAL_CALLBACK_SERVICE" ) +declare -A h_only_arg_strings=( ["SKIP_STARTUP_CONNECTION"]="--skip-startup-connection" + ["VERBOSE_EVENT_LOGGING"]="--verbose-event-logging" + ["EXTERNAL_CALLBACK_SERVICE"]="--external-callbacks") + +declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" ) +declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" ) + +h_commands=() +for i in "${!h_only_args[@]}" +do + if [ "${h_only_args[$i]}" != false ]; then + h_commands+="${h_only_arg_strings[$i]} "; + fi; +done +cb_commands=() +for i in "${!h_and_cb_args[@]}" +do + if [ "${h_and_cb_args[$i]}" != false ]; then + h_commands+="${h_and_cb_arg_strings[$i]} "; + cb_commands+="${h_and_cb_arg_strings[$i]} "; + fi; +done + +if [ "$EXTERNAL_CALLBACK_SERVICE" = true ]; then + hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & +fi + +echo "$(date) Starting Hyperion..." +hyperion `echo $h_commands;`>$start_log_path 2>&1 diff --git a/utility_scripts/docker/healthcheck.sh b/utility_scripts/docker/healthcheck.sh new file mode 100644 index 000000000..4fe8f294a --- /dev/null +++ b/utility_scripts/docker/healthcheck.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Healthcheck script for the container image +curl -f --head -X GET http://localhost:5005/status || exit 1 diff --git a/utility_scripts/docker/i03-compose.yml b/utility_scripts/docker/i03-compose.yml new file mode 100644 index 000000000..751c41f24 --- /dev/null +++ b/utility_scripts/docker/i03-compose.yml @@ -0,0 +1,60 @@ +name: i03-hyperion +services: + hyperion-common: + image: localhost/hyperion + pull_policy: never + expose: + - "5005" + volumes: + - type: bind + source: /dls_sw/i03 + target: /dls_sw/i03 + read-only: true + - type: bind + source: /dls_sw/apps + target: /dls_sw/apps + read-only: true + - type: bind + source: /dls_sw/dasc + target: /dls_sw/dasc + read-only: true + network_mode: "host" + annotations: + # Required in order to read config files readable by dls_dasc + - run.oci.keep_original_groups=1 + hyperion: + extends: + service: hyperion-common + volumes: + - type: bind + source: /dls/i03 + target: /dls/i03 + - type: bind + source: /dls_sw/i03/logs + target: /dls_sw/i03/logs + environment: + BEAMLINE: i03 + HYPERION_LOG_DIR: /dls_sw/i03/logs/bluesky + ZOCALO_GO_USER: gda2 + ZOCALO_GO_HOSTNAME: i03-control + ZOCALO_CONFIG: /dls_sw/apps/zocalo/live/configuration.yaml + ISPYB_CONFIG_PATH: /dls_sw/dasc/mariadb/credentials/ispyb-hyperion-i03.cfg + command: [ "--external-callbacks" ] + hyperion-dev: + extends: + service: hyperion-common + volumes: + # Bind some source folders for easier debugging + - type: bind + source: ../../src + target: /project/src + - type: bind + source: ../../utility_scripts + target: /project/utility_scripts + - type: bind + source: ../../tests + target: /project/tests + environment: + HYPERION_LOG_DIR: /tmp/dev + entrypoint: [ "/bin/bash" ] + \ No newline at end of file From 67c942167a6e3c5d7fc3397bd29e88b36c9c07f5 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Wed, 7 Aug 2024 14:28:26 +0100 Subject: [PATCH 2/6] Ensure the appVersion is set in the helmchart. Update documentation. Allow production to mount source folders. --- docs/deploying.md | 63 +++++++++++++++++++++---- helmchart/templates/deployment.yaml | 6 +-- helmchart/values.yaml | 3 ++ utility_scripts/build_docker_image.sh | 6 ++- utility_scripts/deploy/deploy_to_k8s.sh | 19 +++++++- 5 files changed, 81 insertions(+), 16 deletions(-) diff --git a/docs/deploying.md b/docs/deploying.md index 520db6853..74d361c51 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,6 +1,29 @@ Building the Docker image ==== +Release builds of container images should be built by the github CI on release, ad-hoc builds can be performed via +manual invocation of the Publish Docker Image workflow. + +Development builds of container images can be made by running the `utility_scripts/build_docker_image.sh` script. +By default it will both build and push the image unless you specify `--no-build` or `--no-push`. To push an image you +will first need to create a GH personal +access token and then log in with podman as described below. + +Pushing the docker image +=== + +If building a test image, the image should be pushed to your personal GH account: + +`cat | podman login ghcr.io --username --password-stdin` + +where `mysecretfile` contains your personal access token + +`podman push ghcr.io//` + +Then run the `build_docker_image.sh` script. + +## Troubleshooting + If you run into issues with `podman build .` failing with the error message `io: read/write on closed pipe` then you may be running out of disk space - try setting TMPDIR environment variable @@ -28,18 +51,40 @@ myuser:10000000:65536 For further information, see https://github.com/containers/podman/issues/2542 -Pushing the docker image + +Deploying to kubernetes === -If building a release image, the image should be pushed to github by the release CI scripts _TODO_ +Once the docker image is built, the image can be deployed to kubernetes using the `deploy_to_k8s.sh` script -If building a test image, the image should be pushed to your personal GH account: +### Production deployment -`cat | podman login ghcr.io --username --password-stdin` - -`podman push ghcr.io//` +* Check out the repo to a new folder +```commandline +cd /dls_sw//software/bluesky +git clone git@github.com:DiamondLightSource/hyperion.git hyperion_vX.Y.Z +``` +* Recreate the `hyperion` symlink to point to the new folder +* ssh to the beamline control machine +* change user to the service account user +* cd to the new folder and run +```commandline +git checkout vX.Y.Z +./utility_scripts/deploy/deploy_to_k8s.sh hyperion +``` -Deploying to kubernetes -=== +This will create a helm release "hyperion". + +### development deployment + +From your development workspace, either with a release image or using a development image built with : + +```commandline +./utility_scripts/deploy/deploy_to_k8s.sh --dev --beamline= --repository= hyperion-test +``` + +Please note, the deployment is intended to be run from a checked-out matching version of the git repository. For +production this is expected to be in the normal place defined in `values.yaml`. The source folders will be mounted as +bind mounts to allow the pod to pick up changes in production. -Once the docker image is built, the image can be deployed to kubernetes using the `deploy_to_k8s.sh` script \ No newline at end of file +`helm list` should then show details of the installed release diff --git a/helmchart/templates/deployment.yaml b/helmchart/templates/deployment.yaml index cd7090198..373a5fc96 100644 --- a/helmchart/templates/deployment.yaml +++ b/helmchart/templates/deployment.yaml @@ -30,7 +30,6 @@ spec: hostPath: path: "/dls_sw/dasc" type: Directory - {{- if .Values.hyperion.dev }} # Bind some source folders for easier debugging - name: src hostPath: @@ -44,6 +43,7 @@ spec: hostPath: path: "{{ .Values.hyperion.projectDir }}/utility_scripts" type: Directory + {{- if .Values.hyperion.dev }} - name: devlogs hostPath: path: "{{ .Values.hyperion.projectDir }}/tmp" @@ -51,7 +51,7 @@ spec: {{- end }} containers: - name: hyperion - image: {{ .Values.hyperion.imageRepository}}/hyperion:latest + image: {{ .Values.hyperion.imageRepository}}/hyperion:{{ .Values.hyperion.appVersion }} resources: limits: cpu: "1" @@ -93,13 +93,13 @@ spec: name: dls-sw-dasc readOnly: true mountPropagation: HostToContainer - {{- if .Values.hyperion.dev }} - mountPath: "/project/src" name: src - mountPath: "/project/tests" name: tests - mountPath: "/project/utility_scripts" name: utility-scripts + {{- if .Values.hyperion.dev }} - mountPath: "/project/tmp" name: devlogs {{ end }} diff --git a/helmchart/values.yaml b/helmchart/values.yaml index 126805503..6228b388f 100644 --- a/helmchart/values.yaml +++ b/helmchart/values.yaml @@ -6,5 +6,8 @@ hyperion: beamline: i03 dev: false logDir: "/dls_sw/i03/logs/bluesky" + projectDir: "/dls_sw/i03/software/bluesky/hyperion" + # This should be overridden at install time + appVersion: SET_ON_INSTALL service: type: NodePort \ No newline at end of file diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh index 42c717d2c..5da290689 100755 --- a/utility_scripts/build_docker_image.sh +++ b/utility_scripts/build_docker_image.sh @@ -14,7 +14,7 @@ for option in "$@"; do ;; --help|--info|--h) CMD=`basename $0` - echo "$CMD [options" + echo "$CMD [options]" echo "Builds and/or pushes the docker container image to the repository" echo " --help This help" echo " --no-build Do not build the image" @@ -31,6 +31,7 @@ done PROJECTDIR=`dirname $0`/.. VERSION=$1 if [ -z $VERSION ]; then + python -m setuptools_scm --force-write-version-files VERSION=`hyperion --version | sed -e 's/[^a-zA-Z0-9._-]/_/g'` fi PROJECT=hyperion @@ -49,5 +50,6 @@ if [[ $PUSH == 1 ]]; then exit 1 fi echo "Pushing to ghcr.io/$NAMESPACE/$PROJECT:latest ..." - podman push $PROJECT:latest ghcr.io/$NAMESPACE/$PROJECT:latest + podman push $PROJECT:latest docker://ghcr.io/$NAMESPACE/$PROJECT:latest + podman push $PROJECT:latest docker://ghcr.io/$NAMESPACE/$PROJECT:$VERSION fi \ No newline at end of file diff --git a/utility_scripts/deploy/deploy_to_k8s.sh b/utility_scripts/deploy/deploy_to_k8s.sh index 8f1e06a6a..ccde28b74 100755 --- a/utility_scripts/deploy/deploy_to_k8s.sh +++ b/utility_scripts/deploy/deploy_to_k8s.sh @@ -62,6 +62,21 @@ hyperion.logDir=/project/tmp,\ hyperion.projectDir=$PROJECTDIR \ --replace " mkdir -p $PROJECTDIR/tmp +elif [[ ! -e $PROJECTDIR/src/hyperion/_version.py ]]; then + # Production install requires the _version.py to be created, this needs a minimal virtual environment + echo "Creating _version.py" + if [[ -d $PROJECTDIR/.venv/bin/activate ]]; then + echo "Virtual environment not found - creating" + module load python/3.11 + python -m venv $PROJECTDIR/.venv + . $PROJECTDIR/.venv/bin/activate + pip install setuptools_scm + else + . $PROJECTDIR/.venv/bin/activate + fi + python -m setuptools_scm --force-write-version-files fi - -helm install $HELM_OPTIONS $RELEASE $PROJECTDIR/helmchart +APP_VERSION=`python -m setuptools_scm | sed -e 's/[^a-zA-Z0-9._-]/_/g'` +HELM_OPTIONS+="--set hyperion.appVersion=$APP_VERSION " +helm package $PROJECTDIR/helmchart --app-version $APP_VERSION +helm install $HELM_OPTIONS $RELEASE hyperion-0.0.1.tgz From 5789e754c932a0fc763d0922e27622276536c411 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 8 Aug 2024 17:23:18 +0100 Subject: [PATCH 3/6] Update deployment instructions, deploy_hyperion.sh for use with k8s --- docs/deploying.md | 11 ++- utility_scripts/deploy/deploy_hyperion.py | 88 +++++++++++++---------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/docs/deploying.md b/docs/deploying.md index 74d361c51..ec88226a8 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -59,14 +59,11 @@ Once the docker image is built, the image can be deployed to kubernetes using th ### Production deployment -* Check out the repo to a new folder +* From a development hyperion workspace ```commandline -cd /dls_sw//software/bluesky -git clone git@github.com:DiamondLightSource/hyperion.git hyperion_vX.Y.Z -``` -* Recreate the `hyperion` symlink to point to the new folder -* ssh to the beamline control machine -* change user to the service account user +python utility_scripts/deploy/deploy_hyperion.py --kubernetes +``` + * cd to the new folder and run ```commandline git checkout vX.Y.Z diff --git a/utility_scripts/deploy/deploy_hyperion.py b/utility_scripts/deploy/deploy_hyperion.py index ee587dbf1..aba19626a 100644 --- a/utility_scripts/deploy/deploy_hyperion.py +++ b/utility_scripts/deploy/deploy_hyperion.py @@ -2,6 +2,7 @@ import os import re import subprocess +from typing import NamedTuple from uuid import uuid1 from create_venv import setup_venv @@ -17,7 +18,7 @@ DEV_DEPLOY_LOCATION = "/scratch/30day_tmp/hyperion_release_test/bluesky" -class repo: +class Deployment: # Set name, setup remote origin, get the latest version""" def __init__(self, name: str, repo_args): self.name = name @@ -34,7 +35,7 @@ def __init__(self, name: str, repo_args): print(f"Found {self.name}_versions:\n{os.linesep.join(self.versions)}") self.latest_version_str = self.versions[0] - def deploy(self, url): + def deploy(self): print(f"Cloning latest version {self.name} into {self.deploy_location}") deploy_repo = Repo.init(self.deploy_location) @@ -63,25 +64,41 @@ def set_deploy_location(self, release_area): ) +class Options(NamedTuple): + release_dir: str + kubernetes: bool = False + + # Get the release directory based off the beamline and the latest hyperion version -def get_hyperion_release_dir_from_args() -> str: - parser = argparse.ArgumentParser() +def _get_hyperion_release_dir_from_args() -> Options: + parser = argparse.ArgumentParser( + prog="deploy_hyperion", description="Deploy hyperion to a beamline" + ) + parser.add_argument( + "--kubernetes", + action=argparse.BooleanOptionalAction, + help="Prepare git workspaces for deployment to kubernetes; do not install virtual environment", + ) parser.add_argument( "beamline", type=str, choices=recognised_beamlines, - help="The beamline to deploy hyperion to", + help=f"The beamline to deploy hyperion to. 'dev' installs to {DEV_DEPLOY_LOCATION}", ) args = parser.parse_args() if args.beamline == "dev": print("Running as dev") - return DEV_DEPLOY_LOCATION + release_dir = DEV_DEPLOY_LOCATION else: - return f"/dls_sw/{args.beamline}/software/bluesky" + release_dir = f"/dls_sw/{args.beamline}/software/bluesky" + return Options(release_dir=release_dir, kubernetes=args.kubernetes) -def create_environment_from_control_machine(): + +def _create_environment_from_control_machine( + hyperion_repo, path_to_create_venv, path_to_dls_dev_env +): try: user = os.environ["USER"] except KeyError: @@ -108,15 +125,13 @@ def create_environment_from_control_machine(): process.kill() -if __name__ == "__main__": - # Gives path to /bluesky - release_area = get_hyperion_release_dir_from_args() - +def main(options: Options): + release_area = options.release_dir this_repo_top = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) print(f"Repo top is {this_repo_top}") - hyperion_repo = repo( + hyperion_repo = Deployment( name="hyperion", repo_args=os.path.join(this_repo_top, ".git"), ) @@ -130,7 +145,7 @@ def create_environment_from_control_machine(): print(f"Putting releases into {release_area_version}") - dodal_repo = repo( + dodal_repo = Deployment( name="dodal", repo_args=os.path.join(this_repo_top, "../dodal/.git"), ) @@ -139,32 +154,27 @@ def create_environment_from_control_machine(): hyperion_repo.set_deploy_location(release_area_version) # Deploy hyperion repo - hyperion_repo.deploy(hyperion_repo.origin.url) - - # Get version of dodal that latest hyperion version uses - with open(f"{release_area_version}/hyperion/setup.cfg", "r") as setup_file: - dodal_url = [ - line - for line in setup_file - if "https://github.com/DiamondLightSource/python-dodal" in line - ] + hyperion_repo.deploy() # Now deploy the correct version of dodal - dodal_repo.deploy(dodal_url) + dodal_repo.deploy() - if hyperion_repo.name == "hyperion": - path_to_dls_dev_env = os.path.join( - hyperion_repo.deploy_location, "utility_scripts/dls_dev_env.sh" - ) - path_to_create_venv = os.path.join( - hyperion_repo.deploy_location, "utility_scripts/deploy/create_venv.py" - ) + if not options.kubernetes: + if hyperion_repo.name == "hyperion": + path_to_dls_dev_env = os.path.join( + hyperion_repo.deploy_location, "utility_scripts/dls_dev_env.sh" + ) + path_to_create_venv = os.path.join( + hyperion_repo.deploy_location, "utility_scripts/deploy/create_venv.py" + ) - # SSH into control machine if not in dev mode - if release_area != DEV_DEPLOY_LOCATION: - create_environment_from_control_machine() - else: - setup_venv(path_to_create_venv, hyperion_repo.deploy_location) + # SSH into control machine if not in dev mode + if release_area != DEV_DEPLOY_LOCATION: + _create_environment_from_control_machine( + hyperion_repo, path_to_create_venv, path_to_dls_dev_env + ) + else: + setup_venv(path_to_create_venv, hyperion_repo.deploy_location) def create_symlink_by_tmp_and_rename(dirname, target, linkname): tmp_name = str(uuid1()) @@ -204,3 +214,9 @@ def create_symlink_by_tmp_and_rename(dirname, target, linkname): print("To start this version run hyperion_restart from the beamline's GDA") else: print("Quitting without latest version being updated") + + +if __name__ == "__main__": + # Gives path to /bluesky + options = _get_hyperion_release_dir_from_args() + main(options) From 225ad0fdb27c1cea6b2f02b99060757c14191234 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 9 Aug 2024 09:26:04 +0100 Subject: [PATCH 4/6] Fix healtcheck Add editable dodal to image and helmcharts Allow the appversion to be specified at deployment Allow existing helmcharts to be upgraded --- Dockerfile | 14 ++++-- docs/deploying.md | 24 +++++----- helmchart/templates/deployment.yaml | 16 ++++--- helmchart/values.yaml | 6 ++- utility_scripts/build_docker_image.sh | 1 + utility_scripts/deploy/deploy_to_k8s.sh | 60 +++++++++++++++++-------- utility_scripts/docker/entrypoint.sh | 11 ++++- utility_scripts/docker/healthcheck.sh | 0 8 files changed, 90 insertions(+), 42 deletions(-) mode change 100644 => 100755 utility_scripts/docker/healthcheck.sh diff --git a/Dockerfile b/Dockerfile index 61b55ec99..3edb1245b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,15 @@ FROM python:3.11 AS build -ADD . /project/ -WORKDIR "/project" +ADD . /app/hyperion +WORKDIR "/app/hyperion" RUN pip install -e .[dev] -RUN python -m build -ENTRYPOINT /project/utility_scripts/docker/entrypoint.sh +# Check out and install dodal locally with no dependencies as this may be a different version to what +# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the +# running container +RUN mkdir ../dodal && \ +git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \ +pip install --no-deps -e ../dodal + +ENTRYPOINT /app/hyperion/utility_scripts/docker/entrypoint.sh EXPOSE 5005 \ No newline at end of file diff --git a/docs/deploying.md b/docs/deploying.md index ec88226a8..c677c30b5 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -62,26 +62,28 @@ Once the docker image is built, the image can be deployed to kubernetes using th * From a development hyperion workspace ```commandline python utility_scripts/deploy/deploy_hyperion.py --kubernetes +cd +./utility_scripts/deploy/deploy_to_k8s.sh --beamline= hyperion ``` -* cd to the new folder and run -```commandline -git checkout vX.Y.Z -./utility_scripts/deploy/deploy_to_k8s.sh hyperion -``` - -This will create a helm release "hyperion". +This will create a helm release "hyperion". The source folders will be mounted as +bind mounts to allow the pod to pick up changes in production. For production these are expected to be in the normal +place defined in `values.yaml`. ### development deployment -From your development workspace, either with a release image or using a development image built with : + +From a development `hyperion` workspace, either with a release image or using a development image built with the script +above, you install a dev deployment to the cluster you are currently logged into with `kubectl`: ```commandline ./utility_scripts/deploy/deploy_to_k8s.sh --dev --beamline= --repository= hyperion-test ``` -Please note, the deployment is intended to be run from a checked-out matching version of the git repository. For -production this is expected to be in the normal place defined in `values.yaml`. The source folders will be mounted as -bind mounts to allow the pod to pick up changes in production. +The dev deployment bind-mounts the current `hyperion` workspace and `../dodal` into the container so that you can +run against your own development code. **Clusters do not allow bind mounts from arbitrary directories so +your workspace will have to be in a permitted directory such as your home directory.** + +Please note, the deployment script is intended to be run from a checked-out matching version of the git repository. `helm list` should then show details of the installed release diff --git a/helmchart/templates/deployment.yaml b/helmchart/templates/deployment.yaml index 373a5fc96..e0786a544 100644 --- a/helmchart/templates/deployment.yaml +++ b/helmchart/templates/deployment.yaml @@ -43,6 +43,10 @@ spec: hostPath: path: "{{ .Values.hyperion.projectDir }}/utility_scripts" type: Directory + - name: dodal + hostPath: + path: "{{ .Values.dodal.projectDir | clean }}" + type: Directory {{- if .Values.hyperion.dev }} - name: devlogs hostPath: @@ -78,7 +82,7 @@ spec: {{- end }} readinessProbe: exec: - command: [ "/project/docker/healthcheck.sh" ] + command: [ "/app/hyperion/utility_scripts/docker/healthcheck.sh" ] periodSeconds: 5 volumeMounts: - mountPath: "/dls_sw/{{ .Values.hyperion.beamline }}" @@ -93,13 +97,15 @@ spec: name: dls-sw-dasc readOnly: true mountPropagation: HostToContainer - - mountPath: "/project/src" + - mountPath: "/app/hyperion/src" name: src - - mountPath: "/project/tests" + - mountPath: "/app/hyperion/tests" name: tests - - mountPath: "/project/utility_scripts" + - mountPath: "/app/hyperion/utility_scripts" name: utility-scripts + - mountPath: "/app/dodal" + name: dodal {{- if .Values.hyperion.dev }} - - mountPath: "/project/tmp" + - mountPath: "/app/hyperion/tmp" name: devlogs {{ end }} diff --git a/helmchart/values.yaml b/helmchart/values.yaml index 6228b388f..11dd4e2cc 100644 --- a/helmchart/values.yaml +++ b/helmchart/values.yaml @@ -6,8 +6,10 @@ hyperion: beamline: i03 dev: false logDir: "/dls_sw/i03/logs/bluesky" - projectDir: "/dls_sw/i03/software/bluesky/hyperion" - # This should be overridden at install time + # These should be overridden at install time + projectDir: SET_ON_INSTALL appVersion: SET_ON_INSTALL +dodal: + projectDir: SET_ON_INSTALL service: type: NodePort \ No newline at end of file diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh index 5da290689..cad7c9d78 100755 --- a/utility_scripts/build_docker_image.sh +++ b/utility_scripts/build_docker_image.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # builds the docker image BUILD=1 PUSH=1 diff --git a/utility_scripts/deploy/deploy_to_k8s.sh b/utility_scripts/deploy/deploy_to_k8s.sh index ccde28b74..b3ddee7e9 100755 --- a/utility_scripts/deploy/deploy_to_k8s.sh +++ b/utility_scripts/deploy/deploy_to_k8s.sh @@ -14,6 +14,10 @@ for option in "$@"; do REPOSITORY="${option#*=}" shift ;; + --appVersion=*) + APP_VERSION="${option#*=}" + shift + ;; --help|--info|--h) CMD=`basename $0` echo "$CMD [options] " @@ -22,6 +26,8 @@ for option in "$@"; do echo " --dev Install to a development kubernetes cluster (assumes project checked out under /home)" echo " -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline" echo " --repository=REPOSITORY Override the repository to fetch the image from" + echo " --appVersion=version Version of the image to fetch from the repository otherwise it is deduced + from the setuptools_scm" exit 0 ;; -*|--*) @@ -46,10 +52,34 @@ fi HELM_OPTIONS="" PROJECTDIR=$(readlink -e $(dirname $0)/../..) +ensure_version_py() { + # We require the _version.py to be created, this needs a minimal virtual environment + if [[ ! -d $PROJECTDIR/.venv ]]; then + echo "Creating _version.py" + echo "Virtual environment not found - creating" + module load python/3.11 + python -m venv $PROJECTDIR/.venv + . $PROJECTDIR/.venv/bin/activate + pip install setuptools_scm + fi +} + +app_version() { + ensure_version_py + + . $PROJECTDIR/.venv/bin/activate + python -m setuptools_scm --force-write-version-files +} + if [[ -n $REPOSITORY ]]; then HELM_OPTIONS+="--set hyperion.imageRepository=$REPOSITORY " fi +ensure_version_py +if [[ -z $APP_VERSION ]]; then + APP_VERSION=$(app_version) +fi + if [[ -n $DEV ]]; then GID=`id -g` SUPPLEMENTAL_GIDS=37904 @@ -58,25 +88,17 @@ hyperion.dev=true,\ hyperion.runAsUser=$EUID,\ hyperion.runAsGroup=$GID,\ hyperion.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ -hyperion.logDir=/project/tmp,\ -hyperion.projectDir=$PROJECTDIR \ ---replace " +hyperion.logDir=/app/hyperion/tmp " mkdir -p $PROJECTDIR/tmp -elif [[ ! -e $PROJECTDIR/src/hyperion/_version.py ]]; then - # Production install requires the _version.py to be created, this needs a minimal virtual environment - echo "Creating _version.py" - if [[ -d $PROJECTDIR/.venv/bin/activate ]]; then - echo "Virtual environment not found - creating" - module load python/3.11 - python -m venv $PROJECTDIR/.venv - . $PROJECTDIR/.venv/bin/activate - pip install setuptools_scm - else - . $PROJECTDIR/.venv/bin/activate - fi - python -m setuptools_scm --force-write-version-files + DEPLOYMENT_DIR=$PROJECTDIR +else + DEPLOYMENT_DIR=/dls_sw/i03/software/bluesky/hyperion_v${APP_VERSION}/hyperion fi -APP_VERSION=`python -m setuptools_scm | sed -e 's/[^a-zA-Z0-9._-]/_/g'` -HELM_OPTIONS+="--set hyperion.appVersion=$APP_VERSION " + +HELM_OPTIONS+="--set hyperion.appVersion=$APP_VERSION,\ +hyperion.projectDir=$DEPLOYMENT_DIR,\ +dodal.projectDir=$DEPLOYMENT_DIR/../dodal " + helm package $PROJECTDIR/helmchart --app-version $APP_VERSION -helm install $HELM_OPTIONS $RELEASE hyperion-0.0.1.tgz +# Helm package generates a file suffixed with the chart version +helm upgrade --install $HELM_OPTIONS $RELEASE hyperion-0.0.1.tgz diff --git a/utility_scripts/docker/entrypoint.sh b/utility_scripts/docker/entrypoint.sh index e8f5dfd8b..b8754fef7 100755 --- a/utility_scripts/docker/entrypoint.sh +++ b/utility_scripts/docker/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Entry point for the production docker image that launches the external callbacks # as well as the main server @@ -29,6 +29,13 @@ for option in "$@"; do esac done +kill_active_apps () { + echo "Killing active instances of hyperion and hyperion-callbacks..." + pkill -e -f "python.*hyperion" + pkill -e -f "SCREEN.*hyperion" + echo "done." +} + RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; ) cd ${RELATIVE_SCRIPT_DIR} @@ -64,6 +71,8 @@ do fi; done +trap kill_active_apps TERM + if [ "$EXTERNAL_CALLBACK_SERVICE" = true ]; then hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & fi diff --git a/utility_scripts/docker/healthcheck.sh b/utility_scripts/docker/healthcheck.sh old mode 100644 new mode 100755 From aaf1cab22d02f117dd5cb62b39c4d55ac0c8e2ab Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 15 Aug 2024 13:17:14 +0100 Subject: [PATCH 5/6] Make the docker image smaller --- .dockerignore | 1 + Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6766e07b8..ea4112d7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .github .pytest_cache .ruff_cache +**/__pycache__/ # virtualenv stuff - this gets built by the docker script .venv diff --git a/Dockerfile b/Dockerfile index 3edb1245b..bcbab060e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ FROM python:3.11 AS build ADD . /app/hyperion WORKDIR "/app/hyperion" -RUN pip install -e .[dev] +RUN pip install --no-cache-dir --no-compile -e . # Check out and install dodal locally with no dependencies as this may be a different version to what # is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the # running container RUN mkdir ../dodal && \ git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \ -pip install --no-deps -e ../dodal +pip install --no-cache-dir --no-compile --no-deps -e ../dodal ENTRYPOINT /app/hyperion/utility_scripts/docker/entrypoint.sh From 207e777d3bc5c90b0dbfb08016462904cee13fe7 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 15 Aug 2024 13:52:40 +0100 Subject: [PATCH 6/6] Fix version name mangling --- utility_scripts/deploy/deploy_to_k8s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utility_scripts/deploy/deploy_to_k8s.sh b/utility_scripts/deploy/deploy_to_k8s.sh index b3ddee7e9..e5805ef36 100755 --- a/utility_scripts/deploy/deploy_to_k8s.sh +++ b/utility_scripts/deploy/deploy_to_k8s.sh @@ -68,7 +68,7 @@ app_version() { ensure_version_py . $PROJECTDIR/.venv/bin/activate - python -m setuptools_scm --force-write-version-files + python -m setuptools_scm --force-write-version-files | sed -e 's/[^a-zA-Z0-9._-]/_/g' } if [[ -n $REPOSITORY ]]; then