Changes to containerise hyperion and enable deployment to kubernetes

.dockerignore

@@ -0,0 +1,11 @@
+# List of folders and files to be excluded from the docker image
+.github
+.pytest_cache
+.ruff_cache
+**/__pycache__/
+
+# virtualenv stuff - this gets built by the docker script
+.venv
+activate
+
+tmp

.github/workflows/publish_docker_image.yml

@@ -0,0 +1,47 @@
+name: Publish Docker Image
+on:
+  release:
+    types: [published]
+  # Allow the workflow to be triggered manually
+  workflow_dispatch:
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+jobs:
+  build_and_push_image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        # v4.1.7
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - name: Log in to GHCR
+        # v3.3.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        # v5.5.1
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build and push Docker image
+        # v6.5.0
+        uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+      - name: Generate artifact attestation
+        # v1.4.0
+        uses: actions/attest-build-provenance@210c1913531870065f03ce1f9440dd87bc0938cd
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.gitignore

@@ -37,6 +37,9 @@ MANIFEST
 *.manifest
 *.spec
 
+# Package version
+_version.py
+
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt

.pre-commit-config.yaml

@@ -32,6 +32,7 @@ repos:
       - id: check-ast
       - id: check-yaml
         args: ["--allow-multiple-documents"]
+        exclude: ^helmchart/
       - id: check-merge-conflict
       - id: check-added-large-files
         args: ["--maxkb=500"]

Dockerfile

@@ -1,5 +1,15 @@
 FROM python:3.11 AS build
-ADD . /project/
-WORKDIR "/project"
-RUN pip install -e .[dev]
-RUN python -m build
+ADD . /app/hyperion
+WORKDIR "/app/hyperion"
+RUN pip install --no-cache-dir --no-compile -e .
+
+# Check out and install dodal locally with no dependencies as this may be a different version to what
+# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the
+# running container
+RUN mkdir ../dodal && \
+git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \
+pip install --no-cache-dir --no-compile --no-deps -e ../dodal
+
+ENTRYPOINT /app/hyperion/utility_scripts/docker/entrypoint.sh
+
+EXPOSE 5005

docs/deploying.md

@@ -0,0 +1,89 @@
+Building the Docker image
+====
+
+Release builds of container images should be built by the github CI on release, ad-hoc builds can be performed via 
+manual invocation of the Publish Docker Image workflow.
+
+Development builds of container images can be made by running the `utility_scripts/build_docker_image.sh` script.
+By default it will both build and push the image unless you specify `--no-build` or `--no-push`. To push an image you 
+will first need to create a GH personal 
+access token and then log in with podman as described below.
+
+Pushing the docker image
+===
+
+If building a test image, the image should be pushed to your personal GH account:
+
+`cat <mysecretfile> | podman login ghcr.io --username <your gh login> --password-stdin`
+
+where `mysecretfile` contains your personal access token
+
+`podman push ghcr.io/<your gh login>/`
+
+Then run the `build_docker_image.sh` script.
+
+## Troubleshooting
+
+If you run into issues with `podman build .` failing with the error message
+`io: read/write on closed pipe` then you may be running out of disk space - try setting TMPDIR environment variable
+
+https://github.com/containers/podman/issues/22342
+
+### Building image on ubuntu
+
+If you run into issues such as 
+```commandline
+potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument
+```
+
+* Ensure newuidmap is installed
+`sudo apt-get install uidmap`
+* Add appropriate entries to `/etc/subuid` and `/etc/subgid`
+e.g.
+```
+# subuid/subgid file
+myuser:10000000:65536
+
+# subuid/subgid file
+myuser:10000000:65536
+```
+* kill any existing podman processes and retry
+
+For further information, see https://github.com/containers/podman/issues/2542
+
+
+Deploying to kubernetes
+===
+
+Once the docker image is built, the image can be deployed to kubernetes using the `deploy_to_k8s.sh` script
+
+### Production deployment
+
+* From a development hyperion workspace
+```commandline
+python utility_scripts/deploy/deploy_hyperion.py --kubernetes <beamline>
+cd <path to deployed hyperion folder in /dls_sw>
+./utility_scripts/deploy/deploy_to_k8s.sh --beamline=<beamline> hyperion
+```
+
+This will create a helm release "hyperion". The source folders will be mounted as 
+bind mounts to allow the pod to pick up changes in production. For production these are expected to be in the normal 
+place defined in `values.yaml`.
+
+### development deployment
+
+
+From a development `hyperion` workspace, either with a release image or using a development image built with the script 
+above, you install a dev deployment to the cluster you are currently logged into with `kubectl`:
+
+```commandline
+./utility_scripts/deploy/deploy_to_k8s.sh --dev --beamline=<beamline> --repository=<your image repo> hyperion-test
+```
+
+The dev deployment bind-mounts the current `hyperion` workspace and `../dodal` into the container so that you can 
+run against your own development code. **Clusters do not allow bind mounts from arbitrary directories so 
+your workspace will have to be in a permitted directory such as your home directory.**
+
+Please note, the deployment script is intended to be run from a checked-out matching version of the git repository.
+
+`helm list` should then show details of the installed release 

helmchart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: hyperion
+description: Hyperion server
+type: application
+# version of the chart
+version: 0.0.1

helmchart/templates/deployment.yaml

@@ -0,0 +1,111 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hyperion-deployment
+spec:
+  selector:
+    matchLabels:
+      app: hyperion
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: hyperion
+    spec:
+      securityContext:
+        # gda2
+        runAsUser: {{  .Values.hyperion.runAsUser }}
+        runAsGroup: {{ .Values.hyperion.runAsGroup }}
+        supplementalGroups: {{ .Values.hyperion.supplementalGroups }}
+      volumes:
+        - name: dls-sw-bl
+          hostPath:
+            path: "/dls_sw/{{ .Values.hyperion.beamline }}"
+            type: Directory
+        - name: dls-sw-apps
+          hostPath:
+            path: "/dls_sw/apps"
+            type: Directory
+        - name: dls-sw-dasc
+          hostPath:
+            path: "/dls_sw/dasc"
+            type: Directory
+        # Bind some source folders for easier debugging        
+        - name: src
+          hostPath:
+            path: "{{ .Values.hyperion.projectDir }}/src"
+            type: Directory
+        - name: tests
+          hostPath:
+            path: "{{ .Values.hyperion.projectDir }}/tests"
+            type: Directory
+        - name: utility-scripts
+          hostPath:
+            path: "{{ .Values.hyperion.projectDir }}/utility_scripts"
+            type: Directory
+        - name: dodal
+          hostPath:
+            path: "{{ .Values.dodal.projectDir | clean }}"
+            type: Directory
+        {{- if .Values.hyperion.dev }}
+        - name: devlogs
+          hostPath:
+            path: "{{ .Values.hyperion.projectDir }}/tmp"
+            type: Directory
+        {{- end }}
+      containers:
+      - name: hyperion
+        image: {{ .Values.hyperion.imageRepository}}/hyperion:{{ .Values.hyperion.appVersion }}
+        resources:
+          limits:
+            cpu: "1"
+            memory: "1Gi"
+        ports:
+          - name: hyperion-api
+            containerPort: 5005
+            protocol: TCP
+        env:
+          - name: HYPERION_LOG_DIR
+            value: {{ .Values.hyperion.logDir }}
+          - name: BEAMLINE
+            value: "{{ .Values.hyperion.beamline }}"
+          {{- if not .Values.hyperion.dev }}
+          - name: ZOCALO_GO_USER
+            value: "gda2"
+          - name: ZOCALO_GO_HOSTNAME
+            value: "{{ .Values.hyperion.beamline }}-control"
+          - name: ZOCALO_CONFIG
+            value: "/dls_sw/apps/zocalo/live/configuration.yaml"
+          - name: ISPYB_CONFIG_PATH
+            value: "/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-{{ .Values.hyperion.beamline }}.cfg"
+        args: [ "--external-callbacks" ]
+          {{- end }}
+        readinessProbe:
+          exec:
+            command: [ "/app/hyperion/utility_scripts/docker/healthcheck.sh" ]
+          periodSeconds: 5
+        volumeMounts:
+          - mountPath: "/dls_sw/{{ .Values.hyperion.beamline }}"
+            name: dls-sw-bl
+            readOnly: true
+            mountPropagation: HostToContainer
+          - mountPath: "/dls_sw/apps"
+            name: dls-sw-apps
+            readOnly: true
+            mountPropagation: HostToContainer
+          - mountPath: "/dls_sw/dasc"
+            name: dls-sw-dasc
+            readOnly: true
+            mountPropagation: HostToContainer
+          - mountPath: "/app/hyperion/src"
+            name: src
+          - mountPath: "/app/hyperion/tests"
+            name: tests
+          - mountPath: "/app/hyperion/utility_scripts"
+            name: utility-scripts
+          - mountPath: "/app/dodal"
+            name: dodal
+          {{- if .Values.hyperion.dev }}
+          - mountPath: "/app/hyperion/tmp"
+            name: devlogs
+          {{ end }}

helmchart/templates/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hyperion-svc
+spec:
+  type: LoadBalancer
+  ports:
+    - name: hyperion-api
+      port: 5005
+      protocol: TCP
+      targetPort: 5005
+  selector:
+    app: hyperion

helmchart/values.yaml

@@ -0,0 +1,15 @@
+hyperion:
+  imageRepository: ghcr.io/DiamondLightSource
+  runAsUser: 37500
+  runAsGroup: 37500
+  supplementalGroups: []
+  beamline: i03
+  dev: false
+  logDir: "/dls_sw/i03/logs/bluesky"
+  # These should be overridden at install time
+  projectDir: SET_ON_INSTALL
+  appVersion: SET_ON_INSTALL
+dodal:
+  projectDir: SET_ON_INSTALL
+service:
+  type: NodePort

pyproject.toml

@@ -1,7 +1,10 @@
 [build-system]
-requires = ["setuptools<57", "wheel==0.33.1"]
+requires = ["setuptools<57", "wheel==0.33.1", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
+[project]
+dynamic = ["version"]
+
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 markers = [
@@ -23,3 +26,6 @@ lint.select = [
     "W",    # pycodestyle warnings - https://beta.ruff.rs/docs/rules/#warning-w
     "I001", # isort
 ]
+
+[tool.setuptools_scm]
+write_to = "src/hyperion/_version.py"