Refactor; Package as GRPC Code Server (#147)

#minor

.github/workflows/branch_ci.yml

@@ -0,0 +1,151 @@
+# Workflow that runs on pushes to non-default branches
+
+name: Non-Default Branch Push CI (Python)
+
+on:
+  push:
+    branches-ignore: ['main']
+    paths-ignore: ['README.md']
+
+# Specify concurrency such that only one workflow can run at a time
+# * Different workflow files are not affected
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# Registry for storing Container images
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# Ensure the GitHub token can remove packages
+permissions:
+  packages: write
+
+
+jobs:
+
+  # Job to run a linter and typechecker against the codebase
+  lint-typecheck:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install editable package and required dependencies
+        run: uv sync
+
+      - name: Lint package
+        run: uv run ruff check --output-format=github .
+
+      - name: Typecheck package
+        run: uv run mypy .
+        # TODO: GitHub output when https://github.com/python/mypy/pull/17771 merged
+
+  # Job to run unittests
+  # * Produces a JUnit XML report that can be displayed in the GitHub UI
+  test-unit:
+    runs-on: ubuntu-latest
+    needs: lint-typecheck
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install editable package and required dependencies
+        run: uv sync
+
+      # Run unittests
+      # * Produce JUnit XML report
+      - name: Run unit tests
+        env:
+          CDSAPI_URL: "https://ads.atmosphere.copernicus.eu/api/v2"
+          CDSAPI_KEY: "fake1:fake33-bogus44-falsehood-key55"
+          SS_API_KEY: ${{ secrets.SS_API_KEY }}
+          SS_USER_ID: ${{ secrets.SS_USER_ID }}
+        run: uv run python -m pytest --junitxml=ut-report.xml tests
+
+      # Create test summary to be visualised on the job summary screen on GitHub
+      # * Runs even if previous steps fail
+      - name: Create test summary
+        uses: test-summary/action@v2
+        with:
+          paths: "*t-report.xml"
+          show: "fail, skip"
+        if: always()
+
+  # Job for building container image
+  # * Builds and pushes an OCI Container image to the registry defined in the environment variables
+  # * Only runs if test and lint jobs pass
+  build-container:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    needs: ["lint-typecheck", "test-unit"]
+
+    steps:
+      # Do a non-shallow clone of the repo to ensure tags are present
+      # * This allows setuptools-git-versioning to automatically set the version
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Tag the built image according to the event type
+      # The event is a branch commit, so use the commit sha
+      - name: Extract metadata (tags, labels) for Container
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=ref,event=branch
+
+      # Build and push the Container image to the registry
+      # * Creates a multiplatform-aware image
+      # * Pulls build cache from the registry
+      - name: Build and push container image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Containerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+

.github/workflows/merged_ci.yml

@@ -0,0 +1,102 @@
+# Workflow that runs on closed PRs to the default branch
+
+name: Default Branch PR Merged CI (Python)
+
+on:
+  pull_request:
+    types: ["closed"]
+    branches: ["main"]
+
+# Specify concurrency such that only one workflow can run at a time
+# * Different workflow files are not affected
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+
+  # Define an autotagger job that creates tags on changes to master
+  # Use #major #minor in merge commit messages to bump version beyond patch
+  bump-tag:
+    runs-on: ubuntu-latest
+    if: |
+      github.event_name == 'pull_request' && 
+      github.event.action == 'closed' && 
+      github.event.pull_request.merged == true
+    permissions:
+      contents: write
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Bump version and push tag
+        uses: anothrNick/[email protected]
+        id: tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_BRANCHES: main
+          WITH_V: true
+          DEFAULT_BUMP: patch
+          GIT_API_TAGGING: false
+
+  # Job for building container image
+  # * Builds and pushes an OCI Container image to the registry defined in the environment variables
+  build-container:
+    runs-on: ubuntu-latest
+    needs: bump-tag
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      # Do a non-shallow clone of the repo to ensure tags are present
+      # * This allows setuptools-git-versioning to automatically set the version
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Tag the built image according to the event type
+      # The event is a semver release, so use the version
+      - name: Extract metadata (tags, labels) for Container
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=semver,pattern={{version}},value=${{ needs.bump-tag.outputs.tag }}
+
+      # Build and push the Container image to the registry
+      # * Creates a multiplatform-aware image
+      # * Pulls build cache from the registry and pushes new cache back
+      - name: Build and push container image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Containerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+

Containerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim-bookworm
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+ENV DAGSTER_HOME=/opt/dagster/home
+
+# Add repository code
+WORKDIR /opt/dagster/app
+COPY src /opt/dagster/app
+COPY pyproject.toml /opt/dagster/app
+
+# Checkout and install dagster libraries needed to run the gRPC server by exposing
+# your code location to dagster-webserver and dagster-daemon, and loading the
+# DagsterInstance.
+RUN uv sync
+
+EXPOSE 4266
+
+# Using CMD rather than RUN allows the command to be overridden in
+# run launchers or executors to run other commands using this image.
+# This is important as runs are executed inside this container.
+ENTRYPOINT ["uv", "run"]
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4266", "-m", "dagster_dags"]
+