Build a docker package for development and using in Python

.dockerignore

@@ -0,0 +1,21 @@
+target
+output_data
+docs
+.git
+.venv
+*.parquet
+**/*.rs.bk
+Cargo.lock
+tests/Papers
+node_modules/
+*.bsp
+*.script.bak
+**/*.csv
+*.old
+*.svg
+*.data
+*.folded
+*.venv
+*.parquet
+*.profraw
+lcov.txt

.firebaserc

@@ -0,0 +1,5 @@
+{
+  "projects": {
+    "default": "nyx-space"
+  }
+}

.github/workflows/firebase-hosting-merge.yml

@@ -0,0 +1,24 @@
+# This file was auto-generated by the Firebase CLI
+# https://github.com/firebase/firebase-tools
+
+name: Deploy to Firebase Hosting on merge
+'on':
+  push:
+    branches:
+      - master
+jobs:
+  build_and_deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - run: cargo doc --no-deps
+      - uses: FirebaseExtended/action-hosting-deploy@v0
+        with:
+          repoToken: '${{ secrets.GITHUB_TOKEN }}'
+          firebaseServiceAccount: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_NYX_SPACE }}'
+          channelId: live
+          projectId: nyx-space

.github/workflows/firebase-hosting-pull-request.yml

@@ -0,0 +1,21 @@
+# This file was auto-generated by the Firebase CLI
+# https://github.com/firebase/firebase-tools
+
+name: Deploy to Firebase Hosting on PR
+'on': pull_request
+jobs:
+  build_and_preview:
+    if: '${{ github.event.pull_request.head.repo.full_name == github.repository }}'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - run: cargo doc --no-deps
+      - uses: FirebaseExtended/action-hosting-deploy@v0
+        with:
+          repoToken: '${{ secrets.GITHUB_TOKEN }}'
+          firebaseServiceAccount: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_NYX_SPACE }}'
+          projectId: nyx-space

.github/workflows/python.yml

@@ -28,18 +28,21 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
           args: --release --out dist --find-interpreter -F python
           sccache: 'true'
           manylinux: auto
+
       - name: Upload wheels
         uses: actions/upload-artifact@v3
         with:
           name: wheels
           path: dist
+
       - name: pytest x86_64
         if: ${{ matrix.target == 'x86_64' }}
         shell: bash
@@ -48,6 +51,7 @@ jobs:
           pip install nyx_space --find-links dist --force-reinstall
           pip install pytest numpy pandas plotly pyarrow scipy
           pytest
+
       - name: pytest
         if: ${{ !startsWith(matrix.target, 'x86') && matrix.target != 'ppc64' && matrix.target != 'aarch64' }}
         uses: uraimo/[email protected]
@@ -63,6 +67,7 @@ jobs:
             set -e
             pip3 install nyx_space --find-links dist --force-reinstall
             pytest
+
       - name: Upload python tests HTMLs
         uses: actions/upload-artifact@v3
         with:
@@ -148,7 +153,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    if: "startsWith(github.ref, 'refs/tags/')"
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
     needs: [linux, windows, macos, sdist]
     steps:
       - uses: actions/download-artifact@v3
@@ -161,3 +166,71 @@ jobs:
         with:
           command: upload
           args: --skip-existing *
+
+  packaging:
+    permissions: write-all
+    runs-on: ubuntu-latest
+    # needs: [linux] # Don't package if the tests fail.
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          buildkitd-flags: --debug
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Build development image
+        run: docker build -f Dockerfile.dev -t nyx-build .
+
+      - name: Run development container and build the package
+        run: docker run --name nyx-builder nyx-build maturin build -F python --release
+
+      - name: Copy built package from container to host
+        run: docker cp nyx-builder:/app/target/wheels ./dist
+
+      - name: Get short SHA
+        id: short-sha
+        run: echo "::set-output name=sha::$(echo ${GITHUB_SHA::8})"
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image with built package
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: false
+          tags: |
+            ghcr.io/nyx-space/nyx-fds:${{ steps.short-sha.outputs.sha }}
+            ghcr.io/nyx-space/nyx-fds:${{ github.ref == 'refs/heads/master' && 'latest' || steps.short-sha.outputs.sha }}
+            ghcr.io/nyx-space/nyx-fds:${{ startsWith(github.ref, 'refs/tags/') && steps.get_version.outputs.VERSION || steps.short-sha.outputs.sha }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          outputs: type=oci,dest=/tmp/image.tar
+      -
+        name: Import image in containerd
+        run: |
+          sudo ctr i import --base-name ghcr.io/nyx-space/nyx-fds:${{ steps.short-sha.outputs.sha }} --digests --all-platforms /tmp/image.tar
+      -
+        name: Push image with containerd
+        run: |
+          sudo ctr --debug i push --user "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" ghcr.io/nyx-space/nyx-fds:${{ steps.short-sha.outputs.sha }}
+

.gitignore

@@ -14,4 +14,5 @@ node_modules/
 dist/
 *.parquet
 *.profraw
-lcov.txt
+lcov.txt
+

Dockerfile

@@ -0,0 +1,43 @@
+# Start with OpenSUSE Tumbleweed
+FROM opensuse/tumbleweed
+
+# Install Python 3.10 and other necessary packages
+RUN zypper ref && \
+    zypper -n in python310 python310-pip zsh uuid-runtime shadow git git-lfs && \
+    zypper clean
+
+WORKDIR /app
+
+# Create a non-root user
+RUN useradd -ms /bin/bash nyx-user
+
+# Switch to the new user
+USER nyx-user
+
+# Install oh-my-zsh
+RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
+
+# Set the zsh theme
+RUN sed -i 's/ZSH_THEME="robbyrussell"/ZSH_THEME="fino-time"/' ~/.zshrc
+
+# Change default shell to zsh
+SHELL ["/bin/zsh", "-c"]
+
+RUN mkdir -p $HOME/workspace
+
+WORKDIR /home/nyx-user/workspace
+
+# Install workflow and pipeline features
+RUN pip install kedro kedro-viz ruff jupyter
+
+# Make this available
+ENV PATH="${PATH}:/home/nyx-user/.local/bin"
+
+# Copy over the wheel files
+COPY dist/*.whl /app
+
+# Install the Python wheel
+RUN pip install /app/*.whl
+
+
+CMD ["zsh"]

Dockerfile.dev

@@ -0,0 +1,26 @@
+# This is a container to build the Nyx Python package on OpenSUSE
+# Start with OpenSUSE Tumbleweed
+FROM opensuse/tumbleweed
+
+# Install Python 3.10 and other necessary packages
+RUN zypper ref && \
+    zypper -n in gcc make python310 python310-pip uuid-runtime perl && \
+    zypper clean
+
+WORKDIR /app
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup.sh && sh rustup.sh -y
+
+# Make Rust available
+ENV PATH="${PATH}:/root/.cargo/bin"
+ENV RUST_BACKTRACE=full
+
+# Install maturin to build the package
+RUN pip install maturin
+
+# Finally copy
+COPY . /app
+
+# And build in release only
+CMD ["maturin", "build", "-F", "python", "--release"]