Merge branch 'main' into dependabot/npm_and_yarn/packages/experiment-…

…manager/jupyterlab/filebrowser-3.6.7
QCDIS · Aug 28, 2024 · 89778a7 · 89778a7
2 parents db29ceb + 7ee815e
commit 89778a7
Show file tree

Hide file tree

Showing 66 changed files with 8,442 additions and 1,246 deletions.
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -34,12 +34,23 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Free Up GitHub Actions Ubuntu Runner Disk Space
+      uses: jlumbroso/[email protected]
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: false
+        swap-storage: true
+
     - name: Download artifact
       uses: actions/download-artifact@v3
       with:
         name: ${{ inputs.wheel_artifact_name }}
         path: docker/${{ inputs.docker_folder }}
-    - uses: mikefarah/[email protected].1
+    - uses: mikefarah/[email protected].2
 
     - name: add whl and start files
       run: |
@@ -80,7 +91,7 @@ jobs:
         fi
 
     - name: Build docker
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: docker/${{ inputs.docker_folder }}
         push: ${{ inputs.push }}

diff --git a/.github/workflows/collect-tests.yml b/.github/workflows/collect-tests.yml
@@ -0,0 +1,51 @@
+name: Collect tests
+
+on:
+  workflow_call:
+    inputs:
+      runs-on:
+        default: ubuntu-latest
+        type: string
+
+
+jobs:
+  build-wheel:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          miniforge-version: latest
+          miniforge-variant: Mambaforge
+          python-version: 3.11
+          path: ${{ env.CONDA }}/envs
+          environment-file: environment.yml
+          activate-environment: jupyterlab
+          use-mamba: true
+
+      - name: Cache Conda environment
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-environment-${{ runner.os }}-${{ hashFiles('environment.yml') }}
+          restore-keys: |
+            conda-environment-${{ runner.os }}-
+        id: cache-conda
+
+      - name: pytest
+        shell: bash -l {0}
+        run: |
+          conda activate jupyterlab
+          pip install --upgrade build
+          mkdir -p /tmp/data
+          pytest --collect-only -q | grep 'tests/' | jq -R . | jq -s . > test_list.json
+
+      - name: Upload pytest output as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_list
+          path: test_list.json
diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml
@@ -35,7 +35,7 @@ jobs:
     uses: ./.github/workflows/check_secrets_and_variables.yml
     needs: [configure-workflow]
     with:
-      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+      runs-on: ubuntu-latest
       jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
       vlab_slug: ${{ vars.VLAB_SLUG }}
       cell_github: ${{ vars.CELL_GITHUB }}
@@ -50,23 +50,49 @@ jobs:
       cell_github_token:  ${{ secrets.CELL_GITHUB_TOKEN }}
       naavre_api_token: ${{ secrets.NAAVRE_API_TOKEN }}
       search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
+
   lint:
     uses: ./.github/workflows/lint.yml
     needs: [configure-workflow]
     if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
-    strategy:
-      matrix:
-        group: [1]
     with:
-      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+      runs-on: ubuntu-latest
+
+  call-collect-tests:
+    uses: ./.github/workflows/collect-tests.yml
+
+  split-tests:
+    needs: [call-collect-tests]
+    runs-on: ubuntu-latest
+    outputs:
+      test_list: ${{ steps.split-output.outputs.test_list }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Download pytest output artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: test_list
+          path: ./
+
+      - name: Generate matrix from pytest_output
+        id: split-output
+        run: |
+          cat test_list.json
+          MATRIX=$(jq -c . test_list.json)
+          echo $MATRIX
+          echo "test_list=$MATRIX" >> $GITHUB_OUTPUT
+
 
   call-pytest:
     uses: ./.github/workflows/run-tests.yml
-    needs: [configure-workflow]
+    needs: [configure-workflow,split-tests]
     if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
-#     strategy:
-#       matrix:
-#         group: ["1","2"]
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.split-tests.outputs.test_list) }}
     with:
       runs-on: ${{ needs.configure-workflow.outputs.runner }}
       jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
@@ -77,18 +103,18 @@ jobs:
       search_api_endpoint: ${{ vars.SEARCH_API_ENDPOINT }}
       registry_url: ${{ vars.REGISTRY_URL }}
       debug: ${{ vars.DEBUG }}
-#       group: ${{ matrix.group }}
+      test: ${{ matrix.test }}
     secrets:
       cell_github_token:  ${{ secrets.CELL_GITHUB_TOKEN }}
       naavre_api_token: ${{ secrets.NAAVRE_API_TOKEN }}
       search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
 
   make-release:
     uses: ./.github/workflows/make-release.yml
-    needs: [configure-workflow,call-pytest,lint,check_secrets_and_variables]
+    needs: [configure-workflow]
     if: ${{ needs.configure-workflow.outputs.should_skip != 'true' }}
     with:
-      runs-on: ${{ needs.configure-workflow.outputs.runner }}
+      runs-on: ubuntu-latest
       jupyterhub_user: ${{ vars.JUPYTERHUB_USER }}
       vlab_slug: ${{ vars.VLAB_SLUG }}
       cell_github: ${{ vars.CELL_GITHUB }}
@@ -103,7 +129,7 @@ jobs:
       search_api_token: ${{ secrets.SEARCH_API_TOKEN }}
 
   call-build-container-vanilla:
-    needs: [make-release]
+    needs: [make-release,call-pytest]
     uses: ./.github/workflows/build-container.yml
     with:
       wheel_artifact_name: jupyterlab_vre

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -15,9 +15,9 @@ on:
       debug:
         required: true
         type: string
-#       group:
-#         required: true
-#         type: string
+      test:
+        required: true
+        type: string
       jupyterhub_user:
         required: true
         type: string
@@ -75,11 +75,6 @@ jobs:
           activate-environment: jupyterlab
           use-mamba: true
 
-      - name: print group
-        shell: bash -l {0}
-        run: |
-          echo ${{ inputs.group }}
-
       - name: Cache Conda environment
         uses: actions/cache@v4
         with:
@@ -97,10 +92,10 @@ jobs:
       - name: pytest
         shell: bash -l {0}
         run: |
-          conda activate jupyterlab
-          conda install -y conda-forge::pytest-cov
-          conda install -y conda-forge::pytest-split
-          python docker/repo_utils/conf_vl_repos.py  --force=True
-          pip install --upgrade build
-          mkdir -p /tmp/data
-          pytest  -s --ignore=docker --ignore=cypress --no-header -v --store-durations
+           conda activate jupyterlab
+           conda install -y conda-forge::pytest-cov
+           conda install -y conda-forge::pytest-split
+           python docker/repo_utils/conf_vl_repos.py  --force=True
+           pip install --upgrade build
+           mkdir -p /tmp/data
+           pytest ${{ inputs.test }} -s --ignore=docker --ignore=cypress --no-header
diff --git a/Makefile b/Makefile
@@ -25,6 +25,7 @@ build-frontend: jlpm-install
 	npx lerna run build --scope @jupyter_vre/chart-customs
 	npx lerna run build --scope @jupyter_vre/core
 	npx lerna run build --scope @jupyter_vre/notebook-search
+	npx lerna run build --scope @jupyter_vre/type-detector
 	npx lerna run build --scope @jupyter_vre/dataset-search
 	npx lerna run build --scope @jupyter_vre/components
 	npx lerna run build --scope @jupyter_vre/experiment-manager
@@ -56,6 +57,7 @@ jlpm-install:
 
 install-ui:
 	$(call INSTALL_LAB_EXTENSION,notebook-search)
+	$(call INSTALL_LAB_EXTENSION,type-detector)
 	$(call INSTALL_LAB_EXTENSION,dataset-search)
 	$(call INSTALL_LAB_EXTENSION,core)
 	$(call INSTALL_LAB_EXTENSION,chart-customs)
@@ -66,6 +68,7 @@ install-ui:
 
 link-ui:
 	$(call LINK_LAB_EXTENSION,notebook-search)
+	$(call LINK_LAB_EXTENSION,type-detector)
 	$(call LINK_LAB_EXTENSION,dataset-search)
 	$(call LINK_LAB_EXTENSION,core)
 	$(call LINK_LAB_EXTENSION,chart-customs)
@@ -83,6 +86,7 @@ dist-ui: build-frontend
 	$(call PACKAGE_LAB_EXTENSION,vre-panel)
 	$(call PACKAGE_LAB_EXTENSION,vre-menu)
 	$(call PACKAGE_LAB_EXTENSION,notebook-search)
+	$(call PACKAGE_LAB_EXTENSION,type-detector)
 	$(call PACKAGE_LAB_EXTENSION,dataset-search)
 
 release: dist-ui build-backend

diff --git a/docker/vanilla/Dockerfile b/docker/vanilla/Dockerfile
@@ -48,7 +48,8 @@ RUN jupyter serverextension enable --py jupyter_videochat --user
 RUN jupyter serverextension enable --py jupyterlab_github --user
 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
 RUN jupyter serverextension enable --py jupyterlab_s3_browser
+RUN jupyter serverextension enable voila
 
 RUN jupyter lab build --debug;
 
-cmd ["/usr/local/bin/start-jupyter-venv.sh"]
+CMD ["/usr/local/bin/start-jupyter-venv.sh"]
diff --git a/environment.yml b/environment.yml
@@ -58,6 +58,8 @@ dependencies:
   - rasterio=1.3.9
   - awesome-slugify=1.6.5
   - ipywidgets=8.1.2
+  - antlr4-python3-runtime=4.13.1
   - jupyterlab-s3-browser=0.12.0
+  - voila=0.5.7
   - pip:
     - pytype==2023.10.31