Load examples manifest via HTTP

.github/pull_request_template.md

@@ -15,7 +15,9 @@ To get an auto-generated PR description you can put "copilot:summary" or "copilo
 ### Checklist
 * [ ] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
 * [ ] I've included a screenshot or gif (if applicable)
-* [ ] I have tested [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}) (if applicable)
+* [ ] I have tested the web demo (if applicable):
+  * Full build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html)
+  * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way
 * [ ] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG
 
 - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }})

.github/workflows/on_pull_request.yml

@@ -120,7 +120,6 @@ jobs:
     uses: ./.github/workflows/reusable_upload_web.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
-      UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
 
@@ -151,7 +150,6 @@ jobs:
     uses: ./.github/workflows/reusable_upload_web_demo.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
-      UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
 
@@ -163,7 +161,6 @@ jobs:
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       WHEEL_ARTIFACT_NAME: linux-wheel-fast
-      UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
 
   save-pr-summary:

.github/workflows/reusable_build_web.yml

@@ -55,10 +55,21 @@ jobs:
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
       - name: Build web-viewer (release)
-        uses: actions-rs/cargo@v1
-        with:
-          command: run
-          args: --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }}
+        shell: bash
+        run: |
+          cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }}
+
+      # We build a single manifest pointing to the `commit`
+      # All the `pr`, `nightly`, release tag, etc. variants will always just point to the resolved commit
+      - name: Build examples manifest
+        shell: bash
+        run: |
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
+          sha="$(echo $full_commit | cut -c1-7)"
+
+          cargo run --locked -p re_build_examples_manifest -- \
+            --base-url "https://demo.rerun.io/commit/$sha" \
+            web_viewer/examples_manifest.json
 
       - name: Upload web assets
         uses: actions/upload-artifact@v3

.github/workflows/reusable_publish_web.yml

@@ -104,6 +104,16 @@ jobs:
         run: |
           pixi run cargo run --locked -p re_build_web_viewer -- --release
 
+      - name: Build examples manifest
+        shell: bash
+        run: |
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
+          sha="$(echo $full_commit | cut -c1-7)"
+
+          pixi run build-examples-manifest \
+            --base-url "https://demo.rerun.io/commit/$sha" \
+            web_viewer/examples_manifest.json
+
       - name: Build web demo
         env:
           COMMIT_HASH: ${{ needs.get-commit-sha.outputs.short-sha }}
@@ -125,13 +135,20 @@ jobs:
           destination: "rerun-web-viewer/commit/${{ needs.get-commit-sha.outputs.short-sha }}"
           parent: false
 
-      - name: Upload demo.rerun.io
+      - name: Upload demo.rerun.io (versioned)
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "web_demo"
           destination: "rerun-demo/version/${{ inputs.release-version }}"
           parent: false
 
+      - name: Upload demo.rerun.io (commit)
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: "web_demo"
+          destination: "rerun-demo/commit/${{ needs.get-commit-sha.outputs.short-sha }}"
+          parent: false
+
       - name: Publish app.rerun.io
         if: inputs.update-latest
         shell: bash

.github/workflows/reusable_release_crates.yml

@@ -33,20 +33,6 @@ jobs:
         run: |
           pixi run cargo run --locked -p re_build_web_viewer -- --release
 
-      # Run the build script for `re_viewer`, then commit it this
-      # will ensure that any other dirty files will emit the usual
-      # "dirty git tree" error for all files except the manifest
-      - name: Build examples manifest
-        shell: bash
-        run: |
-          pixi run cargo build --locked -p re_viewer
-
-      - name: Commit examples manifest
-        shell: bash
-        run: |
-          git add ./crates/re_viewer/data/examples_manifest.json
-          git commit -m 'temp'
-
       - name: Publish
         shell: bash
         run: |

.github/workflows/reusable_run_notebook.yml

@@ -10,12 +10,6 @@ on:
         required: false
         type: string
         default: ""
-      # We need this because PRs use a merged commit but we really want
-      # to track uploads based on the source commit.
-      UPLOAD_COMMIT_OVERRIDE:
-        required: false
-        type: string
-        default: ""
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-run-notebook
@@ -77,19 +71,16 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      - name: Add SHORT_SHA env property with commit short sha
+      - name: Get sha
+        id: get-sha
         shell: bash
         run: |
-          if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
-            USED_SHA=${{ github.sha }}
-          else
-            USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
-          fi
-          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload Notebook"
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "/tmp/cube.html"
-          destination: "rerun-builds/commit/${{env.SHORT_SHA}}/notebooks"
+          destination: "rerun-builds/commit/${{ steps.get-sha.outputs.sha }}/notebooks"
           parent: false

.github/workflows/reusable_upload_web.yml

@@ -18,16 +18,6 @@ on:
         required: false
         type: string
         default: "prerelease"
-      # We need this because PRs use a merged commit but we really want
-      # to track uploads based on the source commit.
-      UPLOAD_COMMIT_OVERRIDE:
-        required: false
-        type: string
-        default: ""
-      UPLOAD_COMMIT:
-        required: false
-        type: boolean
-        default: true
       PR_NUMBER:
         required: false
         type: string
@@ -64,22 +54,18 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      - name: Add SHORT_SHA env property with commit short sha
+      - name: Get sha
+        id: get-sha
         shell: bash
         run: |
-          if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
-            USED_SHA=${{ github.sha }}
-          else
-            USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
-          fi
-          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload web-viewer (commit)"
-        if: ${{ inputs.UPLOAD_COMMIT }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "web_viewer"
-          destination: "rerun-web-viewer/commit/${{env.SHORT_SHA}}"
+          destination: "rerun-web-viewer/commit/${{ steps.get-sha.outputs.sha }}"
           parent: false
 
       - name: "Upload web-viewer (tagged)"

.github/workflows/reusable_upload_web_demo.yml

@@ -18,16 +18,6 @@ on:
         required: false
         type: string
         default: "prerelease"
-      # We need this because PRs use a merged commit but we really want
-      # to track uploads based on the source commit.
-      UPLOAD_COMMIT_OVERRIDE:
-        required: false
-        type: string
-        default: ""
-      UPLOAD_COMMIT:
-        required: false
-        type: boolean
-        default: true
       PR_NUMBER:
         type: string
         default: ""
@@ -63,22 +53,18 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      - name: Add SHORT_SHA env property with commit short sha
+      - name: Get sha
+        id: get-sha
         shell: bash
         run: |
-          if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
-            USED_SHA=${{ github.sha }}
-          else
-            USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
-          fi
-          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload web demo (commit)"
-        if: ${{ inputs.UPLOAD_COMMIT }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "web_demo"
-          destination: "rerun-demo/commit/${{env.SHORT_SHA}}"
+          destination: "rerun-demo/commit/${{ steps.get-sha.outputs.sha }}"
           parent: false
 
       - name: "Upload web demo (tagged)"

.prettierignore

@@ -1 +1,8 @@
 rerun_cpp/docs/**/*
+
+rerun_js/web-viewer/re_viewer_bg.js
+rerun_js/web-viewer/re_viewer.js
+web_viewer/re_viewer.js
+crates/re_web_viewer_server/web_viewer/re_viewer.js
+
+.pixi

.prettierrc.toml

@@ -1,5 +1,6 @@
 semi = true
 tabWidth = 4
+trailingComma = "all"
 
 [[overrides]]
 files = ["*.yml", "*.yaml"]

.vscode/settings.json

@@ -83,6 +83,8 @@
     "ruff.lint.args": [
         "--config=rerun_py/pyproject.toml"
     ],
+
+    "prettier.configPath": ".prettierrc.toml",
     "[python]": {
         "editor.defaultFormatter": "charliermarsh.ruff",
     },

Cargo.toml

@@ -28,6 +28,7 @@ version = "0.12.0-alpha.1+dev"
 # re_log_types 0.3.0-alpha.0, NOT 0.3.0-alpha.4 even though it is newer and semver-compatible.
 re_analytics = { path = "crates/re_analytics", version = "=0.12.0-alpha.1", default-features = false }
 re_arrow_store = { path = "crates/re_arrow_store", version = "=0.12.0-alpha.1", default-features = false }
+re_build_examples_manifest = { path = "crates/re_build_examples_manifest", version = "=0.12.0-alpha.1", default-features = false }
 re_build_info = { path = "crates/re_build_info", version = "=0.12.0-alpha.1", default-features = false }
 re_build_tools = { path = "crates/re_build_tools", version = "=0.12.0-alpha.1", default-features = false }
 re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.12.0-alpha.1", default-features = false }

crates/re_build_examples_manifest/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "re_build_examples_manifest"
+authors.workspace = true
+description = "Build the rerun web-viewer Wasm from source"
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[dependencies]
+re_build_tools.workspace = true
+
+# External
+anyhow.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+serde_yaml.workspace = true