From 824d69538ffc33bcadc5f7b4410a5545ad7fa14d Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Thu, 17 Feb 2022 20:03:10 +0100 Subject: [PATCH 01/18] =?UTF-8?q?A=20nix=20derivation=20for=20a=20single?= =?UTF-8?q?=20=E2=80=9Cdeploy=20site=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- default.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/default.nix b/default.nix index af8ce5cfb58..7bd1322875d 100644 --- a/default.nix +++ b/default.nix @@ -617,6 +617,22 @@ rec { ''; }; + netlify-site = nixpkgs.runCommandNoCC "netlify-site" { + buildInputs = [ nixpkgs.tree ]; + } '' + mkdir -p $out + ln -s ${base-doc} $out/base-doc + ln -s ${docs} $out/docs + ln -s ${tests.profiling-graphs} $out/flamegraphs + cd $out; + # generate a simple index.html, listing the entry points + ( echo docs/overview-slides.html; + echo docs/docs/language-guide/motoko.html; + echo base-doc/index.html + echo flamegraphs/index.html ) | \ + tree -H . -l --fromfile -T "Motoko build reports" > index.html + ''; + check-generated = nixpkgs.runCommandNoCC "check-generated" { nativeBuildInputs = [ nixpkgs.diffutils ]; expected = import ./nix/generate.nix { pkgs = nixpkgs; }; From 1252acc62b8d768f2966c00cf703a08827d51681 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:29:34 +0100 Subject: [PATCH 02/18] Push to github pages --- .github/workflows/test.yml | 12 ++++++++++++ default.nix | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ef54758051..77d284f866b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,18 @@ jobs: - name: "nix-build" run: nix-build-uncached --max-jobs 4 -A all-systems-go -build-flags -L + - name: "Pull reports" + run: nix-build -A report-site -o report-site + + - name: Push report to gitub pages + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports' + uses: JamesIves/github-pages-deploy-action@releases/v3 + with: + ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + BRANCH: gh-pages + FOLDER: report-site + SINGLE_COMMIT: true + - name: Calculate performance delta if: runner.os == 'Linux' && github.event_name == 'pull_request' run: | diff --git a/default.nix b/default.nix index 7bd1322875d..e01b3648669 100644 --- a/default.nix +++ b/default.nix @@ -617,7 +617,7 @@ rec { ''; }; - netlify-site = nixpkgs.runCommandNoCC "netlify-site" { + report = nixpkgs.runCommandNoCC "netlify-site" { buildInputs = [ nixpkgs.tree ]; } '' mkdir -p $out From d59aacfdb1d7b44b8af4290b5bb19fc7c69aa275 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:30:39 +0100 Subject: [PATCH 03/18] Fix attribute names --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index e01b3648669..986c0d84f49 100644 --- a/default.nix +++ b/default.nix @@ -617,7 +617,7 @@ rec { ''; }; - report = nixpkgs.runCommandNoCC "netlify-site" { + report-site = nixpkgs.runCommandNoCC "report-site" { buildInputs = [ nixpkgs.tree ]; } '' mkdir -p $out @@ -689,6 +689,7 @@ rec { base-tests base-doc docs + report-site ic-ref-run shell check-formatting From b4a2b51c219e4829dae56126d64e1f8035d2f9e1 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:33:58 +0100 Subject: [PATCH 04/18] Only deploy on linux --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77d284f866b..dea32a62b67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: nix-build -A report-site -o report-site - name: Push report to gitub pages - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports' + if: runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports') uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From d9ea9849480e3ae5c64b9457264341df0fe4a45b Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:40:43 +0100 Subject: [PATCH 05/18] Like this? --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dea32a62b67..794e98cb6d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: nix-build -A report-site -o report-site - name: Push report to gitub pages - if: runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports') + if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports')" uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From a9a0dad816867dcd54f6544abeff2aa9bdc145c8 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:46:25 +0100 Subject: [PATCH 06/18] Like this?? --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 794e98cb6d5..a7cb892c617 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: nix-build -A report-site -o report-site - name: Push report to gitub pages - if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/joachim/deploy-reports')" + if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.head_ref == 'refs/heads/joachim/deploy-reports')" uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From ea81c08dbf70e3721f47d90384328a10c542c6b0 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:54:11 +0100 Subject: [PATCH 07/18] Ok, like this --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7cb892c617..bbdc5c9dedd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: nix-build -A report-site -o report-site - name: Push report to gitub pages - if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.head_ref == 'refs/heads/joachim/deploy-reports')" + #if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.head_ref == 'refs/heads/joachim/deploy-reports')" uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From 44c3ebf3923fd19067798c71d8114a4a8e296269 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 14:55:26 +0100 Subject: [PATCH 08/18] Install nix-build-uncached after setting up cachix --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbdc5c9dedd..6ad9b67ed5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,8 +22,6 @@ jobs: extra_nix_config: | experimental-features = nix-command - - run: nix-env -iA nix-build-uncached -f nix/ - # We are using the ic-hs-test cachix cache that is also used by # dfinity/ic-hs. This is partly laziness (on need to set up a separate # cache), but also to get the ic-ref-test binary without rebuilding @@ -35,6 +33,8 @@ jobs: # until https://github.com/cachix/cachix-action/issues/86 is fixed: - run: cachix watch-store ic-hs-test & + - run: nix-env -iA nix-build-uncached -f nix/ + - name: "nix-build" run: nix-build-uncached --max-jobs 4 -A all-systems-go -build-flags -L From 4240b63fff33e7165c7c36bdf2038a10eb149d96 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:10:00 +0100 Subject: [PATCH 09/18] Only linux --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ad9b67ed5c..915d781270b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: nix-build -A report-site -o report-site - name: Push report to gitub pages - #if: "runner.os == 'Linux' && (github.ref == 'refs/heads/master' || github.head_ref == 'refs/heads/joachim/deploy-reports')" + if: runner.os == 'Linux' # && github.ref == 'refs/heads/master' uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From 08a1d5f554116ed43676f0ae25d72a853c43d314 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:22:56 +0100 Subject: [PATCH 10/18] Separate job --- .github/workflows/test.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 915d781270b..59cb23d920c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,3 +89,25 @@ jobs: ${{ steps.perf.outputs.content }} edit-mode: replace + + reports: + needs: tests + concurrency: ci-${{ github.ref }} + runs-on: ubuntu-latest + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v16 + with: + extra_nix_config: | + experimental-features = nix-command + - uses: cachix/cachix-action@v10 + with: + name: ic-hs-test + - name: Fetch report + run: nix-build -A report-site --jobs 0 -o report-site + - name: Push report to gitub pages + # if: github.ref == 'refs/heads/master' + uses: JamesIves/github-pages-deploy-action@v4.2.5 + with: + branch: gh-pages + folder: report-site + single_commit: true From cd650736acd9619a0ea79a6671a8a853cbc6cc0a Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:24:46 +0100 Subject: [PATCH 11/18] Fix yaml --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59cb23d920c..67da84288e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,9 +91,11 @@ jobs: edit-mode: replace reports: + # if: github.ref == 'refs/heads/master' needs: tests concurrency: ci-${{ github.ref }} runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v16 with: @@ -105,7 +107,6 @@ jobs: - name: Fetch report run: nix-build -A report-site --jobs 0 -o report-site - name: Push report to gitub pages - # if: github.ref == 'refs/heads/master' uses: JamesIves/github-pages-deploy-action@v4.2.5 with: branch: gh-pages From 02e4ae825697e2fc2a3533c991a32e041a05c608 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:26:55 +0100 Subject: [PATCH 12/18] Sigh --- .github/workflows/test.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67da84288e5..87c7960bc46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,15 +41,6 @@ jobs: - name: "Pull reports" run: nix-build -A report-site -o report-site - - name: Push report to gitub pages - if: runner.os == 'Linux' # && github.ref == 'refs/heads/master' - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} - BRANCH: gh-pages - FOLDER: report-site - SINGLE_COMMIT: true - - name: Calculate performance delta if: runner.os == 'Linux' && github.event_name == 'pull_request' run: | From 869b0baff7194d0815c66a2d3127616f6265204a Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:31:08 +0100 Subject: [PATCH 13/18] Not needed --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87c7960bc46..209d8be8ef3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,9 +38,6 @@ jobs: - name: "nix-build" run: nix-build-uncached --max-jobs 4 -A all-systems-go -build-flags -L - - name: "Pull reports" - run: nix-build -A report-site -o report-site - - name: Calculate performance delta if: runner.os == 'Linux' && github.event_name == 'pull_request' run: | From d3b861b95f846e315859254647ab6f63a25d80c1 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:33:28 +0100 Subject: [PATCH 14/18] flag --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 209d8be8ef3..550499969b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,8 +79,9 @@ jobs: edit-mode: replace reports: + # while testing: # if: github.ref == 'refs/heads/master' - needs: tests + #needs: tests concurrency: ci-${{ github.ref }} runs-on: ubuntu-latest steps: @@ -93,7 +94,7 @@ jobs: with: name: ic-hs-test - name: Fetch report - run: nix-build -A report-site --jobs 0 -o report-site + run: nix-build -A report-site --max-jobs 0 -o report-site - name: Push report to gitub pages uses: JamesIves/github-pages-deploy-action@v4.2.5 with: From db01ad6dfb8983b95fad536739bfee7d62b59575 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:35:31 +0100 Subject: [PATCH 15/18] cannot disable local builds due to IFD --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 550499969b0..86e16fb2931 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,7 +94,7 @@ jobs: with: name: ic-hs-test - name: Fetch report - run: nix-build -A report-site --max-jobs 0 -o report-site + run: nix-build -A report-site -o report-site - name: Push report to gitub pages uses: JamesIves/github-pages-deploy-action@v4.2.5 with: From 1db94ebfd71ea47af6ef5aea32880d5485c78f98 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:38:57 +0100 Subject: [PATCH 16/18] Resolve symlinks --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86e16fb2931..3e43ed5fab3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,9 +95,11 @@ jobs: name: ic-hs-test - name: Fetch report run: nix-build -A report-site -o report-site + - name: Resolve symlinks + run: cp -rL report-site report-site-copy - name: Push report to gitub pages uses: JamesIves/github-pages-deploy-action@v4.2.5 with: branch: gh-pages - folder: report-site - single_commit: true + folder: report-site-copy + single-commit: true From a873c5e4b8ba5f42f00a8029db07bda152b37c4b Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:47:45 +0100 Subject: [PATCH 17/18] Update CI.md --- CI.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/CI.md b/CI.md index 0fa1ab84f1b..e553e3d08e6 100644 --- a/CI.md +++ b/CI.md @@ -94,22 +94,28 @@ Mergify reacts to the `automerge-squash` label. Once approved and CI passes, it merges master into the branch (using normal merge, which is important for stashed PRs) and then squash-merges the PR. -Render and show generated base docs ------------------------------------ +Render and provide various reports +---------------------------------- **Use-case:** -To be able to see and share the effects on the generated documentation for the base library, caused by a change to `mo-doc`, without needing the reviewer to generate the documentation locally. +Various build artifacts are most useful when available directly in the browser, namely: -**Implementation (internal):** -Hydra hosts the build product of the `base-doc` CI job. This can be found via the Hydra job status page, and the there is a stable link for the latest build of `master` and of each PR. + * The motoko user guide + * The “overview slides” + * The documentation for `motoko-base`, in the version pinned by motoko + * Flamegraphs for the programs in `tests/perf` -Render and show slides ----------------------- -**Use-case:** -To be able to see and share the “overview slides”, the rendered version should be hosted somewhere. +A stable link to these should exist for `master`, and an easy-to-find link for each PR. **Implementation (internal):** -Hydra hosts the build product of the `overview-slides` CI job. This can be found via the Hydra job status page, and the there is a stable link for the latest build of `master` and of each PR. +Hydra hosts the build products of the relevant jobs, and can be found via the +Hydra job status page, and the there is a stable link for the latest build of +`master` and of each PR. +**Implementation (external):** +The latest `master` version of the file is availble at +. +The reports are calculated in PRs (so failures would be caught), but are not +hosted anywhere. Performance changes are known ----------------------------- From 054a46e9c3ec03433a57c8c50a04a5a99e9d6aff Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Fri, 18 Feb 2022 15:48:03 +0100 Subject: [PATCH 18/18] Only master --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e43ed5fab3..d5eab07fd65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,9 +79,8 @@ jobs: edit-mode: replace reports: - # while testing: - # if: github.ref == 'refs/heads/master' - #needs: tests + if: github.ref == 'refs/heads/master' + needs: tests concurrency: ci-${{ github.ref }} runs-on: ubuntu-latest steps: