merge master

iterative · May 17, 2022 · d214a27 · d214a27 · github-actions · May 17, 2022
2 parents 104d289 + 5409c8f
commit d214a27
Show file tree

Hide file tree

Showing 25 changed files with 499 additions and 361 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -11,7 +11,7 @@ module.exports = {
     SharedArrayBuffer: 'readonly'
   },
   parserOptions: {
-    ecmaVersion: 2018
+    ecmaVersion: 2020
   },
   ignorePatterns: ['assets/', 'dist/', 'node_modules/'],
   rules: {

diff --git a/.github/workflows/checkbot.yml b/.github/workflows/checkbot.yml
@@ -87,8 +87,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           echo 'Check container!' > report.md
-          vl2svg assets/vega-lite.json | cml-publish --md >> report.md
-          vl2png assets/vega-lite.json | cml-publish --md >> report.md
+          vl2svg assets/vega-lite.json > plot.svg
+          cml-publish --md plot.svg >> report.md
+          vl2png assets/vega-lite.json > plot.png
+          cml-publish --md plot.png >> report.md
           cml-send-comment report.md
           cml-send-github-check report.md
 

diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml
@@ -92,36 +92,42 @@ jobs:
           --dry-run' }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: build
+      - name: install ldid
         run: |
-          cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc
-          npx --yes pkg package.json
-          rm assets/magic.mgc
-          for cmd in '' runner publish pr; do
-            build/cml-linux $cmd --version
-          done
-  release:
-    if: github.event_name == 'push'
-    needs: packages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+          sudo apt install --yes libplist-dev
+          git clone --branch v2.1.5 git://git.saurik.com/ldid.git 
+          sudo g++ -pipe -o /usr/bin/ldid ldid/ldid.cpp -I. -x c ldid/{lookup2.c,sha1.h} -lplist -lcrypto
       - id: build
         name: build
         run: |
           echo ::set-output name=tag::${GITHUB_REF#refs/tags/}
-          npm install
           cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc
-          npx --yes pkg package.json
+          npx --yes pkg --no-bytecode --public-packages "*" --public package.json
           rm assets/magic.mgc
+          for cmd in '' runner publish pr; do
+            build/cml-linux-x64 $cmd --version
+          done
+          cp build/cml-linux{-x64,}
+          cp build/cml-macos{-x64,}
       - uses: softprops/action-gh-release@v1
+        if: github.event_name == 'push'
         with:
           name: CML ${{ steps.build.outputs.tag }}
           draft: true
           generate_release_notes: true
           files: |
+            build/cml-alpine-arm64
+            build/cml-alpine-x64
+            build/cml-linux-arm64
+            build/cml-linuxstatic-arm64
+            build/cml-linuxstatic-x64
+            build/cml-linux-x64
             build/cml-linux
+            build/cml-macos-arm64
+            build/cml-macos-x64
             build/cml-macos
+            build/cml-win-arm64.exe
+            build/cml-win-x64.exe
         env:
           GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
   images:

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,6 @@
 ARG BASE_IMAGE=ubuntu:20.04
 FROM ${BASE_IMAGE}
+ARG BASE_IMAGE
 
 LABEL maintainer="CML <[email protected]>"
 
@@ -10,6 +11,20 @@ RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes
 # CONFIGURE SHELL
 SHELL ["/bin/bash", "-c"]
 
+# FIX NVIDIA APT GPG KEYS (https://github.com/NVIDIA/cuda-repo-management/issues/1#issuecomment-1111490201) 🤬
+RUN grep nvidia <<< ${BASE_IMAGE} \
+ && for list in cuda nvidia-ml; do mv /etc/apt/sources.list.d/$list.list{,.backup}; done \
+ && apt-get update \
+ && apt-get install --yes gpg \
+ && apt-key del 7fa2af80 \
+ && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub \
+ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/7fa2af80.pub \
+ && apt-get purge --yes gpg \
+ && apt-get clean \
+ && rm --recursive --force /var/lib/apt/lists/* \
+ && for list in cuda nvidia-ml; do mv /etc/apt/sources.list.d/$list.list{.backup,}; done \
+ || true
+
 # INSTALL CORE DEPENDENCIES
 RUN apt-get update \
  && apt-get install --no-install-recommends \

diff --git a/README.md b/README.md
@@ -104,7 +104,7 @@ jobs:
         run: |
           # Post reports as comments in GitHub PRs
           cat results.txt >> report.md
-          cml-send-comment report.md
+          cml send-comment report.md
 ```
 
 ## Usage
@@ -127,16 +127,16 @@ those reports to your CI system.
 
 | Function                | Description                                                      | Example Inputs                                              |
 | ----------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------- |
-| `cml-runner`            | Launch a runner locally or hosted by a cloud provider            | See [Arguments](https://github.com/iterative/cml#arguments) |
-| `cml-publish`           | Publicly host an image for displaying in a CML report            | `<path to image> --title <image title> --md`                |
-| `cml-send-comment`      | Return CML report as a comment in your GitLab/GitHub workflow    | `<path to report> --head-sha <sha>`                         |
-| `cml-send-github-check` | Return CML report as a check in GitHub                           | `<path to report> --head-sha <sha>`                         |
-| `cml-pr`                | Commit the given files to a new branch and create a pull request | `<path>...`                                                 |
-| `cml-tensorboard-dev`   | Return a link to a Tensorboard.dev page                          | `--logdir <path to logs> --title <experiment title> --md`   |
+| `cml runner`            | Launch a runner locally or hosted by a cloud provider            | See [Arguments](https://github.com/iterative/cml#arguments) |
+| `cml publish`           | Publicly host an image for displaying in a CML report            | `<path to image> --title <image title> --md`                |
+| `cml send-comment`      | Return CML report as a comment in your GitLab/GitHub workflow    | `<path to report> --head-sha <sha>`                         |
+| `cml send-github-check` | Return CML report as a check in GitHub                           | `<path to report> --head-sha <sha>`                         |
+| `cml pr`                | Commit the given files to a new branch and create a pull request | `<path>...`                                                 |
+| `cml tensorboard-dev`   | Return a link to a Tensorboard.dev page                          | `--logdir <path to logs> --title <experiment title> --md`   |
 
 #### CML Reports
 
-The `cml-send-comment` command can be used to post reports. CML reports are
+The `cml send-comment` command can be used to post reports. CML reports are
 written in markdown ([GitHub](https://github.github.com/gfm),
 [GitLab](https://docs.gitlab.com/ee/user/markdown.html), or
 [Bitbucket](https://confluence.atlassian.com/bitbucketserver/markdown-syntax-guide-776639995.html)
@@ -154,11 +154,11 @@ cat results.txt >> report.md
 
 :framed_picture: **Images** Display images using the markdown or HTML. Note that
 if an image is an output of your ML workflow (i.e., it is produced by your
-workflow), you will need to use the `cml-publish` function to include it a CML
+workflow), you will need to use the `cml publish` function to include it a CML
 report. For example, if `graph.png` is output by `python train.py`, run:
 
 ```bash
-cml-publish graph.png --md >> report.md
+cml publish graph.png --md >> report.md
 ```
 
 ### Getting Started
@@ -201,8 +201,8 @@ jobs:
           python train.py
 
           cat metrics.txt >> report.md
-          cml-publish plot.png --md >> report.md
-          cml-send-comment report.md
+          cml publish plot.png --md >> report.md
+          cml send-comment report.md
 ```
 
 3. In your text editor of choice, edit line 16 of `train.py` to `depth = 5`.
@@ -221,7 +221,7 @@ git push origin experiment
 ![](https://static.iterative.ai/img/cml/make_pr.png)
 
 Shortly, you should see a comment from `github-actions` appear in the pull
-request with your CML report. This is a result of the `cml-send-comment`
+request with your CML report. This is a result of the `cml send-comment`
 function in your workflow.
 
 ![](https://static.iterative.ai/img/cml/first_report.png)
@@ -279,14 +279,16 @@ jobs:
           echo "## Plots" >> report.md
           echo "### Class confusions" >> report.md
           dvc plots diff --target classes.csv --template confusion -x actual -y predicted --show-vega master > vega.json
-          vl2png vega.json -s 1.5 | cml-publish --md >> report.md
+          vl2png vega.json -s 1.5 > plot.png
+          cml publish --md plot.png >> report.md
 
           # Publish regularization function diff
           echo "### Effects of regularization" >> report.md
           dvc plots diff --target estimators.csv -x Regularization --show-vega master > vega.json
-          vl2png vega.json -s 1.5 | cml-publish --md >> report.md
+          vl2png vega.json -s 1.5 > plot.png
+          cml publish --md plot.png >> report.md
 
-          cml-send-comment report.md
+          cml send-comment report.md
 ```
 
 > :warning: If you're using DVC with cloud storage, take note of environment
@@ -314,7 +316,7 @@ env:
 > :point_right: `AWS_SESSION_TOKEN` is optional.
 
 > :point_right: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` can also be used
-> by `cml-runner` to launch EC2 instances. See [Environment Variables].
+> by `cml runner` to launch EC2 instances. See [Environment Variables].
 
 </details>
 
@@ -397,19 +399,19 @@ orchestrate your team's shared computing resources, or train in the cloud.
 #### Allocating Cloud Compute Resources with CML
 
 When a workflow requires computational resources (such as GPUs), CML can
-automatically allocate cloud instances using `cml-runner`. You can spin up
+automatically allocate cloud instances using `cml runner`. You can spin up
 instances on AWS, Azure, GCP, or Kubernetes.
 
 For example, the following workflow deploys a `g4dn.xlarge` instance on AWS EC2
 and trains a model on the instance. After the job runs, the instance
 automatically shuts down.
 
 You might notice that this workflow is quite similar to the
-[basic use case](#usage) above. The only addition is `cml-runner` and a few
+[basic use case](#usage) above. The only addition is `cml runner` and a few
 environment variables for passing your cloud service credentials to the
 workflow.
 
-Note that `cml-runner` will also automatically restart your jobs (whether from a
+Note that `cml runner` will also automatically restart your jobs (whether from a
 [GitHub Actions 72-hour timeout](https://docs.github.com/en/actions/reference/usage-limits-billing-and-administration#usage-limits)
 or a
 [AWS EC2 spot instance interruption](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html)).
@@ -429,7 +431,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          cml-runner \
+          cml runner \
             --cloud=aws \
             --cloud-region=us-west \
             --cloud-type=g4dn.xlarge \
@@ -451,7 +453,7 @@ jobs:
           python train.py
 
           cat metrics.txt > report.md
-          cml-send-comment report.md
+          cml send-comment report.md
 ```
 
 In the workflow above, the `deploy-runner` step launches an EC2 `g4dn.xlarge`
@@ -460,7 +462,7 @@ newly-launched instance. See [Environment Variables] below for details on the
 `secrets` required.
 
 > :tada: **Note that jobs can use any Docker container!** To use functions such
-> as `cml-send-comment` from a job, the only requirement is to
+> as `cml send-comment` from a job, the only requirement is to
 > [have CML installed](#local-package).
 
 #### Docker Images
@@ -480,7 +482,7 @@ For example, `docker://iterativeai/cml:0-dvc2-base1-gpu`, or
 
 #### Arguments
 
-The `cml-runner` function accepts the following arguments:
+The `cml runner` function accepts the following arguments:
 
 ```
 --help                      Show help                                [boolean]
@@ -555,12 +557,12 @@ CML support proxy via known environment variables `http_proxy` and
 
 #### On-premise (Local) Runners
 
-This means using on-premise machines as self-hosted runners. The `cml-runner`
+This means using on-premise machines as self-hosted runners. The `cml runner`
 function is used to set up a local self-hosted runner. On a local machine or
 on-premise GPU cluster, [install CML as a package](#local-package) and then run:
 
 ```bash
-cml-runner \
+cml runner \
   --repo=$your_project_repository_url \
   --token=$PERSONAL_ACCESS_TOKEN \
   --labels="local,runner" \

diff --git a/bin/cml/ci.js b/bin/cml/ci.js
@@ -1,5 +1,6 @@
 const kebabcaseKeys = require('kebabcase-keys');
 
+const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml');
 const CML = require('../../src/cml').default;
 
 exports.command = 'ci';
@@ -13,20 +14,35 @@ exports.handler = async (opts) => {
 exports.builder = (yargs) =>
   yargs.env('CML_CI').options(
     kebabcaseKeys({
+      unshallow: {
+        type: 'boolean',
+        description:
+          'Fetch as much as possible, converting a shallow repository to a complete one.'
+      },
+      userEmail: {
+        type: 'string',
+        default: GIT_USER_EMAIL,
+        description: 'Set Git user email.'
+      },
+      userName: {
+        type: 'string',
+        default: GIT_USER_NAME,
+        description: 'Set Git user name.'
+      },
       repo: {
         type: 'string',
         description:
-          'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+          'Set repository to be used. If unspecified, inferred from the environment.'
       },
       token: {
         type: 'string',
         description:
-          'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
+          'Personal access token to be used. If unspecified, inferred from the environment.'
       },
       driver: {
         type: 'string',
         choices: ['github', 'gitlab', 'bitbucket'],
-        description: 'If not specify it infers it from the ENV.'
+        description: 'If unspecified, inferred from the environment.'
       }
     })
   );
diff --git a/bin/cml/ci.test.js b/bin/cml/ci.test.js
@@ -10,15 +10,19 @@ describe('CML e2e', () => {
       Fixes specific CI setups
 
       Options:
-        --help     Show help                                                 [boolean]
-        --version  Show version number                                       [boolean]
-        --log      Maximum log level
+        --help        Show help                                              [boolean]
+        --version     Show version number                                    [boolean]
+        --log         Maximum log level
                 [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
-        --repo     Specifies the repo to be used. If not specified is extracted from
-                   the CI ENV.                                                [string]
-        --token    Personal access token to be used. If not specified in extracted
-                   from ENV REPO_TOKEN.                                       [string]
-        --driver   If not specify it infers it from the ENV.
+        --unshallow   Fetch as much as possible, converting a shallow repository to a
+                      complete one.                                          [boolean]
+        --user-email  Set Git user email.    [string] [default: \\"[email protected]\\"]
+        --user-name   Set Git user name.             [string] [default: \\"Olivaw[bot]\\"]
+        --repo        Set repository to be used. If unspecified, inferred from the
+                      environment.                                            [string]
+        --token       Personal access token to be used. If unspecified, inferred from
+                      the environment.                                        [string]
+        --driver      If unspecified, inferred from the environment.
                                    [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]"
     `);
   });

diff --git a/bin/cml/pr.js b/bin/cml/pr.js
@@ -19,10 +19,21 @@ exports.builder = (yargs) =>
         type: 'boolean',
         description: 'Output in markdown format [](url).'
       },
-      autoMerge: {
+      merge: {
         type: 'boolean',
-        description:
-          'Mark the PR/MR for automatic merging after tests pass (unsupported by Bitbucket).'
+        alias: 'auto-merge',
+        conflicts: ['rebase', 'squash'],
+        description: 'Try to merge the pull request upon creation.'
+      },
+      rebase: {
+        type: 'boolean',
+        conflicts: ['merge', 'squash'],
+        description: 'Try to rebase-merge the pull request upon creation.'
+      },
+      squash: {
+        type: 'boolean',
+        conflicts: ['merge', 'rebase'],
+        description: 'Try to squash-merge the pull request upon creation.'
       },
       remote: {
         type: 'string',