feat(ci): adopt ruff & lychee (#1935)

* feat(ci): adopt ruff & lychee

Signed-off-by: Keming <[email protected]>

* fix typo

Signed-off-by: Keming <[email protected]>

---------

Signed-off-by: Keming <[email protected]>

.github/workflows/envd-lint.yml

@@ -31,6 +31,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Lint
         run: make envd-lint

.github/workflows/link-check.yml

@@ -1,26 +1,27 @@
-name: documentation-check
+name: Link check
 
 on:
   push:
     branches:
       - main
     paths:
-      - ".github/workflows/**"
+      - ".github/workflows/link-check.yml"
       - "**.md"
   pull_request:
     paths:
-      - ".github/workflows/**"
+      - ".github/workflows/link-check.yml"
       - "**.md"
+  workflow_dispatch:
 
 jobs:
-  markdown-link-check:
+  linkChecker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gaurav-nelson/github-action-markdown-link-check@v1
+
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
         with:
-          file-path: "README.md"
-          folder-path: "docs"
-          check-modified-files-only: yes
-          base-branch: main
-          config-file: .markdown-lint.json
+          fail: true
+          args: --verbose --no-progress --format detailed .

.gitignore

@@ -69,3 +69,6 @@ MANIFEST
 .GIT_TAG_INFO
 
 test.tar
+
+# ruff
+.ruff_cache/

.lycheeignore

@@ -0,0 +1,7 @@
+# blog no longer exists
+https://www.iam.rw/
+http://lizheming.top/
+https://bandism.net/
+https://blog.thrimbda.com/
+# github disappeared (rename?)
+https://github.com/yczheng0

Makefile

@@ -254,11 +254,12 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
-black-install:
-	@pip install -q black[jupyter]
+ruff-install:
+	@pip install -q ruff
 
-envd-lint: black-install
-	black --check --include '(\.envd|\.py|\.ipynb)$$' .
+envd-lint: ruff-install
+	@ruff check .
 
-envd-fmt: black-install
-	black --include '(\.envd|\.py|\.ipynb)$$' .
+envd-fmt: ruff-install
+	@ruff format .
+	@ruff check --fix .

e2e/v0/language/testdata/python/conda/setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="envd_e2e_conda_test",

e2e/v0/language/testdata/python/requirements/setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="envd_e2e_conda_test",

e2e/v1/language/testdata/python/conda/setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="envd_e2e_conda_test",

e2e/v1/language/testdata/python/requirements/setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="envd_e2e_conda_test",

envd/api/v0/config.py

@@ -21,7 +21,7 @@
 :::
 """
 
-from typing import Optional, List
+from typing import List, Optional
 
 
 def apt_source(source: Optional[str]):

envd/api/v0/io.py

@@ -21,7 +21,6 @@
 :::
 """
 
-
 from typing import Optional
 
 

envd/api/v0/runtime.py

@@ -21,7 +21,7 @@
 :::
 """
 
-from typing import Dict, Optional, List
+from typing import Dict, List, Optional
 
 
 def command(commands: Dict[str, str]):

envd/api/v1/install.py

@@ -27,7 +27,7 @@
 :::
 """
 
-from typing import List, Optional
+from typing import Optional, Sequence
 
 
 def python(version: str = "3.9"):
@@ -57,59 +57,61 @@ def julia():
     """Install Julia."""
 
 
-def apt_packages(name: List[str] = []):
+def apt_packages(name: Sequence[str] = ()):
     """Install package using the system package manager (apt on Ubuntu).
 
     Args:
-        name (List[str]): apt package name list
+        name (Sequence[str]): apt package name list
     """
 
 
 def python_packages(
-    name: List[str] = [], requirements: str = "", local_wheels: List[str] = []
+    name: Sequence[str] = (), requirements: str = "", local_wheels: Sequence[str] = ()
 ):
     """Install python package by pip.
 
     Args:
-        name (List[str]): package name list
+        name (Sequence[str]): package name list
         requirements (str): requirements file path
-        local_wheels (List[str]): local wheels
+        local_wheels (Sequence[str]): local wheels
             (wheel files should be placed under the current directory)
     """
 
 
-def conda_packages(name: List[str] = [], channel: List[str] = [], env_file: str = ""):
+def conda_packages(
+    name: Sequence[str] = (), channel: Sequence[str] = (), env_file: str = ""
+):
     """Install python package by Conda
 
     Args:
-        name (List[str]): List of package names with optional version assignment,
+        name (Sequence[str]): List of package names with optional version assignment,
             such as ['pytorch', 'tensorflow==1.13.0']
-        channel (List[str]): additional channels
+        channel (Sequence[str]): additional channels
         env_file (str): conda env file path
     """
 
 
-def r_packages(name: List[str]):
+def r_packages(name: Sequence[str]):
     """Install R packages by R package manager.
 
     Args:
-        name (List[str]): package name list
+        name (Sequence[str]): package name list
     """
 
 
-def julia_packages(name: List[str]):
+def julia_packages(name: Sequence[str]):
     """Install Julia packages.
 
     Args:
-        name (List[str]): List of Julia packages
+        name (Sequence[str]): List of Julia packages
     """
 
 
-def vscode_extensions(name: List[str]):
+def vscode_extensions(name: Sequence[str]):
     """Install VS Code extensions
 
     Args:
-        name (List[str]): extension names, such as ['ms-python.python']
+        name (Sequence[str]): extension names, such as ['ms-python.python']
     """
 
 

envd/api/v1/runtime.py

@@ -27,7 +27,7 @@
 :::
 """
 
-from typing import List, Optional, Dict
+from typing import Dict, List, Optional
 
 
 def command(commands: Dict[str, str]):

examples/dgl/train.py

@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import argparse
+
+import dgl.nn as dglnn
 import torch
-import torch.nn as nn
 import torch.nn.functional as F
-import dgl.nn as dglnn
-from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset
 from dgl import AddSelfLoop
-import argparse
+from dgl.data import CiteseerGraphDataset, CoraGraphDataset, PubmedGraphDataset
+from torch import nn
 
 
 class GAT(nn.Module):
@@ -52,10 +53,7 @@ def forward(self, g, inputs):
         h = inputs
         for i, layer in enumerate(self.gat_layers):
             h = layer(g, h)
-            if i == 1:  # last layer
-                h = h.mean(1)
-            else:  # other layer(s)
-                h = h.flatten(1)
+            h = h.mean(1) if i == 1 else h.flatten(1)
         return h
 
 
@@ -102,7 +100,7 @@ def train(g, features, labels, masks, model):
         help="Dataset name ('cora', 'citeseer', 'pubmed').",
     )
     args = parser.parse_args()
-    print(f"Training with DGL built-in GATConv module.")
+    print("Training with DGL built-in GATConv module.")
 
     # load and preprocess dataset
     transform = (

examples/llm-inference/8bit.py

@@ -1,5 +1,4 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
-from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
 name = "bigscience/bloom-3b"
 text = "Hello my name is"

examples/llm-inference/main.py

@@ -1,4 +1,3 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
 from transformers import pipeline
 
 name = "bigscience/bloom-3b"

examples/mnist/main.py

@@ -6,13 +6,14 @@
 # from https://github.com/pytorch/examples/blob/main/mnist/main.py
 
 from __future__ import print_function
+
 import argparse
+
 import torch
-import torch.nn as nn
 import torch.nn.functional as F
-import torch.optim as optim
-from torchvision import datasets, transforms
+from torch import nn, optim
 from torch.optim.lr_scheduler import StepLR
+from torchvision import datasets, transforms
 
 
 class Net(nn.Module):

examples/mnist/mnist.ipynb

@@ -8,13 +8,12 @@
    "outputs": [],
    "source": [
     "from __future__ import print_function\n",
-    "import argparse\n",
+    "\n",
     "import torch\n",
-    "import torch.nn as nn\n",
     "import torch.nn.functional as F\n",
-    "import torch.optim as optim\n",
-    "from torchvision import datasets, transforms\n",
+    "from torch import nn, optim\n",
     "from torch.optim.lr_scheduler import StepLR\n",
+    "from torchvision import datasets, transforms\n",
     "\n",
     "\n",
     "class Net(nn.Module):\n",

examples/pytorch-profiler/main.py

@@ -7,7 +7,7 @@
 import torchvision.datasets
 import torchvision.models
 import torchvision.transforms as T
-from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0
 
 transform = T.Compose(
     [T.Resize(224), T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]

examples/pytorch2/main.py

@@ -7,7 +7,7 @@
 import torchvision.datasets
 import torchvision.models
 import torchvision.transforms as T
-from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0
 
 transform = T.Compose(
     [T.Resize(224), T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]

examples/resnet-serving-v1/client.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Example: Sample Resnet client."""
 
+from http import HTTPStatus
+
 import httpx
 import msgpack  # type: ignore
 
@@ -25,7 +27,7 @@
     "http://localhost:8000/inference",
     data=msgpack.packb({"image": dog_bytes}),
 )
-if prediction.status_code == 200:
+if prediction.status_code == HTTPStatus.OK:
     print(msgpack.unpackb(prediction.content))
 else:
     print(prediction.status_code, prediction.content)

examples/resnet-serving-v1/main.py

@@ -21,12 +21,11 @@
 import numpy as np  # type: ignore
 import torch  # type: ignore
 import torchvision  # type: ignore
-from PIL import Image  # type: ignore
-from torchvision import transforms  # type: ignore
-
 from mosec import Server, Worker
 from mosec.errors import ValidationError
 from mosec.mixin import MsgpackMixin
+from PIL import Image  # type: ignore
+from torchvision import transforms  # type: ignore
 
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)

examples/stable-diffusion/README.md

@@ -3,7 +3,7 @@
 ## Requirements
 
 <ol>
-  <li><a href="https://huggingface.co/join">Sign up to Huggingface</a></li>
+  <li><a href="https://huggingface.co/login">Log In HuggingFace</a></li>
   <li><a href="https://huggingface.co/CompVis/stable-diffusion-v1-4">Accept the Stable Diffusion models agreement</a></li>
   <li><a href="https://huggingface.co/settings/tokens">Create an Access Token</a>. You’ll use it in the Python script below.</li>
 </ol>

examples/stable-diffusion/main.py

@@ -1,9 +1,8 @@
+import os
 import random
 import sys
-import os
 
 from diffusers import StableDiffusionPipeline
-import torch
 from torch import autocast
 
 device = "cuda"