From 7a30065ea004c30dee42f3fa908c821448d2136e Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 8 Jan 2025 20:10:28 +0800 Subject: [PATCH 1/2] feat(ci): adopt ruff & lychee Signed-off-by: Keming --- .github/workflows/envd-lint.yml | 2 +- .github/workflows/link-check.yml | 21 ++++++------ .gitignore | 3 ++ .lycheeignore | 7 ++++ .markdown-lint.json | 7 ---- Makefile | 13 ++++---- .../language/testdata/python/conda/setup.py | 2 +- .../testdata/python/requirements/setup.py | 2 +- .../language/testdata/python/conda/setup.py | 2 +- .../testdata/python/requirements/setup.py | 2 +- envd/api/v0/config.py | 2 +- envd/api/v0/io.py | 1 - envd/api/v0/runtime.py | 2 +- envd/api/v1/install.py | 32 ++++++++++--------- envd/api/v1/runtime.py | 2 +- examples/dgl/train.py | 16 ++++------ examples/llm-inference/8bit.py | 3 +- examples/llm-inference/main.py | 1 - examples/mnist/main.py | 7 ++-- examples/mnist/mnist.ipynb | 7 ++-- examples/pytorch-profiler/main.py | 2 +- examples/pytorch2/main.py | 2 +- examples/resnet-serving-v1/client.py | 4 ++- examples/resnet-serving-v1/main.py | 5 ++- examples/stable-diffusion/README.md | 2 +- examples/stable-diffusion/main.py | 3 +- examples/streamlit-mnist/app.py | 5 ++- examples/streamlit-mnist/train.ipynb | 4 +-- lychee.toml | 4 +++ pyproject.toml | 9 ++++++ setup.py | 11 +++---- 31 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 .lycheeignore delete mode 100644 .markdown-lint.json create mode 100644 lychee.toml diff --git a/.github/workflows/envd-lint.yml b/.github/workflows/envd-lint.yml index bcc80be06..ff7d31edf 100644 --- a/.github/workflows/envd-lint.yml +++ b/.github/workflows/envd-lint.yml @@ -31,6 +31,6 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: Lint run: make envd-lint diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml index 1cfab5b7f..1b6ac8238 100644 --- a/.github/workflows/link-check.yml +++ b/.github/workflows/link-check.yml @@ -1,26 +1,27 @@ -name: documentation-check +name: Link check on: push: branches: - main paths: - - ".github/workflows/**" + - ".github/workflows/link-check.yml" - "**.md" pull_request: paths: - - ".github/workflows/**" + - ".github/workflows/link-check.yml" - "**.md" + workflow_dispatch: jobs: - markdown-link-check: + linkChecker: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: gaurav-nelson/github-action-markdown-link-check@v1 + + - name: Link Checker + id: lychee + uses: lycheeverse/lychee-action@v2 with: - file-path: "README.md" - folder-path: "docs" - check-modified-files-only: yes - base-branch: main - config-file: .markdown-lint.json + fail: true + args: --verbose --no-progress --format detailed . diff --git a/.gitignore b/.gitignore index 82056c4d9..ba8962223 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ MANIFEST .GIT_TAG_INFO test.tar + +# ruff +.ruff_cache/ diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 000000000..b8d629203 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,7 @@ +# blog no longer exists +https://www.iam.rw/ +http://lizheming.top/ +https://bandism.net/ +https://blog.thrimbda.com/ +# github disappeared (rename?) +https://github.com/yczheng0 diff --git a/.markdown-lint.json b/.markdown-lint.json deleted file mode 100644 index 231809a08..000000000 --- a/.markdown-lint.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "ignorePatterns": [ - { - "pattern": "^http://localhost.*" - }, - ] -} \ No newline at end of file diff --git a/Makefile b/Makefile index bec75617b..cecb1d5c5 100644 --- a/Makefile +++ b/Makefile @@ -254,11 +254,12 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... -black-install: - @pip install -q black[jupyter] +ruff-install: + @pip install -q ruff -envd-lint: black-install - black --check --include '(\.envd|\.py|\.ipynb)$$' . +envd-lint: ruff-install + @ruff --check . -envd-fmt: black-install - black --include '(\.envd|\.py|\.ipynb)$$' . +envd-fmt: ruff-install + @ruff format . + @ruff check --fix . diff --git a/e2e/v0/language/testdata/python/conda/setup.py b/e2e/v0/language/testdata/python/conda/setup.py index 0be13a508..d69175316 100644 --- a/e2e/v0/language/testdata/python/conda/setup.py +++ b/e2e/v0/language/testdata/python/conda/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name="envd_e2e_conda_test", diff --git a/e2e/v0/language/testdata/python/requirements/setup.py b/e2e/v0/language/testdata/python/requirements/setup.py index 0be13a508..d69175316 100644 --- a/e2e/v0/language/testdata/python/requirements/setup.py +++ b/e2e/v0/language/testdata/python/requirements/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name="envd_e2e_conda_test", diff --git a/e2e/v1/language/testdata/python/conda/setup.py b/e2e/v1/language/testdata/python/conda/setup.py index 0be13a508..d69175316 100644 --- a/e2e/v1/language/testdata/python/conda/setup.py +++ b/e2e/v1/language/testdata/python/conda/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name="envd_e2e_conda_test", diff --git a/e2e/v1/language/testdata/python/requirements/setup.py b/e2e/v1/language/testdata/python/requirements/setup.py index 0be13a508..d69175316 100644 --- a/e2e/v1/language/testdata/python/requirements/setup.py +++ b/e2e/v1/language/testdata/python/requirements/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name="envd_e2e_conda_test", diff --git a/envd/api/v0/config.py b/envd/api/v0/config.py index dda6bfa31..bc13b382c 100644 --- a/envd/api/v0/config.py +++ b/envd/api/v0/config.py @@ -21,7 +21,7 @@ ::: """ -from typing import Optional, List +from typing import List, Optional def apt_source(source: Optional[str]): diff --git a/envd/api/v0/io.py b/envd/api/v0/io.py index fd47b73cf..8c1ac67b5 100644 --- a/envd/api/v0/io.py +++ b/envd/api/v0/io.py @@ -21,7 +21,6 @@ ::: """ - from typing import Optional diff --git a/envd/api/v0/runtime.py b/envd/api/v0/runtime.py index ed08191b4..081ab2bcd 100644 --- a/envd/api/v0/runtime.py +++ b/envd/api/v0/runtime.py @@ -21,7 +21,7 @@ ::: """ -from typing import Dict, Optional, List +from typing import Dict, List, Optional def command(commands: Dict[str, str]): diff --git a/envd/api/v1/install.py b/envd/api/v1/install.py index b1790474f..ba4eb19a8 100644 --- a/envd/api/v1/install.py +++ b/envd/api/v1/install.py @@ -27,7 +27,7 @@ ::: """ -from typing import List, Optional +from typing import Optional, Sequence def python(version: str = "3.9"): @@ -57,59 +57,61 @@ def julia(): """Install Julia.""" -def apt_packages(name: List[str] = []): +def apt_packages(name: Sequence[str] = ()): """Install package using the system package manager (apt on Ubuntu). Args: - name (List[str]): apt package name list + name (Sequence[str]): apt package name list """ def python_packages( - name: List[str] = [], requirements: str = "", local_wheels: List[str] = [] + name: Sequence[str] = (), requirements: str = "", local_wheels: Sequence[str] = () ): """Install python package by pip. Args: - name (List[str]): package name list + name (Sequence[str]): package name list requirements (str): requirements file path - local_wheels (List[str]): local wheels + local_wheels (Sequence[str]): local wheels (wheel files should be placed under the current directory) """ -def conda_packages(name: List[str] = [], channel: List[str] = [], env_file: str = ""): +def conda_packages( + name: Sequence[str] = (), channel: Sequence[str] = (), env_file: str = "" +): """Install python package by Conda Args: - name (List[str]): List of package names with optional version assignment, + name (Sequence[str]): List of package names with optional version assignment, such as ['pytorch', 'tensorflow==1.13.0'] - channel (List[str]): additional channels + channel (Sequence[str]): additional channels env_file (str): conda env file path """ -def r_packages(name: List[str]): +def r_packages(name: Sequence[str]): """Install R packages by R package manager. Args: - name (List[str]): package name list + name (Sequence[str]): package name list """ -def julia_packages(name: List[str]): +def julia_packages(name: Sequence[str]): """Install Julia packages. Args: - name (List[str]): List of Julia packages + name (Sequence[str]): List of Julia packages """ -def vscode_extensions(name: List[str]): +def vscode_extensions(name: Sequence[str]): """Install VS Code extensions Args: - name (List[str]): extension names, such as ['ms-python.python'] + name (Sequence[str]): extension names, such as ['ms-python.python'] """ diff --git a/envd/api/v1/runtime.py b/envd/api/v1/runtime.py index bfed56f47..b38648da2 100644 --- a/envd/api/v1/runtime.py +++ b/envd/api/v1/runtime.py @@ -27,7 +27,7 @@ ::: """ -from typing import List, Optional, Dict +from typing import Dict, List, Optional def command(commands: Dict[str, str]): diff --git a/examples/dgl/train.py b/examples/dgl/train.py index d707c3c4f..f60326f42 100644 --- a/examples/dgl/train.py +++ b/examples/dgl/train.py @@ -13,13 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse + +import dgl.nn as dglnn import torch -import torch.nn as nn import torch.nn.functional as F -import dgl.nn as dglnn -from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset from dgl import AddSelfLoop -import argparse +from dgl.data import CiteseerGraphDataset, CoraGraphDataset, PubmedGraphDataset +from torch import nn class GAT(nn.Module): @@ -52,10 +53,7 @@ def forward(self, g, inputs): h = inputs for i, layer in enumerate(self.gat_layers): h = layer(g, h) - if i == 1: # last layer - h = h.mean(1) - else: # other layer(s) - h = h.flatten(1) + h = h.mean(1) if i == 1 else h.flatten(1) return h @@ -102,7 +100,7 @@ def train(g, features, labels, masks, model): help="Dataset name ('cora', 'citeseer', 'pubmed').", ) args = parser.parse_args() - print(f"Training with DGL built-in GATConv module.") + print("Training with DGL built-in GATConv module.") # load and preprocess dataset transform = ( diff --git a/examples/llm-inference/8bit.py b/examples/llm-inference/8bit.py index 03c3f6978..380f052f4 100644 --- a/examples/llm-inference/8bit.py +++ b/examples/llm-inference/8bit.py @@ -1,5 +1,4 @@ -from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed -from transformers import pipeline +from transformers import AutoModelForCausalLM, AutoTokenizer name = "bigscience/bloom-3b" text = "Hello my name is" diff --git a/examples/llm-inference/main.py b/examples/llm-inference/main.py index b7435ab1e..ee723d626 100644 --- a/examples/llm-inference/main.py +++ b/examples/llm-inference/main.py @@ -1,4 +1,3 @@ -from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed from transformers import pipeline name = "bigscience/bloom-3b" diff --git a/examples/mnist/main.py b/examples/mnist/main.py index c462f5224..471ca5d76 100644 --- a/examples/mnist/main.py +++ b/examples/mnist/main.py @@ -6,13 +6,14 @@ # from https://github.com/pytorch/examples/blob/main/mnist/main.py from __future__ import print_function + import argparse + import torch -import torch.nn as nn import torch.nn.functional as F -import torch.optim as optim -from torchvision import datasets, transforms +from torch import nn, optim from torch.optim.lr_scheduler import StepLR +from torchvision import datasets, transforms class Net(nn.Module): diff --git a/examples/mnist/mnist.ipynb b/examples/mnist/mnist.ipynb index 189cd9e56..b2f819d91 100644 --- a/examples/mnist/mnist.ipynb +++ b/examples/mnist/mnist.ipynb @@ -8,13 +8,12 @@ "outputs": [], "source": [ "from __future__ import print_function\n", - "import argparse\n", + "\n", "import torch\n", - "import torch.nn as nn\n", "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from torchvision import datasets, transforms\n", + "from torch import nn, optim\n", "from torch.optim.lr_scheduler import StepLR\n", + "from torchvision import datasets, transforms\n", "\n", "\n", "class Net(nn.Module):\n", diff --git a/examples/pytorch-profiler/main.py b/examples/pytorch-profiler/main.py index 1fca3e609..873e15b06 100644 --- a/examples/pytorch-profiler/main.py +++ b/examples/pytorch-profiler/main.py @@ -7,7 +7,7 @@ import torchvision.datasets import torchvision.models import torchvision.transforms as T -from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights +from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0 transform = T.Compose( [T.Resize(224), T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] diff --git a/examples/pytorch2/main.py b/examples/pytorch2/main.py index 98eceb755..a29ae541e 100644 --- a/examples/pytorch2/main.py +++ b/examples/pytorch2/main.py @@ -7,7 +7,7 @@ import torchvision.datasets import torchvision.models import torchvision.transforms as T -from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights +from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0 transform = T.Compose( [T.Resize(224), T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] diff --git a/examples/resnet-serving-v1/client.py b/examples/resnet-serving-v1/client.py index 685b70b0f..d8266c060 100644 --- a/examples/resnet-serving-v1/client.py +++ b/examples/resnet-serving-v1/client.py @@ -13,6 +13,8 @@ # limitations under the License. """Example: Sample Resnet client.""" +from http import HTTPStatus + import httpx import msgpack # type: ignore @@ -25,7 +27,7 @@ "http://localhost:8000/inference", data=msgpack.packb({"image": dog_bytes}), ) -if prediction.status_code == 200: +if prediction.status_code == HTTPStatus.OK: print(msgpack.unpackb(prediction.content)) else: print(prediction.status_code, prediction.content) diff --git a/examples/resnet-serving-v1/main.py b/examples/resnet-serving-v1/main.py index bce3564a0..628d50fa0 100644 --- a/examples/resnet-serving-v1/main.py +++ b/examples/resnet-serving-v1/main.py @@ -21,12 +21,11 @@ import numpy as np # type: ignore import torch # type: ignore import torchvision # type: ignore -from PIL import Image # type: ignore -from torchvision import transforms # type: ignore - from mosec import Server, Worker from mosec.errors import ValidationError from mosec.mixin import MsgpackMixin +from PIL import Image # type: ignore +from torchvision import transforms # type: ignore logger = logging.getLogger() logger.setLevel(logging.DEBUG) diff --git a/examples/stable-diffusion/README.md b/examples/stable-diffusion/README.md index 48e60e0c3..9357c4673 100644 --- a/examples/stable-diffusion/README.md +++ b/examples/stable-diffusion/README.md @@ -3,7 +3,7 @@ ## Requirements
    -
  1. Sign up to Huggingface
    2. +
  2. Log In HuggingFace
  3. Accept the Stable Diffusion models agreement
  4. Create an Access Token. You’ll use it in the Python script below.
diff --git a/examples/stable-diffusion/main.py b/examples/stable-diffusion/main.py index 51c56a57a..409b3ed49 100644 --- a/examples/stable-diffusion/main.py +++ b/examples/stable-diffusion/main.py @@ -1,9 +1,8 @@ +import os import random import sys -import os from diffusers import StableDiffusionPipeline -import torch from torch import autocast device = "cuda" diff --git a/examples/streamlit-mnist/app.py b/examples/streamlit-mnist/app.py index f705e27b9..0b6be82e9 100644 --- a/examples/streamlit-mnist/app.py +++ b/examples/streamlit-mnist/app.py @@ -1,9 +1,8 @@ -import os -import numpy as np import cv2 -from tensorflow.keras.models import load_model +import numpy as np import streamlit as st from streamlit_drawable_canvas import st_canvas +from tensorflow.keras.models import load_model model = load_model("model") diff --git a/examples/streamlit-mnist/train.ipynb b/examples/streamlit-mnist/train.ipynb index 392531e9e..d4c7b7199 100644 --- a/examples/streamlit-mnist/train.ipynb +++ b/examples/streamlit-mnist/train.ipynb @@ -6,8 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import numpy as np" ] }, { diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 000000000..03c2c701f --- /dev/null +++ b/lychee.toml @@ -0,0 +1,4 @@ +scheme = [ "https", "http", "mailto" ] +exclude_loopback = true +exclude_path = ["CHANGELOG.md"] +accept = ["200..=204", "429", "500..=599"] diff --git a/pyproject.toml b/pyproject.toml index 9787c3bdf..57937ecff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,12 @@ [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" + +[tool.ruff] +target-version = "py312" +extend-include = ["*.envd"] +[tool.ruff.lint] +select = ["E", "F", "G", "B", "I", "SIM", "TID", "PL", "RUF"] +ignore = ["E501", "PLW2901", "F821"] +[tool.ruff.lint.pylint] +max-args = 7 diff --git a/setup.py b/setup.py index 495842103..51ef746ac 100644 --- a/setup.py +++ b/setup.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os -from setuptools.command.sdist import sdist -from setuptools import setup, Extension, find_packages +import subprocess + +from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist from wheel.bdist_wheel import bdist_wheel - -import subprocess -import logging - with open("README.md", "r", encoding="utf-8") as f: readme = f.read() From ab29176cb3deb6fd408398074b901336130239ef Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 8 Jan 2025 20:13:47 +0800 Subject: [PATCH 2/2] fix typo Signed-off-by: Keming --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cecb1d5c5..1a3ac5319 100644 --- a/Makefile +++ b/Makefile @@ -258,7 +258,7 @@ ruff-install: @pip install -q ruff envd-lint: ruff-install - @ruff --check . + @ruff check . envd-fmt: ruff-install @ruff format .