Skip to content

Commit

Permalink
Improve python requirements management and create an example venv (#186)
Browse files Browse the repository at this point in the history
Make use of [uv](https://docs.astral.sh/uv/) to manage python
dependencies.

Accomplish this by replacing
`load("@rules_python//python:pip.bzl", "compile_pip_requirements")` with
`load("@rules_uv//uv:pip.bzl", "pip_compile")`
This results in `bazel run //:requirements.update && bazel run
//:gazelle_python_manifest.update` having significantly improved
performance

| branch | fresh | repeated |
| --- | --- | --- |
| master | 42s | 18s |
| better_pip_rules | 24s | 2s |

So, 9x performance improvement when doing a no-op, and 2x improvement
for the "fresh" case. Not exactly the 10-100x gains claimed by uv. But
this is a bit of a different case 🤷

Also
- Update aspect_rules_py
  • Loading branch information
michael-christen authored Feb 2, 2025
1 parent a31ed90 commit c853e45
Show file tree
Hide file tree
Showing 6 changed files with 789 additions and 79 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/flakiness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,5 @@ jobs:
run: |
# Run several times to identify flakiness, but avoid enormous tests
# Can find them with: `bazel query 'attr(size, enormous, //...)'`
bazel test --runs_per_test_detects_flakes --runs_per_test=5 --test_size_filters=-enormous -- //...
bazel test --runs_per_test_detects_flakes --runs_per_test=5 --test_tag_filters=-skip-flakiness -- //...
- run: echo "Status is ${{ job.status }}."
23 changes: 17 additions & 6 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
load("@bazel_gazelle//:def.bzl", "gazelle", "gazelle_binary")
load("@npm//:defs.bzl", "npm_link_all_packages")
load("@pip//:requirements.bzl", "all_whl_requirements")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
load("@rules_uv//uv:pip.bzl", "pip_compile")
load("@rules_uv//uv:venv.bzl", "create_venv")

package(default_visibility = ["//visibility:private"])

Expand All @@ -24,15 +25,24 @@ exports_files(
visibility = ["//visibility:public"],
)

compile_pip_requirements(
pip_compile(
name = "requirements",
# https://bazel.build/reference/be/common-definitions
size = "enormous",
timeout = "eternal", # 60m
requirements_in = "requirements.in",
requirements_txt = "requirements_lock.txt",
# Don't want to type-check requirements building
tags = ["no-mypy"],
tags = [
# Don't want to type-check requirements building
"no-mypy",
# Avoid in flake detection
"skip-flakiness",
],
)

create_venv(
name = "create_venv",
requirements_txt = "//:requirements_lock.txt",
# Example extras
site_packages_extra_files = ["//tools:utils"],
)

# This repository rule fetches the metadata for python packages we
Expand Down Expand Up @@ -130,6 +140,7 @@ gazelle(
# gazelle:resolve py third_party.bazel.proto.build_event_stream_pb2 //third_party/bazel/proto:build_event_stream_py_library
# gazelle:resolve proto pw_protobuf_protos/common.proto @pigweed//pw_protobuf:common_proto
# gazelle:resolve py examples.pigweed.modules.blinky.blinky_pb2 //examples/pigweed/modules/blinky:blinky_pb2
# gazelle:resolve py tools.git_pb2 //tools:git_py_library
# gazelle:resolve py pw_cli @pigweed//pw_system/py:pw_system_lib
# gazelle:resolve py pw_system.console @pigweed//pw_system/py:pw_system_lib
# gazelle:resolve py pw_system.device @pigweed//pw_system/py:pw_system_lib
Expand Down
16 changes: 15 additions & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ bazel_dep(name = "aspect_bazel_lib", version = "2.9.4")
########################################
# Set up rules_python and pip
########################################
bazel_dep(name = "aspect_rules_py", version = "0.7.3")
bazel_dep(name = "aspect_rules_py", version = "1.0.0")

# Minimum version needs:
# feat: add interpreter_version_info to py_runtime by @mattem in #1671
Expand All @@ -35,12 +35,26 @@ python.toolchain(
python_version = PYTHON_VERSION,
)

bazel_dep(name = "rules_uv", version = "0.45.0")

pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
hub_name = "pip",
python_version = PYTHON_VERSION,
requirements_lock = "//:requirements_lock.txt",
# Avoid building wheels with system compiler
# - pygraphviz
# download_only = True,
# Could put --extra-index-url here to grab my own wheels
# extra_pip_args = [],
)

# Could support multi-platform by specifying a different target for each
# platform
# --platform
# --python-version
# --implementation
# --abi
use_repo(pip, "pip")

# TODO(#91): Allow bumping this up to 26 or greater while avoiding segfault
Expand Down
Loading

0 comments on commit c853e45

Please sign in to comment.