From dc0f8a310e86f8428774019755fa358dca57a91a Mon Sep 17 00:00:00 2001 From: Saleha Muzammil <84681153+saleha-muzammil@users.noreply.github.com> Date: Wed, 17 Jul 2024 00:00:19 +0500 Subject: [PATCH] CI setup with Justfiles (#29) * CI setup with Justfiles * Update Justfile * Update Justfile * Update ci.yml * Update struct_parser.py * Fixed mypy errors --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++ Justfile | 20 +++++++++++++ flake.nix | 27 ++++++++++------- probe_src/arena/parse_arena.py | 38 +++++++----------------- probe_src/probe_py/struct_parser.py | 44 ++++++++++++++-------------- 5 files changed, 114 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Justfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..d97b0d1b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Nix + uses: cachix/install-nix-action@v14 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Run Nix development shell + run: nix develop --command just format-nix + + - name: Format Python code with Black + run: nix develop --command just format-python + + - name: Check Python code with Ruff + run: nix develop --command just check-ruff + + - name: Check Python code with Mypy + run: nix develop --command just check-mypy + + - name: Run tests + run: nix develop --command just test + + - name: Run flake checks (optional) + run: nix develop --command just flake-check + continue-on-error: true + + - name: Run developer tests + if: github.event_name == 'push' && github.event.head_commit.message == 'run dev tests' + run: nix develop --command just test-dev diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..6654decb --- /dev/null +++ b/Justfile @@ -0,0 +1,20 @@ +format-nix: + alejandra . + +format-python: + black probe_src/probe_py + +check-ruff: + ruff check probe_src/probe_py + +check-mypy: + (cd probe_src && mypy --package probe_py --strict) + +test: + python -m pytest probe_src/probe_py + +test-dev: + python -m pytest probe_src/probe_py --failed-first --maxfail=1 + +flake-check: + nix flake check --all-systems diff --git a/flake.nix b/flake.nix index 7b11a360..1c4fe398 100644 --- a/flake.nix +++ b/flake.nix @@ -14,17 +14,19 @@ configureFlags = oldAttrs.configureFlags ++ ["--with-pydebug"]; # patches = oldAttrs.patches ++ [ ./python.patch ]; }); - export-and-rename = pkg: file-pairs: pkgs.stdenv.mkDerivation { - pname = "${pkg.pname}-only-bin"; - dontUnpack = true; - version = pkg.version; - buildInputs = [ pkg ]; - buildPhase = builtins.concatStringsSep - "\n" - (builtins.map - (pairs: "install -D ${pkg}/${builtins.elemAt pairs 0} $out/${builtins.elemAt pairs 1}") - file-pairs); - }; + export-and-rename = pkg: file-pairs: + pkgs.stdenv.mkDerivation { + pname = "${pkg.pname}-only-bin"; + dontUnpack = true; + version = pkg.version; + buildInputs = [pkg]; + buildPhase = + builtins.concatStringsSep + "\n" + (builtins.map + (pairs: "install -D ${pkg}/${builtins.elemAt pairs 0} $out/${builtins.elemAt pairs 1}") + file-pairs); + }; in { packages = { python-dbg = python312-debug; @@ -49,6 +51,9 @@ pkgs.bash pkgs.alejandra pkgs.hyperfine + pkgs.just + pkgs.black + pkgs.ruff ] ++ ( # gdb broken on apple silicon diff --git a/probe_src/arena/parse_arena.py b/probe_src/arena/parse_arena.py index 31d3e688..3916fee1 100755 --- a/probe_src/arena/parse_arena.py +++ b/probe_src/arena/parse_arena.py @@ -4,8 +4,7 @@ import dataclasses import pathlib import ctypes -import typing - +from typing import Sequence @dataclasses.dataclass(frozen=True) class MemorySegment: @@ -24,12 +23,6 @@ def _check(self) -> None: def length(self) -> int: return self.stop - self.start - @typing.overload - def __getitem__(self, idx: slice) -> bytes: ... - - @typing.overload - def __getitem__(self, idx: int) -> int: ... - def __getitem__(self, idx: slice | int) -> bytes | int: if isinstance(idx, slice): if not (self.start <= idx.start <= idx.stop <= self.stop): @@ -37,6 +30,8 @@ def __getitem__(self, idx: slice | int) -> bytes | int: return self.buffr[idx.start - self.start : idx.stop - self.start : idx.step] elif isinstance(idx, int): return self.buffr[idx - self.start] + else: + raise TypeError("Invalid index type") def __contains__(self, idx: int) -> bool: return self.start <= idx < self.stop @@ -55,38 +50,28 @@ def __repr__(self) -> str: @dataclasses.dataclass(frozen=True) class MemorySegments: - segments: typing.Sequence[MemorySegment] + segments: Sequence[MemorySegment] def __post_init__(self) -> None: self._check() def _check(self) -> None: - # Memory segments *are* allowed to overlap, - # Since we can de-allocate arenas, - # we can potentially reuse those addresses. - # for a, segment_a in enumerate(self.segments): - # for segment_b in self.segments[a + 1:]: - # assert not segment_a.overlaps(segment_b), (segment_a, "overlaps", segment_b) assert sorted(self.segments, key=lambda segment: segment.start) == self.segments - @typing.override - def __getitem__(self, idx: slice) -> bytes: ... - - @typing.override - def __getitem__(self, idx: int) -> int: ... - def __getitem__(self, idx: slice | int) -> bytes | int: if isinstance(idx, slice): buffr = b'' for segment in self.segments: buffr += segment.buffr[max(idx.start, segment.start) - segment.start : min(idx.stop, segment.stop) - segment.start] return buffr[::idx.step] - else: + elif isinstance(idx, int): for segment in self.segments: if idx in segment: return segment[idx] else: raise IndexError(idx) + else: + raise TypeError("Invalid index type") def __contains__(self, idx: int) -> bool: return any(idx in segment for segment in self.segments) @@ -96,7 +81,6 @@ class CArena(ctypes.Structure): _fields_ = [ ("instantiation", ctypes.c_size_t), ("base_address", ctypes.c_void_p), - # echo -e '#include \n' | cpp | grep uinptr_t ("capacity", ctypes.c_ulong), ("used", ctypes.c_ulong), ] @@ -109,8 +93,8 @@ def parse_arena_buffer(buffr: bytes) -> MemorySegment: return MemorySegment(buffr[ctypes.sizeof(CArena) : c_arena.used], start, stop) -def parse_arena_dir(arena_dir: pathlib.Path) -> typing.Sequence[MemorySegment]: - memory_segments = list[MemorySegment]() +def parse_arena_dir(arena_dir: pathlib.Path) -> Sequence[MemorySegment]: + memory_segments = [] for path in sorted(arena_dir.iterdir()): assert path.name.endswith(".dat") buffr = path.read_bytes() @@ -121,8 +105,8 @@ def parse_arena_dir(arena_dir: pathlib.Path) -> typing.Sequence[MemorySegment]: def parse_arena_dir_tar( arena_dir_tar: tarfile.TarFile, prefix: pathlib.Path = pathlib.Path(), -) -> typing.Sequence[MemorySegment]: - memory_segments = list[MemorySegment]() +) -> Sequence[MemorySegment]: + memory_segments = [] for member in sorted(arena_dir_tar, key=lambda member: member.name): member_path = pathlib.Path(member.name) if member_path.is_relative_to(prefix) and member_path.relative_to(prefix) != pathlib.Path("."): diff --git a/probe_src/probe_py/struct_parser.py b/probe_src/probe_py/struct_parser.py index 67bf3210..842c8596 100644 --- a/probe_src/probe_py/struct_parser.py +++ b/probe_src/probe_py/struct_parser.py @@ -6,7 +6,7 @@ import enum import textwrap import typing -import pycparser # type: ignore +import pycparser _T = typing.TypeVar("_T") @@ -34,7 +34,7 @@ ("unsigned", "__int64"): ctypes.c_ulonglong, ("size_t",): ctypes.c_size_t, ("ssize_t",): ctypes.c_ssize_t, - ("time_t",): ctypes.c_time_t, # type: ignore + ("time_t",): ctypes.c_time_t, ("float",): ctypes.c_float, ("double",): ctypes.c_double, ("long", "double",): ctypes.c_longdouble, @@ -72,7 +72,7 @@ class PyUnionBase: ("unsigned", "__int64"): int, ("size_t",): int, ("ssize_t",): int, - ("time_t",): int, # type: ignore + ("time_t",): int, ("float",): float, ("double",): float, ("long", "double",): int, @@ -177,14 +177,14 @@ def ast_to_cpy_type( if isinstance(inner_c_type, Exception): c_type = inner_c_type else: - c_type = int_representing_pointer(inner_c_type) # type: ignore + c_type = int_representing_pointer(inner_c_type) if isinstance(inner_py_type, Exception): c_type = inner_py_type else: if inner_c_type == ctypes.c_char: py_type = str else: - py_type = list[inner_py_type] # type: ignore + py_type = list[inner_py_type] return c_type, py_type elif isinstance(typ, pycparser.c_ast.ArrayDecl): repetitions = eval_compile_time_int(c_types, py_types, typ.dim, name) @@ -194,7 +194,7 @@ def ast_to_cpy_type( if isinstance(inner_c_type, Exception): array_c_type = inner_c_type else: - array_c_type = inner_c_type * repetitions # type: ignore + array_c_type = inner_c_type * repetitions if isinstance(inner_py_type, Exception): array_py_type = inner_py_type else: @@ -268,7 +268,7 @@ def parse_struct_or_union( zip(field_names, field_py_types), bases=(PyStructBase if is_struct else PyUnionBase,), frozen=True, - ) # type: ignore + ) if c_type_error is None: c_types[(keyword, name)] = type( @@ -355,7 +355,7 @@ def c_type_to_c_source(c_type: CType, top_level: bool = True) -> str: return "\n".join([ keyword + " " + c_type.__name__ + " " + "{", *[ - textwrap.indent(c_type_to_c_source(field[1], False), " ") + " " + field[0] + ";" # type: ignore + textwrap.indent(c_type_to_c_source(field[1], False), " ") + " " + field[0] + ";" for field in c_type._fields_ ], "}", @@ -365,9 +365,9 @@ def c_type_to_c_source(c_type: CType, top_level: bool = True) -> str: elif isinstance(c_type, CArrayType): return c_type_to_c_source(c_type._type_, False) + "[" + str(c_type._length_) + "]" elif isinstance(c_type, type(ctypes._Pointer)): - return c_type_to_c_source(c_type._type_, False) + "*" # type: ignore + return c_type_to_c_source(c_type._type_, False) + "*" elif isinstance(c_type, type(ctypes._SimpleCData)): - name = c_type.__name__ # type: ignore + name = c_type.__name__ return { # Ints "c_byte": "byte", @@ -431,35 +431,35 @@ def convert_c_obj_to_py_obj( elif c_obj.__class__.__name__ == "PointerStruct": assert py_type.__name__ == "list" or py_type is str, (type(c_obj), py_type) if py_type.__name__ == "list": - inner_py_type = py_type.__args__[0] # type: ignore + inner_py_type = py_type.__args__[0] else: inner_py_type = str inner_c_type = c_obj.inner_c_type size = ctypes.sizeof(inner_c_type) pointer_int = _expect_type(int, c_obj.value) if pointer_int == 0: - return None # type: ignore + return None if pointer_int not in memory: raise ValueError(f"Pointer {pointer_int:08x} is outside of memory {memory!s}") - lst: inner_py_type = [] # type: ignore + lst: inner_py_type = [] idx = 0 while True: cont, sub_info = (memory[pointer_int : pointer_int + 1] != b'\0', None) if info is None else info[0](memory, pointer_int) if cont: inner_c_obj = inner_c_type.from_buffer_copy(memory[pointer_int : pointer_int + size]) - inner_py_obj = convert_c_obj_to_py_obj( # type: ignore + inner_py_obj = convert_c_obj_to_py_obj( inner_c_obj, inner_py_type, sub_info, memory, depth + 1, ) - lst.append(inner_py_obj) # type: ignore + lst.append(inner_py_obj) pointer_int += size else: break if py_type is str: - return "".join(lst) # type: ignore + return "".join(lst) else: return lst elif isinstance(c_obj, ctypes.Array): @@ -490,7 +490,7 @@ def convert_c_obj_to_py_obj( memory, depth + 1, ) - return py_type(**fields) # type: ignore + return py_type(**fields) elif isinstance(c_obj, ctypes.Union): if not dataclasses.is_dataclass(py_type): raise TypeError(f"If {type(c_obj)} is a union, then {py_type} should be a dataclass") @@ -509,16 +509,16 @@ def convert_c_obj_to_py_obj( elif isinstance(c_obj, ctypes._SimpleCData): if isinstance(py_type, enum.EnumType): assert isinstance(c_obj.value, int) - return py_type(c_obj.value) # type: ignore + return py_type(c_obj.value) elif py_type is str: assert isinstance(c_obj, ctypes.c_char) - return c_obj.value.decode() # type: ignore + return c_obj.value.decode() else: ret = c_obj.value - return _expect_type(py_type, ret) # type: ignore + return _expect_type(py_type, ret) elif isinstance(c_obj, py_type): - return c_obj # type: ignore + return c_obj elif isinstance(c_obj, int) and isinstance(py_type, enum.EnumType): - return py_type(c_obj) # type: ignore + return py_type(c_obj) else: raise TypeError(f"{c_obj!r} of c_type {type(c_obj)!r} cannot be converted to py_type {py_type!r}")