diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index d701124f6..85450fafa 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -4,11 +4,11 @@ linked against musl, and what musl version is used. """ +import contextlib import functools import operator import os import re -import shutil import struct import subprocess import sys @@ -19,10 +19,11 @@ def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: return struct.unpack(fmt, f.read(struct.calcsize(fmt))) -def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: +def _parse_ld_musl_from_elf(f: IO[bytes]) -> Optional[str]: """Detect musl libc location by parsing the Python executable. - Based on https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html """ f.seek(0) try: @@ -67,39 +68,21 @@ def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: return None -def _get_ld_musl_ldd(executable: str) -> Optional[str]: - ldd = shutil.which("ldd") - if not ldd: # No dynamic program loader. - return None - proc = subprocess.run( - [ldd, executable], stdout=subprocess.PIPE, universal_newlines=True - ) - if proc.returncode != 0: # Not a valid dynamic program. - return None - for line in proc.stdout.splitlines(keepends=False): - path = line.lstrip().rsplit(None, 1)[0] - if "musl" not in path: - continue - return path - return None - - -def _get_ld_musl(executable: str) -> Optional[str]: - try: - with open(executable, "rb") as f: - return _get_ld_musl_elf(f) - except IOError: - return _get_ld_musl_ldd(executable) - - -_version_pat = re.compile(r"Version (\d+)\.(\d+)") - - class _MuslVersion(NamedTuple): major: int minor: int +def _parse_musl_version(output: str) -> Optional[_MuslVersion]: + lines = [n for n in (n.strip() for n in output.splitlines()) if n] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + m = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if not m: + return None + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + + @functools.lru_cache() def _get_musl_version(executable: str) -> Optional[_MuslVersion]: """Detect currently-running musl runtime version. @@ -112,17 +95,16 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: Version 1.2.2 Dynamic Program Loader """ - ld = _get_ld_musl(executable) + with contextlib.ExitStack() as stack: + try: + f = stack.enter_context(open(executable, "rb")) + except IOError: + return None + ld = _parse_ld_musl_from_elf(f) if not ld: return None proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True) - lines = [n for n in (n.strip() for n in proc.stderr.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + return _parse_musl_version(proc.stderr) def platform_tags(arch: str) -> Iterator[str]: @@ -141,7 +123,7 @@ def platform_tags(arch: str) -> Iterator[str]: yield f"musllinux_{sys_musl.major}_{minor}_{arch}" -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import sysconfig plat = sysconfig.get_platform() diff --git a/tests/musllinux/build.sh b/tests/musllinux/build.sh new file mode 100644 index 000000000..b9e994615 --- /dev/null +++ b/tests/musllinux/build.sh @@ -0,0 +1,58 @@ +# Build helper binaries for musllinux tests. +# Usages: +# build.sh # Build everything. +# build.sh $DISTRO $ARCH # Build one executable in $ARCH using $DISTRO. +# +# Either invocation ultimately runs this script in a Docker container with +# `build.sh glibc|musl $ARCH` to actually build the executable. + +set -euo pipefail +set -x + +build_one_in_ubuntu () { + $1 "multiarch/ubuntu-core:${2}-focal" \ + bash "/home/hello-world/musllinux/build.sh" glibc "glibc-$2" +} + +build_one_in_alpine () { + $1 "multiarch/alpine:${2}-edge" \ + sh "/home/hello-world/musllinux/build.sh" musl "musl-$2" +} + +build_in_container () { + local SOURCE="$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))" + local DOCKER="docker run --rm -v ${SOURCE}:/home/hello-world" + + if [[ $# -ne 0 ]]; then + "build_one_in_${1}" "$DOCKER" "$2" + return + fi + + build_one_in_alpine "$DOCKER" x86_64 + build_one_in_alpine "$DOCKER" i386 + build_one_in_alpine "$DOCKER" aarch64 + build_one_in_ubuntu "$DOCKER" x86_64 +} + +if [[ $# -eq 0 ]]; then + build_in_container + exit 0 +elif [[ "$1" == "glibc" ]]; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq \ + && apt-get install -qqy --no-install-recommends gcc libc6-dev +elif [[ "$1" == "musl" ]]; then + apk add -q build-base +else + build_in_container "$@" + exit 0 +fi + +build () { + local CFLAGS="" + local OUT="/home/hello-world/musllinux/${2}" + gcc -Os ${CFLAGS} -o "$OUT-full" "/home/hello-world/hello-world.c" + head -c1024 "$OUT-full" > "$OUT" + rm -f "$OUT-full" +} + +build "$@" diff --git a/tests/musllinux/glibc-x86_64 b/tests/musllinux/glibc-x86_64 new file mode 100755 index 000000000..59996e28c Binary files /dev/null and b/tests/musllinux/glibc-x86_64 differ diff --git a/tests/musllinux/musl-aarch64 b/tests/musllinux/musl-aarch64 new file mode 100755 index 000000000..1e93c3a49 Binary files /dev/null and b/tests/musllinux/musl-aarch64 differ diff --git a/tests/musllinux/musl-i386 b/tests/musllinux/musl-i386 new file mode 100755 index 000000000..e6d031649 Binary files /dev/null and b/tests/musllinux/musl-i386 differ diff --git a/tests/musllinux/musl-x86_64 b/tests/musllinux/musl-x86_64 new file mode 100755 index 000000000..51804566a Binary files /dev/null and b/tests/musllinux/musl-x86_64 differ diff --git a/tests/test_musllinux.py b/tests/test_musllinux.py new file mode 100644 index 000000000..a61d5d334 --- /dev/null +++ b/tests/test_musllinux.py @@ -0,0 +1,124 @@ +import collections +import io +import pathlib +import struct + +import pretend +import pytest + +from packaging import _musllinux +from packaging._musllinux import ( + _MuslVersion, + _get_musl_version, + _parse_musl_version, + _parse_ld_musl_from_elf, +) + + +MUSL_AMD64_122 = "musl libc (x86_64)\nVersion 1.2.2\n" +MUSL_I386_121 = "musl libc (i386)\nVersion 1.2.1\n" +MUSL_AARCH64_1124 = "musl libc (aarch64)\nVersion 1.1.24\n" +MUSL_INVALID = "musl libc (invalid)\n" +MUSL_UNKNOWN = "musl libc (unknown)\nVersion unknown\n" + +MUSL_DIR = pathlib.Path(__file__).with_name("musllinux").resolve() + +BINARY_GLIBC_X86_64 = MUSL_DIR.joinpath("glibc-x86_64") +BINARY_MUSL_X86_64 = MUSL_DIR.joinpath("musl-x86_64") +BINARY_MUSL_I386 = MUSL_DIR.joinpath("musl-i386") +BINARY_MUSL_AARCH64 = MUSL_DIR.joinpath("musl-aarch64") + + +@pytest.mark.parametrize( + "output, version", + [ + (MUSL_AMD64_122, _MuslVersion(1, 2)), + (MUSL_I386_121, _MuslVersion(1, 2)), + (MUSL_AARCH64_1124, _MuslVersion(1, 1)), + (MUSL_INVALID, None), + (MUSL_UNKNOWN, None), + ], + ids=["amd64-1.2.2", "i386-1.2.1", "aarch64-1.1.24", "invalid", "unknown"], +) +def test_parse_musl_version(output, version): + assert _parse_musl_version(output) == version + + +@pytest.mark.parametrize( + "executable, location", + [ + (BINARY_GLIBC_X86_64, None), + (BINARY_MUSL_X86_64, "/lib/ld-musl-x86_64.so.1"), + (BINARY_MUSL_I386, "/lib/ld-musl-i386.so.1"), + (BINARY_MUSL_AARCH64, "/lib/ld-musl-aarch64.so.1"), + ], + ids=["glibc", "x86_64", "i386", "aarch64"], +) +def test_parse_ld_musl_from_elf(executable, location): + with executable.open("rb") as f: + assert _parse_ld_musl_from_elf(f) == location + + +@pytest.mark.parametrize( + "data", + [ + # Too short for magic. + b"\0", + # Enough for magic, but not ELF. + b"#!/bin/bash" + b"\0" * 16, + # ELF, but unknown byte declaration. + b"\x7fELF\3" + b"\0" * 16, + ], + ids=["no-magic", "wrong-magic", "unknown-format"], +) +def test_parse_ld_musl_from_elf_invalid(data): + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "head", + [ + 25, # Enough for magic, but not the section definitions. + 58, # Enough for section definitions, but not the actual sections. + ], +) +def test_parse_ld_musl_from_elf_invalid_section(head): + data = BINARY_MUSL_X86_64.read_bytes()[:head] + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +def test_parse_ld_musl_from_elf_no_interpreter_section(): + with BINARY_MUSL_X86_64.open("rb") as f: + data = f.read() + + # Change all sections to *not* PT_INTERP. + unpacked = struct.unpack("16BHHIQQQIHHH", data[:58]) + *_, e_phoff, _, _, _, e_phentsize, e_phnum = unpacked + for i in range(e_phnum + 1): + sb = e_phoff + e_phentsize * i + se = sb + 56 + section = struct.unpack("IIQQQQQQ", data[sb:se]) + data = data[:sb] + struct.pack("IIQQQQQQ", 0, *section[1:]) + data[se:] + + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "executable, output, version", + [ + (MUSL_DIR.joinpath("does-not-exist"), "error", None), + (BINARY_GLIBC_X86_64, "error", None), + (BINARY_MUSL_X86_64, MUSL_AMD64_122, _MuslVersion(1, 2)), + (BINARY_MUSL_I386, MUSL_I386_121, _MuslVersion(1, 2)), + (BINARY_MUSL_AARCH64, MUSL_AARCH64_1124, _MuslVersion(1, 1)), + ], + ids=["does-not-exist", "glibc", "x86_64", "i386", "aarch64"], +) +def test_get_musl_version(monkeypatch, executable, output, version): + def mock_run(*args, **kwargs): + return collections.namedtuple("Proc", "stderr")(output) + + run_recorder = pretend.call_recorder(mock_run) + monkeypatch.setattr(_musllinux.subprocess, "run", run_recorder) + + assert _get_musl_version(str(executable)) == version