From 529145af139b87721b4129646e71dbce05d56c3e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 7 Apr 2021 08:29:52 +0800 Subject: [PATCH] Tests for parsing musl information --- packaging/_musllinux.py | 62 ++++++---------- tests/musllinux/build.sh | 61 +++++++++++++++ tests/musllinux/glibc-x86_64 | Bin 0 -> 1024 bytes tests/musllinux/musl-aarch64 | Bin 0 -> 1024 bytes tests/musllinux/musl-i386 | Bin 0 -> 1024 bytes tests/musllinux/musl-x86_64 | Bin 0 -> 1024 bytes tests/test_musllinux.py | 140 +++++++++++++++++++++++++++++++++++ 7 files changed, 223 insertions(+), 40 deletions(-) create mode 100644 tests/musllinux/build.sh create mode 100755 tests/musllinux/glibc-x86_64 create mode 100755 tests/musllinux/musl-aarch64 create mode 100755 tests/musllinux/musl-i386 create mode 100755 tests/musllinux/musl-x86_64 create mode 100644 tests/test_musllinux.py diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index d701124f..85450faf 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -4,11 +4,11 @@ linked against musl, and what musl version is used. """ +import contextlib import functools import operator import os import re -import shutil import struct import subprocess import sys @@ -19,10 +19,11 @@ def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: return struct.unpack(fmt, f.read(struct.calcsize(fmt))) -def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: +def _parse_ld_musl_from_elf(f: IO[bytes]) -> Optional[str]: """Detect musl libc location by parsing the Python executable. - Based on https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html """ f.seek(0) try: @@ -67,39 +68,21 @@ def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: return None -def _get_ld_musl_ldd(executable: str) -> Optional[str]: - ldd = shutil.which("ldd") - if not ldd: # No dynamic program loader. - return None - proc = subprocess.run( - [ldd, executable], stdout=subprocess.PIPE, universal_newlines=True - ) - if proc.returncode != 0: # Not a valid dynamic program. - return None - for line in proc.stdout.splitlines(keepends=False): - path = line.lstrip().rsplit(None, 1)[0] - if "musl" not in path: - continue - return path - return None - - -def _get_ld_musl(executable: str) -> Optional[str]: - try: - with open(executable, "rb") as f: - return _get_ld_musl_elf(f) - except IOError: - return _get_ld_musl_ldd(executable) - - -_version_pat = re.compile(r"Version (\d+)\.(\d+)") - - class _MuslVersion(NamedTuple): major: int minor: int +def _parse_musl_version(output: str) -> Optional[_MuslVersion]: + lines = [n for n in (n.strip() for n in output.splitlines()) if n] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + m = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if not m: + return None + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + + @functools.lru_cache() def _get_musl_version(executable: str) -> Optional[_MuslVersion]: """Detect currently-running musl runtime version. @@ -112,17 +95,16 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: Version 1.2.2 Dynamic Program Loader """ - ld = _get_ld_musl(executable) + with contextlib.ExitStack() as stack: + try: + f = stack.enter_context(open(executable, "rb")) + except IOError: + return None + ld = _parse_ld_musl_from_elf(f) if not ld: return None proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True) - lines = [n for n in (n.strip() for n in proc.stderr.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + return _parse_musl_version(proc.stderr) def platform_tags(arch: str) -> Iterator[str]: @@ -141,7 +123,7 @@ def platform_tags(arch: str) -> Iterator[str]: yield f"musllinux_{sys_musl.major}_{minor}_{arch}" -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import sysconfig plat = sysconfig.get_platform() diff --git a/tests/musllinux/build.sh b/tests/musllinux/build.sh new file mode 100644 index 00000000..acd2b94c --- /dev/null +++ b/tests/musllinux/build.sh @@ -0,0 +1,61 @@ +# Build helper binaries for musllinux tests. +# Usages: +# build.sh # Build everything. +# build.sh $DISTRO $ARCH # Build one executable in $ARCH using $DISTRO. +# +# Either invocation ultimately runs this script in a Docker container with +# `build.sh glibc|musl $ARCH` to actually build the executable. + +set -euo pipefail +set -x + +UBUNTU_VERSION='focal' +ALPINE_VERSION='v3.13' + +build_one_in_ubuntu () { + $1 "multiarch/ubuntu-core:${2}-${UBUNTU_VERSION}" \ + bash "/home/hello-world/musllinux/build.sh" glibc "glibc-${2}" +} + +build_one_in_alpine () { + $1 "multiarch/alpine:${2}-${ALPINE_VERSION}" \ + sh "/home/hello-world/musllinux/build.sh" musl "musl-${2}" +} + +build_in_container () { + local SOURCE="$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))" + DOCKER="docker run --rm -v ${SOURCE}:/home/hello-world" + + if [[ $# -ne 0 ]]; then + "build_one_in_${1}" "$DOCKER" "$2" + return + fi + + build_one_in_alpine "$DOCKER" x86_64 + build_one_in_alpine "$DOCKER" i386 + build_one_in_alpine "$DOCKER" aarch64 + build_one_in_ubuntu "$DOCKER" x86_64 +} + +if [[ $# -eq 0 ]]; then + build_in_container + exit 0 +elif [[ "$1" == "glibc" ]]; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq \ + && apt-get install -qqy --no-install-recommends gcc libc6-dev +elif [[ "$1" == "musl" ]]; then + apk add -q build-base +else + build_in_container "$@" + exit 0 +fi + +build () { + local CFLAGS="" + local OUT="/home/hello-world/musllinux/${2}" + gcc -Os ${CFLAGS} -o "${OUT}-full" "/home/hello-world/hello-world.c" + head -c1024 "${OUT}-full" > "$OUT" + rm -f "${OUT}-full" +} + +build "$@" diff --git a/tests/musllinux/glibc-x86_64 b/tests/musllinux/glibc-x86_64 new file mode 100755 index 0000000000000000000000000000000000000000..59996e28c27516a7342a0f0b8052e0205455b65a GIT binary patch literal 1024 zcmbV~ze)o^5XL8oij5G&LP2ypt4lz_c^1kQ{sTpfLA+8V7J*X>BV26M&c@C+$O{NI z!OAjN2zL4a*4o%R-|kE>M-IV(+uzR2emi@!d%3!`me1u#tw75J@2cn`p(Jh=Wx@z4 zpfMh2X@*AFAF?r*4;?t`>@DBVQHd9r&`n}8fYUBaNl47wO*BVasi&C4ukxmD1U1Y2H@GxljXdYJGc;c$+80q>c72Cd5bne)DsG z>f)jOzR<2UTglyH>-kpqKuxTHL9b5&zP<;nasPjfu*`{Gk6w;#AY-U<+ziOgacrEw EAGPB?jsO4v literal 0 HcmV?d00001 diff --git a/tests/musllinux/musl-aarch64 b/tests/musllinux/musl-aarch64 new file mode 100755 index 0000000000000000000000000000000000000000..f6bcd380bb18d6c8f25d52af97d79020eba4e569 GIT binary patch literal 1024 zcmaJ<%}T>S5T2T9i_lVvCodK~w?!!8F(3*;3o1Q$TenT3fi#sQf_TtAfZjY5pP)x? z_UJqK44yoQZ{SRJ21}c|ko|VP*>84!@^IKbTFK==mIZG>mc3>en;oMaJ|Nhz4@KN- zumyB2pK``;8$hpF?;KjN$h?I}I#^+14mX(=lne=1*_Q>6{D2^&)tH@fos^7fQZf+* zFZlhjJNjGKPly!nqm!Q;l@|ieP-)q6VH;*ktgJ9YV+v6LxtUE{NA$q z&bcl@>1Q+Wd(FV94{qY1u65ME+G{o9VPlu$l58D6x}WAX!b9H95l3tK^i%%&_%0hO zd$*tf;fPCIs4mAvO5lej-zkYE zqmq*;a1vAWLX?p}Brr$B0T(*NBs4(}(;h(mv5y3TzF=RjA%}P!5+R0zdC%7%0(21) z(fnRPq_^y21PM8SeX}M?NxBHE__%^5=(EPoqgzjF@B++Yj-W^E!Uz2t%xBws@T2=D zu(t$86zlAM*4u%UYxz8(`%X3KQt&^B9cLsSDw zl&W=fhPq`Q8<04EO*yV^yQ-<1Euk)Wx6uhNLn+%hFdf&hRm0Ys2Ff5Jah0C}lM}H1#|HttScG-Q~ literal 0 HcmV?d00001 diff --git a/tests/musllinux/musl-x86_64 b/tests/musllinux/musl-x86_64 new file mode 100755 index 0000000000000000000000000000000000000000..d70261b717941b6388c6f75d0d64d7a54e24eae9 GIT binary patch literal 1024 zcma)4%Syvg5S_NZl@fv*ck!`pT0u%zB`$QM3+v9+H6|@wNNPzU6hT@)zz=ZmPxupp z_zkXHs5_U=Br{TD0uJ1q%$ajKccz&f93NII6_BVw7buI3jB%u}FE0QNbYKOan_$5L z(sOtmF6XgT%?g4`tg9iCZ^ZRHZm>>JZb(SJvasf}iXiYDEnp9_XC=ijPwAwb<8UpWu{zg0jz(*Tkp2tXcTL$m32%gCJ=s zdg!Bm{|#B+DfcUz?}P6t$SE`febEg3MtBtmjd7=~+Ix0RYHzoWw@I6oS9wq}&1I$1vZ|u^m8|dpkrnqBI#fcjwDs=S_D7XJ? JTq(aFeFG$;NBaN( literal 0 HcmV?d00001 diff --git a/tests/test_musllinux.py b/tests/test_musllinux.py new file mode 100644 index 00000000..0edd73e2 --- /dev/null +++ b/tests/test_musllinux.py @@ -0,0 +1,140 @@ +import collections +import io +import pathlib +import struct +import subprocess + +import pretend +import pytest + +from packaging import _musllinux +from packaging._musllinux import ( + _get_musl_version, + _MuslVersion, + _parse_ld_musl_from_elf, + _parse_musl_version, +) + +MUSL_AMD64 = "musl libc (x86_64)\nVersion 1.2.2\n" +MUSL_I386 = "musl libc (i386)\nVersion 1.2.1\n" +MUSL_AARCH64 = "musl libc (aarch64)\nVersion 1.1.24\n" +MUSL_INVALID = "musl libc (invalid)\n" +MUSL_UNKNOWN = "musl libc (unknown)\nVersion unknown\n" + +MUSL_DIR = pathlib.Path(__file__).with_name("musllinux").resolve() + +BIN_GLIBC_X86_64 = MUSL_DIR.joinpath("glibc-x86_64") +BIN_MUSL_X86_64 = MUSL_DIR.joinpath("musl-x86_64") +BIN_MUSL_I386 = MUSL_DIR.joinpath("musl-i386") +BIN_MUSL_AARCH64 = MUSL_DIR.joinpath("musl-aarch64") + +LD_MUSL_X86_64 = "/lib/ld-musl-x86_64.so.1" +LD_MUSL_I386 = "/lib/ld-musl-i386.so.1" +LD_MUSL_AARCH64 = "/lib/ld-musl-aarch64.so.1" + + +@pytest.mark.parametrize( + "output, version", + [ + (MUSL_AMD64, _MuslVersion(1, 2)), + (MUSL_I386, _MuslVersion(1, 2)), + (MUSL_AARCH64, _MuslVersion(1, 1)), + (MUSL_INVALID, None), + (MUSL_UNKNOWN, None), + ], + ids=["amd64-1.2.2", "i386-1.2.1", "aarch64-1.1.24", "invalid", "unknown"], +) +def test_parse_musl_version(output, version): + assert _parse_musl_version(output) == version + + +@pytest.mark.parametrize( + "executable, location", + [ + (BIN_GLIBC_X86_64, None), + (BIN_MUSL_X86_64, LD_MUSL_X86_64), + (BIN_MUSL_I386, LD_MUSL_I386), + (BIN_MUSL_AARCH64, LD_MUSL_AARCH64), + ], + ids=["glibc", "x86_64", "i386", "aarch64"], +) +def test_parse_ld_musl_from_elf(executable, location): + with executable.open("rb") as f: + assert _parse_ld_musl_from_elf(f) == location + + +@pytest.mark.parametrize( + "data", + [ + # Too short for magic. + b"\0", + # Enough for magic, but not ELF. + b"#!/bin/bash" + b"\0" * 16, + # ELF, but unknown byte declaration. + b"\x7fELF\3" + b"\0" * 16, + ], + ids=["no-magic", "wrong-magic", "unknown-format"], +) +def test_parse_ld_musl_from_elf_invalid(data): + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "head", + [ + 25, # Enough for magic, but not the section definitions. + 58, # Enough for section definitions, but not the actual sections. + ], +) +def test_parse_ld_musl_from_elf_invalid_section(head): + data = BIN_MUSL_X86_64.read_bytes()[:head] + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +def test_parse_ld_musl_from_elf_no_interpreter_section(): + with BIN_MUSL_X86_64.open("rb") as f: + data = f.read() + + # Change all sections to *not* PT_INTERP. + unpacked = struct.unpack("16BHHIQQQIHHH", data[:58]) + *_, e_phoff, _, _, _, e_phentsize, e_phnum = unpacked + for i in range(e_phnum + 1): + sb = e_phoff + e_phentsize * i + se = sb + 56 + section = struct.unpack("IIQQQQQQ", data[sb:se]) + data = data[:sb] + struct.pack("IIQQQQQQ", 0, *section[1:]) + data[se:] + + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "executable, output, version, ld_musl", + [ + (MUSL_DIR.joinpath("does-not-exist"), "error", None, None), + (BIN_GLIBC_X86_64, "error", None, None), + (BIN_MUSL_X86_64, MUSL_AMD64, _MuslVersion(1, 2), LD_MUSL_X86_64), + (BIN_MUSL_I386, MUSL_I386, _MuslVersion(1, 2), LD_MUSL_I386), + (BIN_MUSL_AARCH64, MUSL_AARCH64, _MuslVersion(1, 1), LD_MUSL_AARCH64), + ], + ids=["does-not-exist", "glibc", "x86_64", "i386", "aarch64"], +) +def test_get_musl_version(monkeypatch, executable, output, version, ld_musl): + def mock_run(*args, **kwargs): + return collections.namedtuple("Proc", "stderr")(output) + + run_recorder = pretend.call_recorder(mock_run) + monkeypatch.setattr(_musllinux.subprocess, "run", run_recorder) + + assert _get_musl_version(str(executable)) == version + + if ld_musl is not None: + expected_calls = [ + pretend.call( + [ld_musl], + stderr=subprocess.PIPE, + universal_newlines=True, + ) + ] + else: + expected_calls = [] + assert run_recorder.calls == expected_calls