From 6112e4c12ee145a9d2bd09521e926411898baf2f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 18 Mar 2021 18:33:06 +0800 Subject: [PATCH 1/7] Implement musllinux tag generation Tests coming soon... --- packaging/_musllinux.py | 75 +++++++++++++++++++++++++++++++++++++++++ packaging/tags.py | 3 ++ 2 files changed, 78 insertions(+) create mode 100644 packaging/_musllinux.py diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py new file mode 100644 index 00000000..29108b68 --- /dev/null +++ b/packaging/_musllinux.py @@ -0,0 +1,75 @@ +"""PEP 656 support. + +This module implements logic to detect if the currently running Python is +linked against musl, and what musl version is used. +""" + +import functools +import re +import shutil +import subprocess +import sys +from typing import Iterator, NamedTuple, Optional + + +class _MuslVersion(NamedTuple): + major: int + minor: int + + +def _get_ld_musl(executable: str) -> Optional[str]: + ldd = shutil.which("ldd") + if not ldd: # No dynamic program loader. + return None + proc = subprocess.run( + [ldd, executable], stdout=subprocess.PIPE, universal_newlines=True + ) + if proc.returncode != 0: # Not a valid dynamic program. + return None + ld_musl_pat = re.compile(r"^.+/ld-musl-.+$") + for line in proc.stdout.splitlines(): + m = ld_musl_pat.match(line) + if not m: + continue + return m.string.strip().rsplit(None, 1)[0] + return None # Musl ldd path not found -- program not linked against musl. + + +_version_pat = re.compile(r"^Version (\d+)\.(\d+)", flags=re.MULTILINE) + + +@functools.lru_cache() +def _get_musl_version(executable: str) -> Optional[_MuslVersion]: + """Detect currently-running musl runtime version. + + This is done by checking the specified executable's dynamic linking + information, and invoking the loader to parse its output for a version + string. If the loader is musl, the output would be something like:: + + musl libc (x86_64) + Version 1.2.2 + Dynamic Program Loader + """ + ld_musl = _get_ld_musl(executable) + if not ld_musl: + return None + proc = subprocess.run([ld_musl], stderr=subprocess.PIPE, universal_newlines=True) + for m in _version_pat.finditer(proc.stderr): + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + return None + + +def platform_tags(arch: str) -> Iterator[str]: + """Generate musllinux tags compatible to the current platform. + + :param arch: Should be the part of platform tag after the ``linux_`` + prefix, e.g. ``x86_64``. The ``linux_`` prefix is assumed as a + prerequisite for the current platform to be musllinux-compatible. + + :returns: An iterator of compatible musllinux tags. + """ + sys_musl = _get_musl_version(sys.executable) + if sys_musl is None: # Python not dynamically linked against musl. + return + for minor in range(sys_musl.minor, -1, -1): + yield f"musllinux_{sys_musl.major}_{minor}_{arch}" diff --git a/packaging/tags.py b/packaging/tags.py index 2c10a23b..9cc2c5e1 100644 --- a/packaging/tags.py +++ b/packaging/tags.py @@ -26,6 +26,8 @@ cast, ) +from . import _musllinux + logger = logging.getLogger(__name__) PythonVersion = Sequence[int] @@ -723,6 +725,7 @@ def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: _, arch = linux.split("_", 1) if _have_compatible_manylinux_abi(arch): yield from _manylinux_tags(linux, arch) + yield from _musllinux.platform_tags(arch) yield linux From 01046b3c2e127e83a9187bbc61bbff347465faa5 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 19 Mar 2021 17:41:05 +0800 Subject: [PATCH 2/7] Parse musl libc path from header This is probably slightly faster and more robust than the ldd method since it does not require ldd on PATH, and avoids some terminal encoding issues. The ldd approach is kept in edge cases where the executable is somehow not readable. I suspect ldd would not be very useful in this scenario either, but there's not harm being safe? --- packaging/_musllinux.py | 78 +++++++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 11 deletions(-) diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index 29108b68..1cb67a65 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -5,19 +5,63 @@ """ import functools +import operator +import os import re import shutil +import struct import subprocess import sys -from typing import Iterator, NamedTuple, Optional +from typing import IO, Iterator, NamedTuple, Optional, Tuple -class _MuslVersion(NamedTuple): - major: int - minor: int +def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: + return struct.unpack(fmt, f.read(struct.calcsize(fmt))) -def _get_ld_musl(executable: str) -> Optional[str]: +def _get_ld_musl_ctypes(f: IO[bytes]) -> Optional[str]: + """Detect musl libc location by parsing the Python executable. + + Based on https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + """ + f.seek(0) + try: + ident = _read_unpacked(f, "16B") + except struct.error: + return None + if ident[:4] != tuple(b"\x7fELF"): # Invalid magic, not ELF. + return None + f.seek(struct.calcsize("HHI"), 1) # Skip file type, machine, and version. + + try: + # e_fmt: Format for program header. + # p_fmt: Format for section header. + # p_idx: Indexes to find p_type, p_offset, and p_filesz. + e_fmt, p_fmt, p_idx = { + 1: ("IIIIHHH", "IIIIIIII", (0, 1, 4)), # 32-bit. + 2: ("QQQIHHH", "IIQQQQQQ", (0, 2, 5)), # 64-bit. + }[ident[4]] + except KeyError: + return None + else: + p_get = operator.itemgetter(*p_idx) + + # Find the interpreter section and return its content. + _, e_phoff, _, _, _, e_phentsize, e_phnum = _read_unpacked(f, e_fmt) + for i in range(e_phnum + 1): + f.seek(e_phoff + e_phentsize * i) + p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt)) + if p_type != 3: + continue + f.seek(p_offset) + interpreter = os.fsdecode(f.read(p_filesz)).strip("\0") + if "musl" not in interpreter: + return None + return interpreter + return None + + +def _get_ld_musl_ldd(executable: str) -> Optional[str]: ldd = shutil.which("ldd") if not ldd: # No dynamic program loader. return None @@ -26,18 +70,30 @@ def _get_ld_musl(executable: str) -> Optional[str]: ) if proc.returncode != 0: # Not a valid dynamic program. return None - ld_musl_pat = re.compile(r"^.+/ld-musl-.+$") - for line in proc.stdout.splitlines(): - m = ld_musl_pat.match(line) - if not m: + for line in proc.stdout.splitlines(keepends=False): + path = line.lstrip().rsplit(None, 1)[0] + if "musl" not in path: continue - return m.string.strip().rsplit(None, 1)[0] - return None # Musl ldd path not found -- program not linked against musl. + return path + return None + + +def _get_ld_musl(executable: str) -> Optional[str]: + try: + with open(executable, "rb") as f: + return _get_ld_musl_ctypes(f) + except IOError: + return _get_ld_musl_ldd(executable) _version_pat = re.compile(r"^Version (\d+)\.(\d+)", flags=re.MULTILINE) +class _MuslVersion(NamedTuple): + major: int + minor: int + + @functools.lru_cache() def _get_musl_version(executable: str) -> Optional[_MuslVersion]: """Detect currently-running musl runtime version. From 5fea775762580d697694ba3a1f0f97d818201afe Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 19 Mar 2021 18:03:57 +0800 Subject: [PATCH 3/7] Catch struct unpacking errors --- packaging/_musllinux.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index 1cb67a65..a31cfe97 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -47,10 +47,16 @@ def _get_ld_musl_ctypes(f: IO[bytes]) -> Optional[str]: p_get = operator.itemgetter(*p_idx) # Find the interpreter section and return its content. - _, e_phoff, _, _, _, e_phentsize, e_phnum = _read_unpacked(f, e_fmt) + try: + _, e_phoff, _, _, _, e_phentsize, e_phnum = _read_unpacked(f, e_fmt) + except struct.error: + return None for i in range(e_phnum + 1): f.seek(e_phoff + e_phentsize * i) - p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt)) + try: + p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt)) + except struct.error: + return None if p_type != 3: continue f.seek(p_offset) From 818715ee6b0cf5eb6e02c747082c7f5c7082f8ff Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 19 Mar 2021 21:22:51 +0800 Subject: [PATCH 4/7] Cehck the target ld so is actually musl --- packaging/_musllinux.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index a31cfe97..0a5a6b41 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -57,7 +57,7 @@ def _get_ld_musl_ctypes(f: IO[bytes]) -> Optional[str]: p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt)) except struct.error: return None - if p_type != 3: + if p_type != 3: # Not PT_INTERP. continue f.seek(p_offset) interpreter = os.fsdecode(f.read(p_filesz)).strip("\0") @@ -92,7 +92,7 @@ def _get_ld_musl(executable: str) -> Optional[str]: return _get_ld_musl_ldd(executable) -_version_pat = re.compile(r"^Version (\d+)\.(\d+)", flags=re.MULTILINE) +_version_pat = re.compile(r"Version (\d+)\.(\d+)") class _MuslVersion(NamedTuple): @@ -112,13 +112,17 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: Version 1.2.2 Dynamic Program Loader """ - ld_musl = _get_ld_musl(executable) - if not ld_musl: + ld = _get_ld_musl(executable) + if not ld: return None - proc = subprocess.run([ld_musl], stderr=subprocess.PIPE, universal_newlines=True) - for m in _version_pat.finditer(proc.stderr): - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) - return None + proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True) + lines = [n for n in (n.strip() for n in proc.stderr.splitlines()) if n] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + m = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if not m: + return None + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) def platform_tags(arch: str) -> Iterator[str]: From 3836079a745e93eee698d2d20128e968e9c535ae Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 20 Mar 2021 03:47:39 +0800 Subject: [PATCH 5/7] Change internal function name --- packaging/_musllinux.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index 0a5a6b41..d701124f 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -19,7 +19,7 @@ def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: return struct.unpack(fmt, f.read(struct.calcsize(fmt))) -def _get_ld_musl_ctypes(f: IO[bytes]) -> Optional[str]: +def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: """Detect musl libc location by parsing the Python executable. Based on https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca @@ -87,7 +87,7 @@ def _get_ld_musl_ldd(executable: str) -> Optional[str]: def _get_ld_musl(executable: str) -> Optional[str]: try: with open(executable, "rb") as f: - return _get_ld_musl_ctypes(f) + return _get_ld_musl_elf(f) except IOError: return _get_ld_musl_ldd(executable) @@ -139,3 +139,16 @@ def platform_tags(arch: str) -> Iterator[str]: return for minor in range(sys_musl.minor, -1, -1): yield f"musllinux_{sys_musl.major}_{minor}_{arch}" + + +if __name__ == "__main__": + import sysconfig + + plat = sysconfig.get_platform() + assert plat.startswith("linux-"), "not linux" + + print("plat:", plat) + print("musl:", _get_musl_version(sys.executable)) + print("tags:", end=" ") + for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])): + print(t, end="\n ") From 529145af139b87721b4129646e71dbce05d56c3e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 7 Apr 2021 08:29:52 +0800 Subject: [PATCH 6/7] Tests for parsing musl information --- packaging/_musllinux.py | 62 ++++++---------- tests/musllinux/build.sh | 61 +++++++++++++++ tests/musllinux/glibc-x86_64 | Bin 0 -> 1024 bytes tests/musllinux/musl-aarch64 | Bin 0 -> 1024 bytes tests/musllinux/musl-i386 | Bin 0 -> 1024 bytes tests/musllinux/musl-x86_64 | Bin 0 -> 1024 bytes tests/test_musllinux.py | 140 +++++++++++++++++++++++++++++++++++ 7 files changed, 223 insertions(+), 40 deletions(-) create mode 100644 tests/musllinux/build.sh create mode 100755 tests/musllinux/glibc-x86_64 create mode 100755 tests/musllinux/musl-aarch64 create mode 100755 tests/musllinux/musl-i386 create mode 100755 tests/musllinux/musl-x86_64 create mode 100644 tests/test_musllinux.py diff --git a/packaging/_musllinux.py b/packaging/_musllinux.py index d701124f..85450faf 100644 --- a/packaging/_musllinux.py +++ b/packaging/_musllinux.py @@ -4,11 +4,11 @@ linked against musl, and what musl version is used. """ +import contextlib import functools import operator import os import re -import shutil import struct import subprocess import sys @@ -19,10 +19,11 @@ def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]: return struct.unpack(fmt, f.read(struct.calcsize(fmt))) -def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: +def _parse_ld_musl_from_elf(f: IO[bytes]) -> Optional[str]: """Detect musl libc location by parsing the Python executable. - Based on https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca + ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html """ f.seek(0) try: @@ -67,39 +68,21 @@ def _get_ld_musl_elf(f: IO[bytes]) -> Optional[str]: return None -def _get_ld_musl_ldd(executable: str) -> Optional[str]: - ldd = shutil.which("ldd") - if not ldd: # No dynamic program loader. - return None - proc = subprocess.run( - [ldd, executable], stdout=subprocess.PIPE, universal_newlines=True - ) - if proc.returncode != 0: # Not a valid dynamic program. - return None - for line in proc.stdout.splitlines(keepends=False): - path = line.lstrip().rsplit(None, 1)[0] - if "musl" not in path: - continue - return path - return None - - -def _get_ld_musl(executable: str) -> Optional[str]: - try: - with open(executable, "rb") as f: - return _get_ld_musl_elf(f) - except IOError: - return _get_ld_musl_ldd(executable) - - -_version_pat = re.compile(r"Version (\d+)\.(\d+)") - - class _MuslVersion(NamedTuple): major: int minor: int +def _parse_musl_version(output: str) -> Optional[_MuslVersion]: + lines = [n for n in (n.strip() for n in output.splitlines()) if n] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + m = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if not m: + return None + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + + @functools.lru_cache() def _get_musl_version(executable: str) -> Optional[_MuslVersion]: """Detect currently-running musl runtime version. @@ -112,17 +95,16 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: Version 1.2.2 Dynamic Program Loader """ - ld = _get_ld_musl(executable) + with contextlib.ExitStack() as stack: + try: + f = stack.enter_context(open(executable, "rb")) + except IOError: + return None + ld = _parse_ld_musl_from_elf(f) if not ld: return None proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True) - lines = [n for n in (n.strip() for n in proc.stderr.splitlines()) if n] - if len(lines) < 2 or lines[0][:4] != "musl": - return None - m = re.match(r"Version (\d+)\.(\d+)", lines[1]) - if not m: - return None - return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + return _parse_musl_version(proc.stderr) def platform_tags(arch: str) -> Iterator[str]: @@ -141,7 +123,7 @@ def platform_tags(arch: str) -> Iterator[str]: yield f"musllinux_{sys_musl.major}_{minor}_{arch}" -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import sysconfig plat = sysconfig.get_platform() diff --git a/tests/musllinux/build.sh b/tests/musllinux/build.sh new file mode 100644 index 00000000..acd2b94c --- /dev/null +++ b/tests/musllinux/build.sh @@ -0,0 +1,61 @@ +# Build helper binaries for musllinux tests. +# Usages: +# build.sh # Build everything. +# build.sh $DISTRO $ARCH # Build one executable in $ARCH using $DISTRO. +# +# Either invocation ultimately runs this script in a Docker container with +# `build.sh glibc|musl $ARCH` to actually build the executable. + +set -euo pipefail +set -x + +UBUNTU_VERSION='focal' +ALPINE_VERSION='v3.13' + +build_one_in_ubuntu () { + $1 "multiarch/ubuntu-core:${2}-${UBUNTU_VERSION}" \ + bash "/home/hello-world/musllinux/build.sh" glibc "glibc-${2}" +} + +build_one_in_alpine () { + $1 "multiarch/alpine:${2}-${ALPINE_VERSION}" \ + sh "/home/hello-world/musllinux/build.sh" musl "musl-${2}" +} + +build_in_container () { + local SOURCE="$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))" + DOCKER="docker run --rm -v ${SOURCE}:/home/hello-world" + + if [[ $# -ne 0 ]]; then + "build_one_in_${1}" "$DOCKER" "$2" + return + fi + + build_one_in_alpine "$DOCKER" x86_64 + build_one_in_alpine "$DOCKER" i386 + build_one_in_alpine "$DOCKER" aarch64 + build_one_in_ubuntu "$DOCKER" x86_64 +} + +if [[ $# -eq 0 ]]; then + build_in_container + exit 0 +elif [[ "$1" == "glibc" ]]; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq \ + && apt-get install -qqy --no-install-recommends gcc libc6-dev +elif [[ "$1" == "musl" ]]; then + apk add -q build-base +else + build_in_container "$@" + exit 0 +fi + +build () { + local CFLAGS="" + local OUT="/home/hello-world/musllinux/${2}" + gcc -Os ${CFLAGS} -o "${OUT}-full" "/home/hello-world/hello-world.c" + head -c1024 "${OUT}-full" > "$OUT" + rm -f "${OUT}-full" +} + +build "$@" diff --git a/tests/musllinux/glibc-x86_64 b/tests/musllinux/glibc-x86_64 new file mode 100755 index 0000000000000000000000000000000000000000..59996e28c27516a7342a0f0b8052e0205455b65a GIT binary patch literal 1024 zcmbV~ze)o^5XL8oij5G&LP2ypt4lz_c^1kQ{sTpfLA+8V7J*X>BV26M&c@C+$O{NI z!OAjN2zL4a*4o%R-|kE>M-IV(+uzR2emi@!d%3!`me1u#tw75J@2cn`p(Jh=Wx@z4 zpfMh2X@*AFAF?r*4;?t`>@DBVQHd9r&`n}8fYUBaNl47wO*BVasi&C4ukxmD1U1Y2H@GxljXdYJGc;c$+80q>c72Cd5bne)DsG z>f)jOzR<2UTglyH>-kpqKuxTHL9b5&zP<;nasPjfu*`{Gk6w;#AY-U<+ziOgacrEw EAGPB?jsO4v literal 0 HcmV?d00001 diff --git a/tests/musllinux/musl-aarch64 b/tests/musllinux/musl-aarch64 new file mode 100755 index 0000000000000000000000000000000000000000..f6bcd380bb18d6c8f25d52af97d79020eba4e569 GIT binary patch literal 1024 zcmaJ<%}T>S5T2T9i_lVvCodK~w?!!8F(3*;3o1Q$TenT3fi#sQf_TtAfZjY5pP)x? z_UJqK44yoQZ{SRJ21}c|ko|VP*>84!@^IKbTFK==mIZG>mc3>en;oMaJ|Nhz4@KN- zumyB2pK``;8$hpF?;KjN$h?I}I#^+14mX(=lne=1*_Q>6{D2^&)tH@fos^7fQZf+* zFZlhjJNjGKPly!nqm!Q;l@|ieP-)q6VH;*ktgJ9YV+v6LxtUE{NA$q z&bcl@>1Q+Wd(FV94{qY1u65ME+G{o9VPlu$l58D6x}WAX!b9H95l3tK^i%%&_%0hO zd$*tf;fPCIs4mAvO5lej-zkYE zqmq*;a1vAWLX?p}Brr$B0T(*NBs4(}(;h(mv5y3TzF=RjA%}P!5+R0zdC%7%0(21) z(fnRPq_^y21PM8SeX}M?NxBHE__%^5=(EPoqgzjF@B++Yj-W^E!Uz2t%xBws@T2=D zu(t$86zlAM*4u%UYxz8(`%X3KQt&^B9cLsSDw zl&W=fhPq`Q8<04EO*yV^yQ-<1Euk)Wx6uhNLn+%hFdf&hRm0Ys2Ff5Jah0C}lM}H1#|HttScG-Q~ literal 0 HcmV?d00001 diff --git a/tests/musllinux/musl-x86_64 b/tests/musllinux/musl-x86_64 new file mode 100755 index 0000000000000000000000000000000000000000..d70261b717941b6388c6f75d0d64d7a54e24eae9 GIT binary patch literal 1024 zcma)4%Syvg5S_NZl@fv*ck!`pT0u%zB`$QM3+v9+H6|@wNNPzU6hT@)zz=ZmPxupp z_zkXHs5_U=Br{TD0uJ1q%$ajKccz&f93NII6_BVw7buI3jB%u}FE0QNbYKOan_$5L z(sOtmF6XgT%?g4`tg9iCZ^ZRHZm>>JZb(SJvasf}iXiYDEnp9_XC=ijPwAwb<8UpWu{zg0jz(*Tkp2tXcTL$m32%gCJ=s zdg!Bm{|#B+DfcUz?}P6t$SE`febEg3MtBtmjd7=~+Ix0RYHzoWw@I6oS9wq}&1I$1vZ|u^m8|dpkrnqBI#fcjwDs=S_D7XJ? JTq(aFeFG$;NBaN( literal 0 HcmV?d00001 diff --git a/tests/test_musllinux.py b/tests/test_musllinux.py new file mode 100644 index 00000000..0edd73e2 --- /dev/null +++ b/tests/test_musllinux.py @@ -0,0 +1,140 @@ +import collections +import io +import pathlib +import struct +import subprocess + +import pretend +import pytest + +from packaging import _musllinux +from packaging._musllinux import ( + _get_musl_version, + _MuslVersion, + _parse_ld_musl_from_elf, + _parse_musl_version, +) + +MUSL_AMD64 = "musl libc (x86_64)\nVersion 1.2.2\n" +MUSL_I386 = "musl libc (i386)\nVersion 1.2.1\n" +MUSL_AARCH64 = "musl libc (aarch64)\nVersion 1.1.24\n" +MUSL_INVALID = "musl libc (invalid)\n" +MUSL_UNKNOWN = "musl libc (unknown)\nVersion unknown\n" + +MUSL_DIR = pathlib.Path(__file__).with_name("musllinux").resolve() + +BIN_GLIBC_X86_64 = MUSL_DIR.joinpath("glibc-x86_64") +BIN_MUSL_X86_64 = MUSL_DIR.joinpath("musl-x86_64") +BIN_MUSL_I386 = MUSL_DIR.joinpath("musl-i386") +BIN_MUSL_AARCH64 = MUSL_DIR.joinpath("musl-aarch64") + +LD_MUSL_X86_64 = "/lib/ld-musl-x86_64.so.1" +LD_MUSL_I386 = "/lib/ld-musl-i386.so.1" +LD_MUSL_AARCH64 = "/lib/ld-musl-aarch64.so.1" + + +@pytest.mark.parametrize( + "output, version", + [ + (MUSL_AMD64, _MuslVersion(1, 2)), + (MUSL_I386, _MuslVersion(1, 2)), + (MUSL_AARCH64, _MuslVersion(1, 1)), + (MUSL_INVALID, None), + (MUSL_UNKNOWN, None), + ], + ids=["amd64-1.2.2", "i386-1.2.1", "aarch64-1.1.24", "invalid", "unknown"], +) +def test_parse_musl_version(output, version): + assert _parse_musl_version(output) == version + + +@pytest.mark.parametrize( + "executable, location", + [ + (BIN_GLIBC_X86_64, None), + (BIN_MUSL_X86_64, LD_MUSL_X86_64), + (BIN_MUSL_I386, LD_MUSL_I386), + (BIN_MUSL_AARCH64, LD_MUSL_AARCH64), + ], + ids=["glibc", "x86_64", "i386", "aarch64"], +) +def test_parse_ld_musl_from_elf(executable, location): + with executable.open("rb") as f: + assert _parse_ld_musl_from_elf(f) == location + + +@pytest.mark.parametrize( + "data", + [ + # Too short for magic. + b"\0", + # Enough for magic, but not ELF. + b"#!/bin/bash" + b"\0" * 16, + # ELF, but unknown byte declaration. + b"\x7fELF\3" + b"\0" * 16, + ], + ids=["no-magic", "wrong-magic", "unknown-format"], +) +def test_parse_ld_musl_from_elf_invalid(data): + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "head", + [ + 25, # Enough for magic, but not the section definitions. + 58, # Enough for section definitions, but not the actual sections. + ], +) +def test_parse_ld_musl_from_elf_invalid_section(head): + data = BIN_MUSL_X86_64.read_bytes()[:head] + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +def test_parse_ld_musl_from_elf_no_interpreter_section(): + with BIN_MUSL_X86_64.open("rb") as f: + data = f.read() + + # Change all sections to *not* PT_INTERP. + unpacked = struct.unpack("16BHHIQQQIHHH", data[:58]) + *_, e_phoff, _, _, _, e_phentsize, e_phnum = unpacked + for i in range(e_phnum + 1): + sb = e_phoff + e_phentsize * i + se = sb + 56 + section = struct.unpack("IIQQQQQQ", data[sb:se]) + data = data[:sb] + struct.pack("IIQQQQQQ", 0, *section[1:]) + data[se:] + + assert _parse_ld_musl_from_elf(io.BytesIO(data)) is None + + +@pytest.mark.parametrize( + "executable, output, version, ld_musl", + [ + (MUSL_DIR.joinpath("does-not-exist"), "error", None, None), + (BIN_GLIBC_X86_64, "error", None, None), + (BIN_MUSL_X86_64, MUSL_AMD64, _MuslVersion(1, 2), LD_MUSL_X86_64), + (BIN_MUSL_I386, MUSL_I386, _MuslVersion(1, 2), LD_MUSL_I386), + (BIN_MUSL_AARCH64, MUSL_AARCH64, _MuslVersion(1, 1), LD_MUSL_AARCH64), + ], + ids=["does-not-exist", "glibc", "x86_64", "i386", "aarch64"], +) +def test_get_musl_version(monkeypatch, executable, output, version, ld_musl): + def mock_run(*args, **kwargs): + return collections.namedtuple("Proc", "stderr")(output) + + run_recorder = pretend.call_recorder(mock_run) + monkeypatch.setattr(_musllinux.subprocess, "run", run_recorder) + + assert _get_musl_version(str(executable)) == version + + if ld_musl is not None: + expected_calls = [ + pretend.call( + [ld_musl], + stderr=subprocess.PIPE, + universal_newlines=True, + ) + ] + else: + expected_calls = [] + assert run_recorder.calls == expected_calls From 3c8a5dcfc2df7fdd32c7ec321753862a5f00dcbd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 7 Apr 2021 09:00:47 +0800 Subject: [PATCH 7/7] Test musllinux integration into packaging.tags --- tests/test_tags.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_tags.py b/tests/test_tags.py index 6c087f59..269b4003 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -10,6 +10,7 @@ except ImportError: ctypes = None import os +import pathlib import platform import sys import sysconfig @@ -592,6 +593,44 @@ def test_linux_platforms_manylinux_glibc3(self, monkeypatch): ) assert platforms == expected + @pytest.mark.parametrize( + "native_arch, cross32_arch, musl_version", + [ + ("aarch64", "armv7l", (1, 1)), + ("i386", "i386", (1, 2)), + ("x86_64", "i686", (1, 2)), + ], + ) + @pytest.mark.parametrize("cross32", [True, False], ids=["cross", "native"]) + def test_linux_platforms_musllinux( + self, monkeypatch, native_arch, cross32_arch, musl_version, cross32 + ): + fake_executable = str( + pathlib.Path(__file__) + .parent.joinpath("musllinux", f"musl-{native_arch}") + .resolve() + ) + monkeypatch.setattr(tags._musllinux.sys, "executable", fake_executable) + monkeypatch.setattr(sysconfig, "get_platform", lambda: f"linux_{native_arch}") + monkeypatch.setattr( + tags, + "_is_manylinux_compatible", + lambda *_: False, + ) + monkeypatch.setattr( + tags, + "_have_compatible_manylinux_abi", + lambda *_: False, + ) + + platforms = list(tags._linux_platforms(is_32bit=cross32)) + target_arch = cross32_arch if cross32 else native_arch + expected = [ + f"musllinux_{musl_version[0]}_{minor}_{target_arch}" + for minor in range(musl_version[1], -1, -1) + ] + [f"linux_{target_arch}"] + assert platforms == expected + def test_linux_platforms_manylinux2014_armv6l(self, monkeypatch): monkeypatch.setattr( tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"