diff --git a/scripts/create-arch-wheels.sh b/scripts/create-arch-wheels.sh new file mode 100755 index 00000000..55c7f982 --- /dev/null +++ b/scripts/create-arch-wheels.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# This script is used to create wheels for unsupported architectures +# in order to extend coverage and check errors with those. + +set -eux + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +INTEGRATION_TEST_DIR="${SCRIPT_DIR}/../tests/integration" +mkdir -p "${INTEGRATION_TEST_DIR}/arch-wheels" + +# "mips64le" built with buildpack-deps:bookworm and renamed cp313-cp313 +# "386" "amd64" "arm/v5" "arm/v7" "arm64/v8" +for ARCH in "ppc64le" "riscv64" "s390x"; do + docker run --platform linux/${ARCH} -i --rm -v "${INTEGRATION_TEST_DIR}:/tests" debian:trixie-20250203 << "EOF" +# for, "arm/v5" QEMU will report armv7l, running on aarch64 will report aarch64, force armv5l/armv7l +case "$(dpkg --print-architecture)" in + armel) export _PYTHON_HOST_PLATFORM="linux-armv5l";; + armhf) export _PYTHON_HOST_PLATFORM="linux-armv7l";; + *) ;; +esac +DEBIAN_FRONTEND=noninteractive apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3-pip python3-dev +python3 -m pip wheel --no-deps -w /tests/arch-wheels /tests/testsimple +EOF + +done diff --git a/src/auditwheel/architecture.py b/src/auditwheel/architecture.py new file mode 100644 index 00000000..dfa64aeb --- /dev/null +++ b/src/auditwheel/architecture.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import functools +import platform +import struct +import sys +from enum import Enum + + +class Architecture(Enum): + value: str + + aarch64 = "aarch64" + armv7l = "armv7l" + i686 = "i686" + loongarch64 = "loongarch64" + ppc64 = "ppc64" + ppc64le = "ppc64le" + riscv64 = "riscv64" + s390x = "s390x" + x86_64 = "x86_64" + x86_64_v2 = "x86_64_v2" + x86_64_v3 = "x86_64_v3" + x86_64_v4 = "x86_64_v4" + + def __str__(self): + return self.value + + @property + def baseline(self): + if self.value.startswith("x86_64"): + return Architecture.x86_64 + return self + + @classmethod + @functools.lru_cache(None) + def _member_list(cls) -> list[Architecture]: + return list(cls) + + def is_subset(self, other: Architecture) -> bool: + if self.baseline != other.baseline: + return False + member_list = Architecture._member_list() + return member_list.index(self) <= member_list.index(other) + + def is_superset(self, other: Architecture) -> bool: + if self.baseline != other.baseline: + return False + return other.is_subset(self) + + @staticmethod + def get_native_architecture(*, bits: int | None = None) -> Architecture: + machine = platform.machine() + if sys.platform.startswith("win"): + machine = {"AMD64": "x86_64", "ARM64": "aarch64", "x86": "i686"}.get( + machine, machine + ) + elif sys.platform.startswith("darwin"): + machine = {"arm64": "aarch64"}.get(machine, machine) + + if bits is None: + # c.f. https://github.com/pypa/packaging/pull/711 + bits = 8 * struct.calcsize("P") + + if machine in {"x86_64", "i686"}: + machine = {64: "x86_64", 32: "i686"}[bits] + elif machine in {"aarch64", "armv8l"}: + # use armv7l policy for 64-bit arm kernel in 32-bit mode (armv8l) + machine = {64: "aarch64", 32: "armv7l"}[bits] + + return Architecture(machine) diff --git a/src/auditwheel/json.py b/src/auditwheel/json.py new file mode 100644 index 00000000..a4b8e2f7 --- /dev/null +++ b/src/auditwheel/json.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import dataclasses +import json +from enum import Enum +from typing import Any + + +def _encode_value(value: Any) -> Any: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return dataclasses.asdict(value) + if isinstance(value, frozenset): + return sorted(value) + if isinstance(value, Enum): + return repr(value) + msg = f"object of type {value.__class__.__name__!r} can't be encoded to JSON" + raise TypeError(msg) + + +def dumps(obj: Any) -> str: + return json.dumps(obj, indent=4, default=_encode_value) diff --git a/src/auditwheel/lddtree.py b/src/auditwheel/lddtree.py index ffd31a1a..b521130f 100644 --- a/src/auditwheel/lddtree.py +++ b/src/auditwheel/lddtree.py @@ -8,8 +8,7 @@ """Read the ELF dependency tree This does not work like `ldd` in that we do not execute/load code (only read -files on disk), and we parse the dependency structure as a tree rather than - a flat list. +files on disk). """ from __future__ import annotations @@ -19,16 +18,144 @@ import glob import logging import os +from dataclasses import dataclass from fnmatch import fnmatch from pathlib import Path -from typing import Any +from elftools.elf.constants import E_FLAGS from elftools.elf.elffile import ELFFile +from elftools.elf.sections import NoteSection +from .architecture import Architecture from .libc import Libc, get_libc log = logging.getLogger(__name__) -__all__ = ["lddtree"] +__all__ = ["DynamicExecutable", "DynamicLibrary", "ldd"] + + +@dataclass(frozen=True) +class Platform: + _elf_osabi: str + _elf_class: int + _elf_little_endian: bool + _elf_machine: str + _base_arch: Architecture | None + _ext_arch: Architecture | None + _error_msg: str | None + + def is_compatible(self, other: Platform) -> bool: + os_abis = frozenset((self._elf_osabi, other._elf_osabi)) + compat_sets = ( + frozenset(f"ELFOSABI_{x}" for x in ("NONE", "SYSV", "GNU", "LINUX")), + ) + return ( + (len(os_abis) == 1 or any(os_abis.issubset(x) for x in compat_sets)) + and self._elf_class == other._elf_class + and self._elf_little_endian == other._elf_little_endian + and self._elf_machine == other._elf_machine + ) + + @property + def baseline_architecture(self) -> Architecture: + if self._base_arch is not None: + return self._base_arch + raise ValueError(self._error_msg) + + @property + def extended_architecture(self) -> Architecture | None: + if self._error_msg is not None: + raise ValueError(self._error_msg) + return self._ext_arch + + +@dataclass(frozen=True) +class DynamicLibrary: + soname: str + path: str | None + realpath: str | None + platform: Platform | None = None + needed: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class DynamicExecutable: + interpreter: str | None + path: str + realpath: str + platform: Platform + needed: frozenset[str] + rpath: tuple[str, ...] + runpath: tuple[str, ...] + libraries: dict[str, DynamicLibrary] + + +def _get_platform(elf: ELFFile) -> Platform: + elf_osabi = elf.header["e_ident"]["EI_OSABI"] + elf_class = elf.elfclass + elf_little_endian = elf.little_endian + elf_machine = elf["e_machine"] + base_arch = { + ("EM_386", 32, True): Architecture.i686, + ("EM_X86_64", 64, True): Architecture.x86_64, + ("EM_PPC64", 64, True): Architecture.ppc64le, + ("EM_PPC64", 64, False): Architecture.ppc64, + ("EM_RISCV", 64, True): Architecture.riscv64, + ("EM_AARCH64", 64, True): Architecture.aarch64, + ("EM_S390", 64, False): Architecture.s390x, + ("EM_ARM", 32, True): Architecture.armv7l, + ("EM_LOONGARCH", 64, True): Architecture.loongarch64, + }.get((elf_machine, elf_class, elf_little_endian), None) + ext_arch: Architecture | None = None + error_msg: str | None = None + flags = elf["e_flags"] + assert base_arch is None or base_arch.baseline == base_arch + if base_arch is None: + error_msg = "Unknown architecture" + elif base_arch == Architecture.x86_64: + for section in elf.iter_sections(): + if not isinstance(section, NoteSection): + continue + for note in section.iter_notes(): + if note["n_type"] != "NT_GNU_PROPERTY_TYPE_0": + continue + if note["n_name"] != "GNU": + continue + for prop in note["n_desc"]: + if prop.pr_type != "GNU_PROPERTY_X86_ISA_1_NEEDED": + continue + if prop.pr_datasz != 4: + continue + data = prop.pr_data + data -= data & 1 # clear baseline + if data & 8 == 8: + ext_arch = Architecture.x86_64_v4 + break + if data & 4 == 4: + ext_arch = Architecture.x86_64_v3 + break + if data & 2 == 2: + ext_arch = Architecture.x86_64_v2 + break + if data != 0: + error_msg = "unknown x86_64 ISA" + break + elif base_arch == Architecture.armv7l: + if (flags & E_FLAGS.EF_ARM_EABIMASK) != E_FLAGS.EF_ARM_EABI_VER5: + error_msg = "Invalid ARM EABI version for armv7l" + elif (flags & E_FLAGS.EF_ARM_ABI_FLOAT_HARD) != E_FLAGS.EF_ARM_ABI_FLOAT_HARD: + error_msg = "armv7l shall use hard-float" + if error_msg is not None: + base_arch = None + + return Platform( + elf_osabi, + elf_class, + elf_little_endian, + elf_machine, + base_arch, + ext_arch, + error_msg, + ) def normpath(path: str) -> str: @@ -225,43 +352,16 @@ def load_ld_paths(root: str = "/", prefix: str = "") -> dict[str, list[str]]: return ldpaths -def compatible_elfs(elf1: ELFFile, elf2: ELFFile) -> bool: - """See if two ELFs are compatible - - This compares the aspects of the ELF to see if they're compatible: - bit size, endianness, machine type, and operating system. - - Parameters - ---------- - elf1 : ELFFile - elf2 : ELFFile - - Returns - ------- - True if compatible, False otherwise - """ - osabis = frozenset(e.header["e_ident"]["EI_OSABI"] for e in (elf1, elf2)) - compat_sets = ( - frozenset(f"ELFOSABI_{x}" for x in ("NONE", "SYSV", "GNU", "LINUX")), - ) - return ( - (len(osabis) == 1 or any(osabis.issubset(x) for x in compat_sets)) - and elf1.elfclass == elf2.elfclass - and elf1.little_endian == elf2.little_endian - and elf1.header["e_machine"] == elf2.header["e_machine"] - ) - - def find_lib( - elf: ELFFile, lib: str, ldpaths: list[str], root: str = "/" + platform: Platform, lib: str, ldpaths: list[str], root: str = "/" ) -> tuple[str | None, str | None]: """Try to locate a ``lib`` that is compatible to ``elf`` in the given ``ldpaths`` Parameters ---------- - elf : ELFFile - The elf which the library should be compatible with (ELF wise) + platform : Platform + The platform which the library should be compatible with (ELF wise) lib : str The library (basename) to search for ldpaths : list[str] @@ -281,21 +381,21 @@ def find_lib( if os.path.exists(target): with open(target, "rb") as f: libelf = ELFFile(f) - if compatible_elfs(elf, libelf): - return (target, path) + if platform.is_compatible(_get_platform(libelf)): + return target, path - return (None, None) + return None, None -def lddtree( +def ldd( path: str, root: str = "/", prefix: str = "", ldpaths: dict[str, list[str]] | None = None, display: str | None = None, exclude: frozenset[str] = frozenset(), - _all_libs: dict | None = None, -) -> dict: + _all_libs: dict[str, DynamicLibrary] | None = None, +) -> DynamicExecutable: """Parse the ELF dependency tree of the specified file Parameters @@ -343,21 +443,15 @@ def lddtree( if _all_libs is None: _all_libs = {} - ret: dict[str, Any] = { - "interp": None, - "path": path if display is None else display, - "realpath": path, - "needed": [], - "rpath": [], - "runpath": [], - "libs": _all_libs, - } + log.debug("ldd(%s)", path) - log.debug("lddtree(%s)", path) + interpreter: str | None = None + needed: set[str] = set() + rpaths: list[str] = [] + runpaths: list[str] = [] with open(path, "rb") as f: elf = ELFFile(f) - # If this is the first ELF, extract the interpreter. if _first: for segment in elf.iter_segments(): @@ -366,12 +460,7 @@ def lddtree( interp = segment.get_interp_name() log.debug(" interp = %s", interp) - ret["interp"] = normpath(root + interp) - ret["libs"][os.path.basename(interp)] = { - "path": ret["interp"], - "realpath": readlink(ret["interp"], root, prefixed=True), - "needed": [], - } + interpreter = normpath(root + interp) # XXX: Should read it and scan for /lib paths. ldpaths["interp"] = [ normpath(root + os.path.dirname(interp)), @@ -382,11 +471,10 @@ def lddtree( log.debug(" ldpaths[interp] = %s", ldpaths["interp"]) break + # get the platform + platform = _get_platform(elf) + # Parse the ELF's dynamic tags. - libs: list[str] = [] - rpaths: list[str] = [] - runpaths: list[str] = [] - _excluded_libs: set[str] = set() for segment in elf.iter_segments(): if segment.header.p_type != "PT_DYNAMIC": continue @@ -397,13 +485,7 @@ def lddtree( elif t.entry.d_tag == "DT_RUNPATH": runpaths = parse_ld_paths(t.runpath, path=path, root=root) elif t.entry.d_tag == "DT_NEEDED": - if t.needed in _excluded_libs or any( - fnmatch(t.needed, e) for e in exclude - ): - log.info("Excluding %s", t.needed) - _excluded_libs.add(t.needed) - else: - libs.append(t.needed) + needed.add(t.needed) if runpaths: # If both RPATH and RUNPATH are set, only the latter is used. rpaths = [] @@ -411,57 +493,67 @@ def lddtree( # XXX: We assume there is only one PT_DYNAMIC. This is # probably fine since the runtime ldso does the same. break - if _first: - # Propagate the rpaths used by the main ELF since those will be - # used at runtime to locate things. - ldpaths["rpath"] = rpaths - ldpaths["runpath"] = runpaths - log.debug(" ldpaths[rpath] = %s", rpaths) - log.debug(" ldpaths[runpath] = %s", runpaths) - ret["rpath"] = rpaths - ret["runpath"] = runpaths - - # Search for the libs this ELF uses. - all_ldpaths: list[str] | None = None - for lib in libs: - if lib in _all_libs: - continue - if all_ldpaths is None: - all_ldpaths = ( - ldpaths["rpath"] - + rpaths - + runpaths - + ldpaths["env"] - + ldpaths["runpath"] - + ldpaths["conf"] - + ldpaths["interp"] - ) - realpath, fullpath = find_lib(elf, lib, all_ldpaths, root) - if lib in _excluded_libs or ( - realpath is not None and any(fnmatch(realpath, e) for e in exclude) - ): - log.info("Excluding %s", realpath) - _excluded_libs.add(lib) - continue - _all_libs[lib] = { - "realpath": realpath, - "path": fullpath, - "needed": [], - } - if realpath and fullpath: - lret = lddtree( - realpath, - root, - prefix, - ldpaths, - display=fullpath, - exclude=exclude, - _all_libs=_all_libs, - ) - _all_libs[lib]["needed"] = lret["needed"] del elf - ret["needed"] = [lib for lib in libs if lib not in _excluded_libs] - - return ret + if _first: + # Propagate the rpaths used by the main ELF since those will be + # used at runtime to locate things. + ldpaths["rpath"] = rpaths + ldpaths["runpath"] = runpaths + log.debug(" ldpaths[rpath] = %s", rpaths) + log.debug(" ldpaths[runpath] = %s", runpaths) + + # Search for the libs this ELF uses. + all_ldpaths = ( + ldpaths["rpath"] + + rpaths + + runpaths + + ldpaths["env"] + + ldpaths["runpath"] + + ldpaths["conf"] + + ldpaths["interp"] + ) + _excluded_libs: set[str] = set() + for soname in needed: + if soname in _all_libs: + continue + if soname in _excluded_libs: + continue + if any(fnmatch(soname, e) for e in exclude): + log.info("Excluding %s", soname) + _excluded_libs.add(soname) + continue + realpath, fullpath = find_lib(platform, soname, all_ldpaths, root) + if realpath is not None and any(fnmatch(realpath, e) for e in exclude): + log.info("Excluding %s", realpath) + _excluded_libs.add(soname) + continue + _all_libs[soname] = DynamicLibrary(soname, fullpath, realpath) + if realpath is None or fullpath is None: + continue + dependency = ldd(realpath, root, prefix, ldpaths, fullpath, exclude, _all_libs) + _all_libs[soname] = DynamicLibrary( + soname, + fullpath, + realpath, + dependency.platform, + dependency.needed, + ) + + if interpreter is not None: + soname = os.path.basename(interpreter) + _all_libs[soname] = DynamicLibrary( + soname, interpreter, readlink(interpreter, root, prefixed=True), platform + ) + + return DynamicExecutable( + interpreter, + path if display is None else display, + path, + platform, + frozenset(needed - _excluded_libs), + tuple(rpaths), + tuple(runpaths), + _all_libs, + ) diff --git a/src/auditwheel/main_lddtree.py b/src/auditwheel/main_lddtree.py index 29e13ce8..d0c9a212 100644 --- a/src/auditwheel/main_lddtree.py +++ b/src/auditwheel/main_lddtree.py @@ -14,8 +14,7 @@ def configure_subparser(sub_parsers): def execute(args, p: argparse.ArgumentParser): # noqa: ARG001 - import json + from . import json + from .lddtree import ldd - from .lddtree import lddtree - - logger.info(json.dumps(lddtree(args.file), indent=4)) + logger.info(json.dumps(ldd(args.file))) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index f2bbb64b..49255853 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -101,6 +101,13 @@ def configure_parser(sub_parsers): help="Do not check for higher policy compatibility", default=False, ) + p.add_argument( + "--disable-isa-ext-check", + dest="DISABLE_ISA_EXT_CHECK", + action="store_true", + help="Do not check for extended ISA compatibility (e.g. x86_64_v2)", + default=False, + ) p.set_defaults(func=execute) @@ -123,9 +130,11 @@ def execute(args, parser: argparse.ArgumentParser): os.makedirs(args.WHEEL_DIR) try: - wheel_abi = analyze_wheel_abi(wheel_policy, wheel_file, exclude) - except NonPlatformWheel: - logger.info(NonPlatformWheel.LOG_MESSAGE) + wheel_abi = analyze_wheel_abi( + wheel_policy, wheel_file, exclude, args.DISABLE_ISA_EXT_CHECK + ) + except NonPlatformWheel as e: + logger.info(e.message) return 1 policy = wheel_policy.get_policy_by_name(args.PLAT) @@ -155,6 +164,13 @@ def execute(args, parser: argparse.ArgumentParser): ) parser.error(msg) + if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.machine_tag): + msg = ( + f'cannot repair "{wheel_file}" to "{args.PLAT}" ABI because it ' + "depends on unsupported ISA extensions." + ) + parser.error(msg) + abis = [policy["name"]] + policy["aliases"] if (not args.ONLY_PLAT) and reqd_tag < wheel_policy.get_priority_by_name( wheel_abi.overall_tag diff --git a/src/auditwheel/main_show.py b/src/auditwheel/main_show.py index 7d736342..45b50ea6 100644 --- a/src/auditwheel/main_show.py +++ b/src/auditwheel/main_show.py @@ -12,6 +12,13 @@ def configure_parser(sub_parsers): help = "Audit a wheel for external shared library dependencies." p = sub_parsers.add_parser("show", help=help, description=help) p.add_argument("WHEEL_FILE", help="Path to wheel file.") + p.add_argument( + "--disable-isa-ext-check", + dest="DISABLE_ISA_EXT_CHECK", + action="store_true", + help="Do not check for extended ISA compatibility (e.g. x86_64_v2)", + default=False, + ) p.set_defaults(func=execute) @@ -23,9 +30,9 @@ def printp(text: str) -> None: def execute(args, parser: argparse.ArgumentParser): - import json from os.path import basename, isfile + from . import json from .wheel_abi import NonPlatformWheel, analyze_wheel_abi wheel_policy = WheelPolicies() @@ -36,9 +43,11 @@ def execute(args, parser: argparse.ArgumentParser): parser.error(f"cannot access {args.WHEEL_FILE}. No such file") try: - winfo = analyze_wheel_abi(wheel_policy, args.WHEEL_FILE, frozenset()) - except NonPlatformWheel: - logger.info(NonPlatformWheel.LOG_MESSAGE) + winfo = analyze_wheel_abi( + wheel_policy, args.WHEEL_FILE, frozenset(), args.DISABLE_ISA_EXT_CHECK + ) + except NonPlatformWheel as e: + logger.info(e.message) return 1 libs_with_versions = [ @@ -70,6 +79,14 @@ def execute(args, parser: argparse.ArgumentParser): if args.verbose < 1: return None + if ( + wheel_policy.get_priority_by_name(winfo.machine_tag) + < wheel_policy.priority_highest + ): + printp("This wheel depends on unsupported ISA extensions.") + if args.verbose < 1: + return None + if len(libs_with_versions) == 0: printp( "The wheel references no external versioned symbols from " @@ -99,7 +116,7 @@ def execute(args, parser: argparse.ArgumentParser): printp("The wheel requires no external shared libraries! :)") else: printp("The following external shared libraries are required by the wheel:") - print(json.dumps(dict(sorted(libs.items())), indent=4)) + print(json.dumps(dict(sorted(libs.items())))) for p in sorted(wheel_policy.policies, key=lambda p: p["priority"]): if p["priority"] > wheel_policy.get_priority_by_name(winfo.overall_tag): diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index 5d6f2032..805c3935 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -2,10 +2,7 @@ import json import logging -import platform as _platform_module import re -import struct -import sys from collections import defaultdict from collections.abc import Generator from os.path import abspath, dirname, join @@ -14,6 +11,8 @@ from auditwheel.elfutils import filter_undefined_symbols, is_subdir +from ..architecture import Architecture +from ..lddtree import DynamicExecutable from ..libc import Libc, get_libc from ..musllinux import find_musl_libc, get_musl_version @@ -35,7 +34,7 @@ def __init__( *, libc: Libc | None = None, musl_policy: str | None = None, - arch: str | None = None, + arch: Architecture | None = None, ) -> None: if libc is None: libc = get_libc() if musl_policy is None else Libc.MUSL @@ -50,13 +49,14 @@ def __init__( msg = f"Invalid 'musl_policy': '{musl_policy}'" raise ValueError(msg) if arch is None: - arch = get_arch_name() + arch = Architecture.get_native_architecture() policies = json.loads(_POLICY_JSON_MAP[libc].read_text()) self._policies = [] - self._arch_name = arch + self._architecture = arch self._libc_variant = libc self._musl_policy = musl_policy + base_arch = arch.baseline.value _validate_pep600_compliance(policies) for policy in policies: if self._musl_policy is not None and policy["name"] not in { @@ -64,17 +64,12 @@ def __init__( self._musl_policy, }: continue - if ( - self._arch_name in policy["symbol_versions"] - or policy["name"] == "linux" - ): + if arch.value in policy["symbol_versions"] or policy["name"] == "linux": if policy["name"] != "linux": - policy["symbol_versions"] = policy["symbol_versions"][ - self._arch_name - ] - policy["name"] = policy["name"] + "_" + self._arch_name + policy["symbol_versions"] = policy["symbol_versions"][base_arch] + policy["name"] = policy["name"] + "_" + base_arch policy["aliases"] = [ - alias + "_" + self._arch_name for alias in policy["aliases"] + alias + "_" + base_arch for alias in policy["aliases"] ] policy["lib_whitelist"] = _fixup_musl_libc_soname( libc, arch, policy["lib_whitelist"] @@ -84,6 +79,10 @@ def __init__( if self._libc_variant == Libc.MUSL: assert len(self._policies) == 2, self._policies + @property + def architecture(self) -> Architecture: + return self._architecture + @property def policies(self): return self._policies @@ -96,29 +95,30 @@ def priority_highest(self): def priority_lowest(self): return min(p["priority"] for p in self._policies) - def get_policy_by_name(self, name: str) -> dict | None: + def get_policy_by_name(self, name: str) -> dict: matches = [ p for p in self._policies if p["name"] == name or name in p["aliases"] ] if len(matches) == 0: - return None + msg = f"no policy named {name!r} found" + raise LookupError(msg) if len(matches) > 1: msg = "Internal error. Policies should be unique" raise RuntimeError(msg) return matches[0] - def get_policy_name(self, priority: int) -> str | None: + def get_policy_name(self, priority: int) -> str: matches = [p["name"] for p in self._policies if p["priority"] == priority] if len(matches) == 0: - return None + msg = f"no policy with priority {priority} found" + raise LookupError(msg) if len(matches) > 1: msg = "Internal error. priorities should be unique" raise RuntimeError(msg) return matches[0] - def get_priority_by_name(self, name: str) -> int | None: - policy = self.get_policy_by_name(name) - return None if policy is None else policy["priority"] + def get_priority_by_name(self, name: str) -> int: + return self.get_policy_by_name(name)["priority"] def versioned_symbols_policy(self, versioned_symbols: dict[str, set[str]]) -> int: def policy_is_satisfied( @@ -159,10 +159,10 @@ def policy_is_satisfied( return max(matching_policies) - def lddtree_external_references(self, lddtree: dict, wheel_path: str) -> dict: - # XXX: Document the lddtree structure, or put it in something - # more stable than a big nested dict - def filter_libs(libs: set[str], whitelist: set[str]) -> Generator[str]: + def lddtree_external_references( + self, lddtree: DynamicExecutable, wheel_path: str + ) -> dict: + def filter_libs(libs: frozenset[str], whitelist: set[str]) -> Generator[str]: for lib in libs: if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]: # always exclude ELF dynamic linker/loader @@ -185,7 +185,7 @@ def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: while libs: lib = libs.pop() reqs.add(lib) - for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist): + for dep in filter_libs(lddtree.libraries[lib].needed, whitelist): if dep not in reqs: libs.add(dep) return reqs @@ -201,23 +201,23 @@ def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: # whitelist is the complete set of all libraries. so nothing # is considered "external" that needs to be copied in. whitelist = set(p["lib_whitelist"]) - blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"]) + blacklist_libs = set(p["blacklist"].keys()) & lddtree.needed blacklist = {k: p["blacklist"][k] for k in blacklist_libs} - blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist) + blacklist = filter_undefined_symbols(lddtree.realpath, blacklist) needed_external_libs = get_req_external( - set(filter_libs(lddtree["needed"], whitelist)), whitelist + set(filter_libs(lddtree.needed, whitelist)), whitelist ) pol_ext_deps = {} for lib in needed_external_libs: - if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path): + if is_subdir(lddtree.libraries[lib].realpath, wheel_path): # we didn't filter libs that resolved via RPATH out # earlier because we wanted to make sure to pick up # our elf's indirect dependencies. But now we want to # filter these ones out, since they're not "external". logger.debug("RPATH FTW: %s", lib) continue - pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"] + pol_ext_deps[lib] = lddtree.libraries[lib].realpath ret[p["name"]] = { "libs": pol_ext_deps, "priority": p["priority"], @@ -226,23 +226,6 @@ def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: return ret -def get_arch_name(*, bits: int | None = None) -> str: - machine = _platform_module.machine() - if sys.platform == "darwin" and machine == "arm64": - return "aarch64" - - if bits is None: - # c.f. https://github.com/pypa/packaging/pull/711 - bits = 8 * struct.calcsize("P") - - if machine in {"x86_64", "i686"}: - return {64: "x86_64", 32: "i686"}[bits] - if machine in {"aarch64", "armv8l"}: - # use armv7l policy for 64-bit arm kernel in 32-bit mode (armv8l) - return {64: "aarch64", 32: "armv7l"}[bits] - return machine - - def _validate_pep600_compliance(policies) -> None: symbol_versions: dict[str, dict[str, set[str]]] = {} lib_whitelist: set[str] = set() @@ -275,25 +258,25 @@ def _validate_pep600_compliance(policies) -> None: symbol_versions[arch] = symbol_versions_arch -def _fixup_musl_libc_soname(libc: Libc, arch: str, whitelist): +def _fixup_musl_libc_soname(libc: Libc, arch: Architecture, whitelist): if libc != Libc.MUSL: return whitelist soname_map = { "libc.so": { - "x86_64": "libc.musl-x86_64.so.1", - "i686": "libc.musl-x86.so.1", - "aarch64": "libc.musl-aarch64.so.1", - "s390x": "libc.musl-s390x.so.1", - "ppc64le": "libc.musl-ppc64le.so.1", - "armv7l": "libc.musl-armv7.so.1", - "riscv64": "libc.musl-riscv64.so.1", - "loongarch64": "libc.musl-loongarch64.so.1", + Architecture.x86_64: "libc.musl-x86_64.so.1", + Architecture.i686: "libc.musl-x86.so.1", + Architecture.aarch64: "libc.musl-aarch64.so.1", + Architecture.s390x: "libc.musl-s390x.so.1", + Architecture.ppc64le: "libc.musl-ppc64le.so.1", + Architecture.armv7l: "libc.musl-armv7.so.1", + Architecture.riscv64: "libc.musl-riscv64.so.1", + Architecture.loongarch64: "libc.musl-loongarch64.so.1", } } new_whitelist = [] for soname in whitelist: if soname in soname_map: - new_soname = soname_map[soname][arch] + new_soname = soname_map[soname][arch.baseline] logger.debug("Replacing whitelisted '%s' by '%s'", soname, new_soname) new_whitelist.append(new_soname) else: diff --git a/src/auditwheel/wheel_abi.py b/src/auditwheel/wheel_abi.py index 2e953655..85718d32 100644 --- a/src/auditwheel/wheel_abi.py +++ b/src/auditwheel/wheel_abi.py @@ -2,14 +2,17 @@ import functools import itertools -import json import logging import os -from collections import defaultdict, namedtuple +from collections import defaultdict from collections.abc import Mapping from copy import deepcopy +from dataclasses import dataclass from os.path import basename +from typing import Any +from . import json +from .architecture import Architecture from .elfutils import ( elf_file_filter, elf_find_ucs2_symbols, @@ -18,23 +21,23 @@ elf_references_PyFPE_jbuf, ) from .genericpkgctx import InGenericPkgCtx -from .lddtree import lddtree +from .lddtree import DynamicExecutable, ldd from .policy import WheelPolicies log = logging.getLogger(__name__) -WheelAbIInfo = namedtuple( # noqa: PYI024 - "WheelAbIInfo", - [ - "overall_tag", - "external_refs", - "ref_tag", - "versioned_symbols", - "sym_tag", - "ucs_tag", - "pyfpe_tag", - "blacklist_tag", - ], -) + + +@dataclass(frozen=True) +class WheelAbIInfo: + overall_tag: str + external_refs: dict[str, Any] + ref_tag: str + versioned_symbols: dict[str, set[str]] + sym_tag: str + ucs_tag: str + pyfpe_tag: str + blacklist_tag: str + machine_tag: str class WheelAbiError(Exception): @@ -44,11 +47,26 @@ class WheelAbiError(Exception): class NonPlatformWheel(WheelAbiError): """No ELF binaries in the wheel""" - LOG_MESSAGE = ( - "This does not look like a platform wheel, no ELF executable " - "or shared library file (including compiled Python C extension) " - "found in the wheel archive" - ) + def __init__(self, architecture: Architecture, libraries: list[str]) -> None: + if not libraries: + msg = ( + "This does not look like a platform wheel, no ELF executable " + "or shared library file (including compiled Python C extension) " + "found in the wheel archive" + ) + else: + libraries_str = "\n\t".join(libraries) + msg = ( + "Invalid binary wheel: no ELF executable or shared library file " + "(including compiled Python C extension) with a " + f"{architecture.value!r} architecure found. The following " + f"ELF files were found:\n\t{libraries_str}\n" + ) + super().__init__(msg) + + @property + def message(self) -> str: + return self.args[0] @functools.lru_cache @@ -64,25 +82,38 @@ def get_wheel_elfdata( with InGenericPkgCtx(wheel_fn) as ctx: shared_libraries_in_purelib = [] + shared_libraries_with_invalid_machine = [] platform_wheel = False for fn, elf in elf_file_filter(ctx.iter_files()): - platform_wheel = True - # Check for invalid binary wheel format: no shared library should # be found in purelib so_path_split = fn.split(os.sep) + so_name = so_path_split[-1] # If this is in purelib, add it to the list of shared libraries in # purelib if "purelib" in so_path_split: - shared_libraries_in_purelib.append(so_path_split[-1]) + shared_libraries_in_purelib.append(so_name) # If at least one shared library exists in purelib, this is going # to fail and there's no need to do further checks if not shared_libraries_in_purelib: log.debug("processing: %s", fn) - elftree = lddtree(fn, exclude=exclude) + elftree = ldd(fn, exclude=exclude) + + try: + arch = elftree.platform.baseline_architecture + if arch != wheel_policy.architecture.baseline: + shared_libraries_with_invalid_machine.append(so_name) + log.warning("ignoring: %s with %s architecture", so_name, arch) + continue + except ValueError: + shared_libraries_with_invalid_machine.append(so_name) + log.warning("ignoring: %s with unknown architecture", so_name) + continue + + platform_wheel = True for key, value in elf_find_versioned_symbols(elf): log.debug("key %s, value %s", key, value) @@ -109,9 +140,6 @@ def get_wheel_elfdata( # its internal references later. nonpy_elftree[fn] = elftree - if not platform_wheel: - raise NonPlatformWheel - # If at least one shared library exists in purelib, raise an error if shared_libraries_in_purelib: libraries = "\n\t".join(shared_libraries_in_purelib) @@ -123,11 +151,16 @@ def get_wheel_elfdata( ) raise RuntimeError(msg) + if not platform_wheel: + raise NonPlatformWheel( + wheel_policy.architecture, shared_libraries_with_invalid_machine + ) + # Get a list of all external libraries needed by ELFs in the wheel. needed_libs = { lib for elf in itertools.chain(full_elftree.values(), nonpy_elftree.values()) - for lib in elf["needed"] + for lib in elf.needed } for fn, elf_tree in nonpy_elftree.items(): @@ -144,10 +177,9 @@ def get_wheel_elfdata( elf_tree, ctx.path ) - log.debug("full_elftree:\n%s", json.dumps(full_elftree, indent=4)) + log.debug("full_elftree:\n%s", json.dumps(full_elftree)) log.debug( - "full_external_refs (will be repaired):\n%s", - json.dumps(full_external_refs, indent=4), + "full_external_refs (will be repaired):\n%s", json.dumps(full_external_refs) ) return ( @@ -227,8 +259,50 @@ def get_symbol_policies( return result +def _get_machine_policy( + wheel_policy: WheelPolicies, + elftree_by_fn: dict[str, DynamicExecutable], + external_so_names: frozenset[str], +) -> int: + result = wheel_policy.priority_highest + machine_to_check = {} + for fn, dynamic_executable in elftree_by_fn.items(): + if fn in machine_to_check: + continue + machine_to_check[fn] = dynamic_executable.platform.extended_architecture + for dependency in dynamic_executable.libraries.values(): + if dependency.soname not in external_so_names: + continue + if dependency.realpath is None: + continue + assert dependency.platform is not None + if dependency.realpath in machine_to_check: + continue + machine_to_check[dependency.realpath] = ( + dependency.platform.extended_architecture + ) + + for fn, extended_architecture in machine_to_check.items(): + if extended_architecture is None: + continue + if wheel_policy.architecture.is_superset(extended_architecture): + continue + log.warning( + "ELF file %r requires %r instruction set, not in %r", + fn, + extended_architecture.value, + wheel_policy.architecture.value, + ) + result = wheel_policy.priority_lowest + + return result + + def analyze_wheel_abi( - wheel_policy: WheelPolicies, wheel_fn: str, exclude: frozenset[str] + wheel_policy: WheelPolicies, + wheel_fn: str, + exclude: frozenset[str], + disable_isa_ext_check: bool, ) -> WheelAbIInfo: external_refs = { p["name"]: {"libs": {}, "blacklist": {}, "priority": p["priority"]} @@ -247,7 +321,7 @@ def analyze_wheel_abi( update(external_refs, external_refs_by_fn[fn]) log.debug("external reference info") - log.debug(json.dumps(external_refs, indent=4)) + log.debug(json.dumps(external_refs)) external_libs = get_external_libs(external_refs) external_versioned_symbols = get_versioned_symbols(external_libs) @@ -272,6 +346,13 @@ def analyze_wheel_abi( default=wheel_policy.priority_lowest, ) + if disable_isa_ext_check: + machine_policy = wheel_policy.priority_highest + else: + machine_policy = _get_machine_policy( + wheel_policy, elftree_by_fn, frozenset(external_libs.values()) + ) + if has_ucs2: ucs_policy = wheel_policy.priority_lowest else: @@ -287,8 +368,16 @@ def analyze_wheel_abi( ucs_tag = wheel_policy.get_policy_name(ucs_policy) pyfpe_tag = wheel_policy.get_policy_name(pyfpe_policy) blacklist_tag = wheel_policy.get_policy_name(blacklist_policy) + machine_tag = wheel_policy.get_policy_name(machine_policy) overall_tag = wheel_policy.get_policy_name( - min(symbol_policy, ref_policy, ucs_policy, pyfpe_policy, blacklist_policy) + min( + symbol_policy, + ref_policy, + ucs_policy, + pyfpe_policy, + blacklist_policy, + machine_policy, + ) ) return WheelAbIInfo( @@ -300,6 +389,7 @@ def analyze_wheel_abi( ucs_tag, pyfpe_tag, blacklist_tag, + machine_tag, ) diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_aarch64.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_aarch64.whl new file mode 100644 index 00000000..2c277261 Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_aarch64.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv5l.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv5l.whl new file mode 100644 index 00000000..7c907faa Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv5l.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv7l.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv7l.whl new file mode 100644 index 00000000..92da3c7d Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_armv7l.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_i686.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_i686.whl new file mode 100644 index 00000000..1ad4d066 Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_i686.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_mips64.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_mips64.whl new file mode 100644 index 00000000..2e4992a3 Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_mips64.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_ppc64le.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_ppc64le.whl new file mode 100644 index 00000000..b799bc0a Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_ppc64le.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_riscv64.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_riscv64.whl new file mode 100644 index 00000000..98b2a759 Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_riscv64.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_s390x.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_s390x.whl new file mode 100644 index 00000000..a8fc3f72 Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_s390x.whl differ diff --git a/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_x86_64.whl b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_x86_64.whl new file mode 100644 index 00000000..85b0408b Binary files /dev/null and b/tests/integration/arch-wheels/testsimple-0.0.1-cp313-cp313-linux_x86_64.whl differ diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index b4d14a0f..7b9fb18a 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -16,9 +16,10 @@ import pytest from auditwheel import lddtree, main_repair +from auditwheel.architecture import Architecture from auditwheel.libc import Libc from auditwheel.policy import WheelPolicies -from auditwheel.wheel_abi import analyze_wheel_abi +from auditwheel.wheel_abi import NonPlatformWheel, analyze_wheel_abi HERE = Path(__file__).parent.resolve() @@ -67,8 +68,8 @@ def test_analyze_wheel_abi(file, external_libs, exclude): cp.setenv("LD_LIBRARY_PATH", f"{HERE}") importlib.reload(lddtree) - wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch="x86_64") - winfo = analyze_wheel_abi(wheel_policies, str(HERE / file), exclude) + wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) + winfo = analyze_wheel_abi(wheel_policies, str(HERE / file), exclude, False) assert ( set(winfo.external_refs["manylinux_2_5_x86_64"]["libs"]) == external_libs ), f"{HERE}, {exclude}, {os.environ}" @@ -78,11 +79,12 @@ def test_analyze_wheel_abi(file, external_libs, exclude): def test_analyze_wheel_abi_pyfpe(): - wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch="x86_64") + wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) winfo = analyze_wheel_abi( wheel_policies, str(HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl"), frozenset(), + False, ) assert ( winfo.sym_tag == "manylinux_2_5_x86_64" @@ -92,6 +94,17 @@ def test_analyze_wheel_abi_pyfpe(): ) # but for having the pyfpe reference, it gets just linux +def test_analyze_wheel_abi_bad_architecture(): + wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.aarch64) + with pytest.raises(NonPlatformWheel): + analyze_wheel_abi( + wheel_policies, + str(HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl"), + frozenset(), + False, + ) + + @pytest.mark.skipif(platform.machine() != "x86_64", reason="only checked on x86_64") def test_wheel_source_date_epoch(tmp_path, monkeypatch): wheel_build_path = tmp_path / "wheel" @@ -120,6 +133,7 @@ def test_wheel_source_date_epoch(tmp_path, monkeypatch): WHEEL_DIR=str(wheel_output_path), WHEEL_FILE=[str(wheel_path)], EXCLUDE=[], + DISABLE_ISA_EXT_CHECK=False, cmd="repair", func=Mock(), prog="auditwheel", diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index f5b10503..a2d4f7bf 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -16,12 +16,13 @@ import pytest from elftools.elf.elffile import ELFFile -from auditwheel.policy import WheelPolicies, get_arch_name +from auditwheel.architecture import Architecture +from auditwheel.policy import WheelPolicies logger = logging.getLogger(__name__) ENCODING = "utf-8" -PLATFORM = get_arch_name() +PLATFORM = Architecture.get_native_architecture().value MANYLINUX1_IMAGE_ID = f"quay.io/pypa/manylinux1_{PLATFORM}:latest" MANYLINUX2010_IMAGE_ID = f"quay.io/pypa/manylinux2010_{PLATFORM}:latest" MANYLINUX2014_IMAGE_ID = f"quay.io/pypa/manylinux2014_{PLATFORM}:latest" @@ -89,7 +90,7 @@ NUMPY_VERSION = NUMPY_VERSION_MAP[PYTHON_ABI_MAJ_MIN] ORIGINAL_NUMPY_WHEEL = f"numpy-{NUMPY_VERSION}-{PYTHON_ABI}-linux_{PLATFORM}.whl" SHOW_RE = re.compile( - r'[\s](?P\S+) is consistent with the following platform tag: "(?P\S+)"', + r'.*[\s](?P\S+) is consistent with the following platform tag: "(?P\S+)".*', flags=re.DOTALL, ) TAG_RE = re.compile(r"^manylinux_(?P[0-9]+)_(?P[0-9]+)_(?P\S+)$") @@ -189,11 +190,14 @@ def tmp_docker_image(base, commands, setup_env=None): client.images.remove(image.id) -def assert_show_output(manylinux_ctr, wheel, expected_tag, strict): - output = docker_exec(manylinux_ctr, f"auditwheel show /io/{wheel}") +def assert_show_output(manylinux_ctr, wheel, expected_tag, strict, isa_ext_check=True): + isa_ext_check_arg = "" if isa_ext_check else "--disable-isa-ext-check" + output = docker_exec( + manylinux_ctr, f"auditwheel show {isa_ext_check_arg} /io/{wheel}" + ) output = output.replace("\n", " ") match = SHOW_RE.match(output) - assert match + assert match, f"{SHOW_RE.pattern!r} not found in:\n{output}" assert match["wheel"] == wheel if strict or "musllinux" in expected_tag: assert match["tag"] == expected_tag @@ -218,7 +222,7 @@ def build_numpy(container, policy, output_dir): # https://github.com/numpy/numpy/issues/27932 fix_hwcap = "echo '#define HWCAP_S390_VX 2048' >> /usr/include/bits/hwcap.h" docker_exec(container, f'sh -c "{fix_hwcap}"') - elif policy.startswith("manylinux_2_28_"): + elif policy.startswith(("manylinux_2_28_", "manylinux_2_34_")): docker_exec(container, "dnf install -y openblas-devel") else: if tuple(int(part) for part in NUMPY_VERSION.split(".")[:2]) >= (1, 26): @@ -391,16 +395,18 @@ def test_build_wheel_with_binary_executable( orig_wheel = filenames[0] assert "manylinux" not in orig_wheel + # manylinux_2_34_x86_64 uses x86_64_v2 for this test + isa_ext_check = policy != "manylinux_2_34_x86_64" + isa_ext_check_arg = "" if isa_ext_check else "--disable-isa-ext-check" + # Repair the wheel using the appropriate manylinux container - repair_command = ( - f"auditwheel repair --plat {policy} --only-plat -w /io /io/{orig_wheel}" - ) + repair_command = f"auditwheel repair --plat {policy} {isa_ext_check_arg} --only-plat -w /io /io/{orig_wheel}" docker_exec(manylinux_ctr, repair_command) filenames = os.listdir(io_folder) assert len(filenames) == 2 repaired_wheel = f"testpackage-0.0.1-py3-none-{tag}.whl" assert repaired_wheel in filenames - assert_show_output(manylinux_ctr, repaired_wheel, policy, False) + assert_show_output(manylinux_ctr, repaired_wheel, policy, False, isa_ext_check) docker_exec(docker_python, "pip install /io/" + repaired_wheel) output = docker_exec( @@ -812,6 +818,57 @@ def test_glibcxx_3_4_25(self, any_manylinux_container, docker_python, io_folder) ], ) + @pytest.mark.skipif( + PLATFORM != "x86_64", reason="ISA extension only implemented on x86_64" + ) + @pytest.mark.parametrize("isa_ext", ["x86-64-v2", "x86-64-v3", "x86-64-v4"]) + def test_isa_variants(self, any_manylinux_container, io_folder, isa_ext): + policy, tag, manylinux_ctr = any_manylinux_container + if policy.startswith(("manylinux_2_5_", "manylinux_2_12_", "manylinux_2_17_")): + pytest.skip("skip old gcc") + build_command = ( + "cd /auditwheel_src/tests/integration/testdependencies && " + "if [ -d ./build ]; then rm -rf ./build ./*.egg-info; fi && " + f"WITH_DEPENDENCY=1 WITH_ARCH={isa_ext} python -m pip wheel --no-deps -w /io ." + ) + docker_exec( + manylinux_ctr, + [ + "bash", + "-c", + build_command, + ], + ) + + filenames = os.listdir(io_folder) + orig_wheel = filenames[0] + assert "manylinux" not in orig_wheel + + # repair failure with ISA check + repair_command = ( + "LD_LIBRARY_PATH=" + "/auditwheel_src/tests/integration/testdependencies:$LD_LIBRARY_PATH " + f"auditwheel repair --plat {policy} -w /io /io/{orig_wheel}" + ) + with pytest.raises(CalledProcessError): + docker_exec(manylinux_ctr, ["bash", "-c", repair_command]) + + repair_command = ( + "LD_LIBRARY_PATH=" + "/auditwheel_src/tests/integration/testdependencies:$LD_LIBRARY_PATH " + f"auditwheel repair --disable-isa-ext-check --plat {policy} -w /io /io/{orig_wheel}" + ) + docker_exec(manylinux_ctr, ["bash", "-c", repair_command]) + + filenames = os.listdir(io_folder) + assert len(filenames) == 2 + repaired_wheel = f"testdependencies-0.0.1-{PYTHON_ABI}-{policy}.whl" + assert repaired_wheel in filenames + assert_show_output(manylinux_ctr, repaired_wheel, policy, True, False) + + # with ISA check, we shall not report a manylinux/musllinux policy + assert_show_output(manylinux_ctr, repaired_wheel, f"linux_{PLATFORM}", True) + class TestManylinux(Anylinux): @pytest.fixture(scope="session") diff --git a/tests/integration/test_nonplatform_wheel.py b/tests/integration/test_nonplatform_wheel.py index 614643ac..c66c062e 100644 --- a/tests/integration/test_nonplatform_wheel.py +++ b/tests/integration/test_nonplatform_wheel.py @@ -5,11 +5,13 @@ import pytest +from auditwheel.architecture import Architecture + HERE = pathlib.Path(__file__).parent.resolve() @pytest.mark.parametrize("mode", ["repair", "show"]) -def test_non_platform_wheel_repair(mode): +def test_non_platform_wheel_pure(mode): wheel = HERE / "plumbum-1.6.8-py2.py3-none-any.whl" proc = subprocess.run( ["auditwheel", mode, str(wheel)], @@ -20,3 +22,39 @@ def test_non_platform_wheel_repair(mode): assert proc.returncode == 1 assert "This does not look like a platform wheel" in proc.stderr assert "AttributeError" not in proc.stderr + + +@pytest.mark.parametrize("mode", ["repair", "show"]) +@pytest.mark.parametrize("arch", ["armv5l", "mips64"]) +def test_non_platform_wheel_unknown_arch(mode, arch): + wheel = HERE / "arch-wheels" / f"testsimple-0.0.1-cp313-cp313-linux_{arch}.whl" + proc = subprocess.run( + ["auditwheel", mode, str(wheel)], + stderr=subprocess.PIPE, + text=True, + check=False, + ) + assert proc.returncode == 1 + assert "Invalid binary wheel: no ELF executable or" in proc.stderr + assert "unknown architecture" in proc.stderr + assert "AttributeError" not in proc.stderr + + +@pytest.mark.parametrize("mode", ["repair", "show"]) +@pytest.mark.parametrize( + "arch", ["aarch64", "armv7l", "i686", "x86_64", "ppc64le", "s390x"] +) +def test_non_platform_wheel_bad_arch(mode, arch): + if Architecture.get_native_architecture().value == arch: + pytest.skip("host architecture") + wheel = HERE / "arch-wheels" / f"testsimple-0.0.1-cp313-cp313-linux_{arch}.whl" + proc = subprocess.run( + ["auditwheel", mode, str(wheel)], + stderr=subprocess.PIPE, + text=True, + check=False, + ) + assert proc.returncode == 1 + assert "Invalid binary wheel: no ELF executable or" in proc.stderr + assert f"{arch} architecture" in proc.stderr + assert "AttributeError" not in proc.stderr diff --git a/tests/integration/testdependencies/dependency.c b/tests/integration/testdependencies/dependency.c index d1be1444..09055cb8 100644 --- a/tests/integration/testdependencies/dependency.c +++ b/tests/integration/testdependencies/dependency.c @@ -4,13 +4,17 @@ #include #include #include -#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 28) +#if defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 28) #include #endif +#endif int dep_run() { -#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 34) +#if defined(__GLIBC_PREREQ) + +#if __GLIBC_PREREQ(2, 34) // pthread_mutexattr_init was moved to libc.so.6 in manylinux_2_34+ pthread_mutexattr_t attr; int sts = pthread_mutexattr_init(&attr); @@ -29,4 +33,8 @@ int dep_run() #else return 0; #endif + +#else + return 0; +#endif } diff --git a/tests/integration/testdependencies/setup.py b/tests/integration/testdependencies/setup.py index 9dcf8216..b4f76b3e 100644 --- a/tests/integration/testdependencies/setup.py +++ b/tests/integration/testdependencies/setup.py @@ -9,18 +9,23 @@ define_macros = [("_GNU_SOURCE", None)] libraries = [] library_dirs = [] +extra_compile_args = [] if getenv("WITH_DEPENDENCY", "0") == "1": libraries.append("dependency") library_dirs.append(path.abspath(path.dirname(__file__))) define_macros.append(("WITH_DEPENDENCY", "1")) +if getenv("WITH_ARCH", "") != "": + extra_compile_args.extend((f"-march={getenv('WITH_ARCH')}", "-mneeded")) + libraries.extend(["m", "c"]) class BuildExt(build_ext): def run(self) -> None: - cmd = "gcc -shared -fPIC -D_GNU_SOURCE dependency.c -o libdependency.so -lm -lc" + cflags = ("-shared", "-fPIC", "-D_GNU_SOURCE", *extra_compile_args) + cmd = f"gcc {' '.join(cflags)} dependency.c -o libdependency.so -lm -lc" subprocess.check_call(cmd.split()) super().run() @@ -33,6 +38,7 @@ def run(self) -> None: Extension( "testdependencies", sources=["testdependencies.c"], + extra_compile_args=extra_compile_args, define_macros=define_macros, libraries=libraries, library_dirs=library_dirs, diff --git a/tests/integration/testdependencies/testdependencies.c b/tests/integration/testdependencies/testdependencies.c index b201f4a9..e28a0108 100644 --- a/tests/integration/testdependencies/testdependencies.c +++ b/tests/integration/testdependencies/testdependencies.c @@ -6,10 +6,12 @@ #include #include #include -#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 28) +#if defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 28) #include #endif #endif +#endif #include static __thread int tres = 0; @@ -24,21 +26,27 @@ run(PyObject *self, PyObject *args) #ifdef WITH_DEPENDENCY res = dep_run(); -#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 34) +#elif defined(__GLIBC_PREREQ) + +#if __GLIBC_PREREQ(2, 34) // pthread_mutexattr_init was moved to libc.so.6 in manylinux_2_34+ pthread_mutexattr_t attr; res = pthread_mutexattr_init(&attr); if (res == 0) { pthread_mutexattr_destroy(&attr); } -#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 28) +#elif __GLIBC_PREREQ(2, 28) res = thrd_equal(thrd_current(), thrd_current()) ? 0 : 1; -#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 24) +#elif __GLIBC_PREREQ(2, 24) res = (int)nextupf(0.0F); -#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17) +#elif __GLIBC_PREREQ(2, 17) res = (int)(intptr_t)secure_getenv("NON_EXISTING_ENV_VARIABLE"); -#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10) +#elif __GLIBC_PREREQ(2, 10) res = malloc_info(0, stdout); +#else + res = 0; +#endif + #else res = 0; #endif diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py new file mode 100644 index 00000000..ea7b8dfa --- /dev/null +++ b/tests/unit/test_architecture.py @@ -0,0 +1,101 @@ +import platform +import struct +import sys + +import pytest + +from auditwheel.architecture import Architecture + + +@pytest.mark.parametrize( + ("sys_platform", "reported_arch", "expected_arch"), + [ + ("linux", "armv7l", Architecture.armv7l), + ("linux", "armv8l", Architecture.armv7l), + ("linux", "aarch64", Architecture.armv7l), + ("linux", "i686", Architecture.i686), + ("linux", "x86_64", Architecture.i686), + ("win32", "x86", Architecture.i686), + ("win32", "AMD64", Architecture.i686), + ], +) +def test_32bits_arch_name(sys_platform, reported_arch, expected_arch, monkeypatch): + monkeypatch.setattr(sys, "platform", sys_platform) + monkeypatch.setattr(platform, "machine", lambda: reported_arch) + machine = Architecture.get_native_architecture(bits=32) + assert machine == expected_arch + + +@pytest.mark.parametrize( + ("sys_platform", "reported_arch", "expected_arch"), + [ + ("linux", "armv8l", Architecture.aarch64), + ("linux", "aarch64", Architecture.aarch64), + ("linux", "ppc64le", Architecture.ppc64le), + ("linux", "i686", Architecture.x86_64), + ("linux", "x86_64", Architecture.x86_64), + ("darwin", "arm64", Architecture.aarch64), + ("darwin", "x86_64", Architecture.x86_64), + ("win32", "ARM64", Architecture.aarch64), + ("win32", "AMD64", Architecture.x86_64), + ], +) +def test_64bits_arch_name(sys_platform, reported_arch, expected_arch, monkeypatch): + monkeypatch.setattr(sys, "platform", sys_platform) + monkeypatch.setattr(platform, "machine", lambda: reported_arch) + machine = Architecture.get_native_architecture(bits=64) + assert machine == expected_arch + + +@pytest.mark.parametrize( + ("maxsize", "sizeof_voidp", "expected"), + [ + # 64-bit + (9223372036854775807, 8, Architecture.x86_64), + # 32-bit + (2147483647, 4, Architecture.i686), + # 64-bit w/ 32-bit sys.maxsize: GraalPy, IronPython, Jython + (2147483647, 8, Architecture.x86_64), + ], +) +def test_arch_name_bits(maxsize, sizeof_voidp, expected, monkeypatch): + def _calcsize(fmt): + assert fmt == "P" + return sizeof_voidp + + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + monkeypatch.setattr(sys, "maxsize", maxsize) + monkeypatch.setattr(struct, "calcsize", _calcsize) + machine = Architecture.get_native_architecture() + assert machine == expected + + +@pytest.mark.parametrize( + ("smaller", "larger"), + [ + (Architecture.x86_64, Architecture.x86_64_v4), + (Architecture.x86_64, Architecture.x86_64), + (Architecture.x86_64, Architecture.x86_64_v2), + (Architecture.x86_64_v2, Architecture.x86_64_v3), + (Architecture.x86_64_v3, Architecture.x86_64_v4), + ], +) +def test_order_valid(smaller, larger): + assert smaller.is_subset(larger) + assert larger.is_superset(smaller) + + +@pytest.mark.parametrize( + ("smaller", "larger"), + [ + (Architecture.x86_64, Architecture.x86_64_v4), + (Architecture.x86_64, Architecture.x86_64_v2), + (Architecture.x86_64_v2, Architecture.x86_64_v3), + (Architecture.x86_64_v3, Architecture.x86_64_v4), + (Architecture.aarch64, Architecture.x86_64), + (Architecture.x86_64, Architecture.aarch64), + ], +) +def test_order_invalid(smaller, larger): + assert not smaller.is_superset(larger) + assert not larger.is_subset(smaller) diff --git a/tests/unit/test_json.py b/tests/unit/test_json.py new file mode 100644 index 00000000..a5504fe8 --- /dev/null +++ b/tests/unit/test_json.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass +from enum import Enum +from json import loads + +import pytest + +from auditwheel.json import dumps + + +def test_dataclass(): + @dataclass(frozen=True) + class Dummy: + first: str = "val0" + second: int = 2 + + assert loads(dumps(Dummy())) == {"first": "val0", "second": 2} + + +def test_enum(): + class Dummy(Enum): + value: str + + TEST = "dummy" + + def __repr__(self): + return self.value + + assert Dummy.TEST.value == loads(dumps(Dummy.TEST)) + + +def test_frozenset(): + obj = frozenset((3, 9, 6, 5, 21)) + data = loads(dumps(obj)) + assert data == sorted(obj) + + +def test_invalid_type(): + class Dummy: + pass + + with pytest.raises(TypeError): + dumps(Dummy()) diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index 427d2d80..a65f477d 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -1,19 +1,17 @@ from __future__ import annotations -import platform import re -import struct -import sys from contextlib import nullcontext as does_not_raise import pytest +from auditwheel.architecture import Architecture from auditwheel.error import InvalidLibc +from auditwheel.lddtree import DynamicExecutable, DynamicLibrary, Platform from auditwheel.libc import Libc from auditwheel.policy import ( WheelPolicies, _validate_pep600_compliance, - get_arch_name, get_libc, get_replace_platforms, ) @@ -35,62 +33,6 @@ def raises(exception, match=None, escape=True): return pytest.raises(exception, match=match) -@pytest.mark.parametrize( - ("reported_arch", "expected_arch"), - [ - ("armv6l", "armv6l"), - ("armv7l", "armv7l"), - ("armv8l", "armv7l"), - ("aarch64", "armv7l"), - ("i686", "i686"), - ("x86_64", "i686"), - ], -) -def test_32bits_arch_name(reported_arch, expected_arch, monkeypatch): - monkeypatch.setattr(platform, "machine", lambda: reported_arch) - machine = get_arch_name(bits=32) - assert machine == expected_arch - - -@pytest.mark.parametrize( - ("reported_arch", "expected_arch"), - [ - ("armv8l", "aarch64"), - ("aarch64", "aarch64"), - ("ppc64le", "ppc64le"), - ("i686", "x86_64"), - ("x86_64", "x86_64"), - ], -) -def test_64bits_arch_name(reported_arch, expected_arch, monkeypatch): - monkeypatch.setattr(platform, "machine", lambda: reported_arch) - machine = get_arch_name(bits=64) - assert machine == expected_arch - - -@pytest.mark.parametrize( - ("maxsize", "sizeof_voidp", "expected"), - [ - # 64-bit - (9223372036854775807, 8, "x86_64"), - # 32-bit - (2147483647, 4, "i686"), - # 64-bit w/ 32-bit sys.maxsize: GraalPy, IronPython, Jython - (2147483647, 8, "x86_64"), - ], -) -def test_arch_name_bits(maxsize, sizeof_voidp, expected, monkeypatch): - def _calcsize(fmt): - assert fmt == "P" - return sizeof_voidp - - monkeypatch.setattr(platform, "machine", lambda: "x86_64") - monkeypatch.setattr(sys, "maxsize", maxsize) - monkeypatch.setattr(struct, "calcsize", _calcsize) - machine = get_arch_name() - assert machine == expected - - @pytest.mark.parametrize( ("name", "expected"), [ @@ -195,19 +137,20 @@ def test_pep600_compliance(): class TestPolicyAccess: def test_get_by_priority(self): - _arch = get_arch_name() + arch = Architecture.get_native_architecture() wheel_policy = WheelPolicies() - assert wheel_policy.get_policy_name(65) == f"manylinux_2_27_{_arch}" - assert wheel_policy.get_policy_name(70) == f"manylinux_2_24_{_arch}" - assert wheel_policy.get_policy_name(80) == f"manylinux_2_17_{_arch}" - if _arch in {"x86_64", "i686"}: - assert wheel_policy.get_policy_name(90) == f"manylinux_2_12_{_arch}" - assert wheel_policy.get_policy_name(100) == f"manylinux_2_5_{_arch}" - assert wheel_policy.get_policy_name(0) == f"linux_{_arch}" + assert wheel_policy.get_policy_name(65) == f"manylinux_2_27_{arch}" + assert wheel_policy.get_policy_name(70) == f"manylinux_2_24_{arch}" + assert wheel_policy.get_policy_name(80) == f"manylinux_2_17_{arch}" + if arch in {Architecture.x86_64, Architecture.i686}: + assert wheel_policy.get_policy_name(90) == f"manylinux_2_12_{arch}" + assert wheel_policy.get_policy_name(100) == f"manylinux_2_5_{arch}" + assert wheel_policy.get_policy_name(0) == f"linux_{arch}" def test_get_by_priority_missing(self): wheel_policy = WheelPolicies() - assert wheel_policy.get_policy_name(101) is None + with pytest.raises(LookupError): + wheel_policy.get_policy_name(101) def test_get_by_priority_duplicate(self): wheel_policy = WheelPolicies() @@ -219,21 +162,22 @@ def test_get_by_priority_duplicate(self): wheel_policy.get_policy_name(0) def test_get_by_name(self): - _arch = get_arch_name() + arch = Architecture.get_native_architecture() wheel_policy = WheelPolicies() - assert wheel_policy.get_priority_by_name(f"manylinux_2_27_{_arch}") == 65 - assert wheel_policy.get_priority_by_name(f"manylinux_2_24_{_arch}") == 70 - assert wheel_policy.get_priority_by_name(f"manylinux2014_{_arch}") == 80 - assert wheel_policy.get_priority_by_name(f"manylinux_2_17_{_arch}") == 80 - if _arch in {"x86_64", "i686"}: - assert wheel_policy.get_priority_by_name(f"manylinux2010_{_arch}") == 90 - assert wheel_policy.get_priority_by_name(f"manylinux_2_12_{_arch}") == 90 - assert wheel_policy.get_priority_by_name(f"manylinux1_{_arch}") == 100 - assert wheel_policy.get_priority_by_name(f"manylinux_2_5_{_arch}") == 100 + assert wheel_policy.get_priority_by_name(f"manylinux_2_27_{arch}") == 65 + assert wheel_policy.get_priority_by_name(f"manylinux_2_24_{arch}") == 70 + assert wheel_policy.get_priority_by_name(f"manylinux2014_{arch}") == 80 + assert wheel_policy.get_priority_by_name(f"manylinux_2_17_{arch}") == 80 + if arch in {Architecture.x86_64, Architecture.i686}: + assert wheel_policy.get_priority_by_name(f"manylinux2010_{arch}") == 90 + assert wheel_policy.get_priority_by_name(f"manylinux_2_12_{arch}") == 90 + assert wheel_policy.get_priority_by_name(f"manylinux1_{arch}") == 100 + assert wheel_policy.get_priority_by_name(f"manylinux_2_5_{arch}") == 100 def test_get_by_name_missing(self): wheel_policy = WheelPolicies() - assert wheel_policy.get_priority_by_name("nosuchpolicy") is None + with pytest.raises(LookupError): + wheel_policy.get_priority_by_name("nosuchpolicy") def test_get_by_name_duplicate(self): wheel_policy = WheelPolicies() @@ -261,12 +205,19 @@ def test_filter_libs(self): ] unfiltered_libs = ["libfoo.so.1.0", "libbar.so.999.999.999"] libs = filtered_libs + unfiltered_libs - - lddtree = { - "realpath": "/path/to/lib", - "needed": libs, - "libs": {lib: {"needed": [], "realpath": "/path/to/lib"} for lib in libs}, - } + lddtree = DynamicExecutable( + interpreter=None, + path="/path/to/lib", + realpath="/path/to/lib", + platform=Platform("", 64, True, "EM_X86_64", "x86_64", None, None), + needed=frozenset(libs), + libraries={ + lib: DynamicLibrary(lib, f"/path/to/{lib}", f"/path/to/{lib}") + for lib in libs + }, + rpath=(), + runpath=(), + ) wheel_policy = WheelPolicies() full_external_refs = wheel_policy.lddtree_external_references( lddtree, "/path/to/wheel" @@ -285,7 +236,7 @@ def test_filter_libs(self): (Libc.GLIBC, None, None, does_not_raise()), (Libc.MUSL, "musllinux_1_1", None, does_not_raise()), (None, "musllinux_1_1", None, does_not_raise()), - (None, None, "aarch64", does_not_raise()), + (None, None, Architecture.aarch64, does_not_raise()), # invalid ( Libc.GLIBC, @@ -295,7 +246,6 @@ def test_filter_libs(self): ), (Libc.MUSL, "manylinux_1_1", None, raises(ValueError, "Invalid 'musl_policy'")), (Libc.MUSL, "musllinux_5_1", None, raises(AssertionError)), - (Libc.MUSL, "musllinux_1_1", "foo", raises(AssertionError)), # platform dependant ( Libc.MUSL, @@ -314,4 +264,4 @@ def test_wheel_policies_args(libc, musl_policy, arch, exception): if musl_policy is not None: assert wheel_policies._musl_policy == musl_policy if arch is not None: - assert wheel_policies._arch_name == arch + assert wheel_policies.architecture == arch