python3-maturin: Fix cross compilation issue for armv7l, mips64, ppc

When bitbaking python3-rpds-py it built extension module as: site-packages/rpds/rpds.cpython-312-armv7l-linux-gnueabihf.so Which caused error on target: root@qemuarm:~# python3 -c "from rpds import HashTrieMap, HashTrieSet, List" Traceback (most recent call last): File "<string>", line 1, in <module> File "/usr/lib/python3.12/site-packages/rpds/__init__.py", line 1, in <module> from .rpds import * ModuleNotFoundError: No module named 'rpds.rpds' Where as it should have been: site-packages/rpds/rpds.cpython-312-arm-linux-gnueabihf.so Associated upstream bug report: PyO3/maturin#2203 Associated upstream pull request: PyO3/maturin#2204 Note - mitigation has not been tested with musl: PyO3/maturin#2204 (comment) Signed-off-by: Vesa Jääskeläinen <[email protected]> Signed-off-by: Niko Mauno <[email protected]> Signed-off-by: Richard Purdie <[email protected]>
openembedded · Sep 5, 2024 · cc347ce · cc347ce
1 parent 52fb076
commit cc347ce
Show file tree

Hide file tree

Showing 6 changed files with 438 additions and 0 deletions.
diff --git a/...ls/python/python3-maturin/0001-Extract-extension-architecture-name-resolvation-code.patch b/...ls/python/python3-maturin/0001-Extract-extension-architecture-name-resolvation-code.patch
@@ -0,0 +1,107 @@
+From 42a97ee7100ad158d4b1ba6133ea13cc864a567f Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
+ <[email protected]>
+Date: Sun, 1 Sep 2024 09:23:10 +0300
+Subject: [PATCH 1/5] Extract extension architecture name resolvation code as
+ helper
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This commit introduces helper InterpreterConfig.get_python_ext_arch() that
+can be used to determine the extension architecture name python uses in
+`ext_suffix` for this architecture.
+
+Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/42a97ee7100ad158d4b1ba6133ea13cc864a567f]
+
+Signed-off-by: Vesa Jääskeläinen <[email protected]>
+---
+ src/python_interpreter/config.rs | 18 ++++++------------
+ src/target.rs                    | 16 ++++++++++++++++
+ 2 files changed, 22 insertions(+), 12 deletions(-)
+
+diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
+index 912f9218..d76606f2 100644
+--- a/src/python_interpreter/config.rs
++++ b/src/python_interpreter/config.rs
+@@ -47,15 +47,7 @@ impl InterpreterConfig {
+             // Python 2 is not supported
+             return None;
+         }
+-        let python_arch = if matches!(target.target_arch(), Arch::Armv6L | Arch::Armv7L) {
+-            "arm"
+-        } else if matches!(target.target_arch(), Arch::Powerpc64Le) && python_impl == PyPy {
+-            "ppc_64"
+-        } else if matches!(target.target_arch(), Arch::X86) && python_impl == PyPy {
+-            "x86"
+-        } else {
+-            target.get_python_arch()
+-        };
++        let python_ext_arch = target.get_python_ext_arch(python_impl);
+         // See https://github.com/pypa/auditwheel/issues/349
+         let target_env = match python_impl {
+             CPython => {
+@@ -77,7 +69,7 @@ impl InterpreterConfig {
+                 let ldversion = format!("{}{}{}", major, minor, abiflags);
+                 let ext_suffix = format!(
+                     ".cpython-{}-{}-linux-{}.so",
+-                    ldversion, python_arch, target_env
++                    ldversion, python_ext_arch, target_env
+                 );
+                 Some(Self {
+                     major,
+@@ -90,7 +82,8 @@ impl InterpreterConfig {
+             }
+             (Os::Linux, PyPy) => {
+                 let abi_tag = format!("pypy{}{}-{}", major, minor, PYPY_ABI_TAG);
+-                let ext_suffix = format!(".{}-{}-linux-{}.so", abi_tag, python_arch, target_env);
++                let ext_suffix =
++                    format!(".{}-{}-linux-{}.so", abi_tag, python_ext_arch, target_env);
+                 Some(Self {
+                     major,
+                     minor,
+@@ -204,7 +197,8 @@ impl InterpreterConfig {
+             }
+             (Os::Emscripten, CPython) => {
+                 let ldversion = format!("{}{}", major, minor);
+-                let ext_suffix = format!(".cpython-{}-{}-emscripten.so", ldversion, python_arch);
++                let ext_suffix =
++                    format!(".cpython-{}-{}-emscripten.so", ldversion, python_ext_arch);
+                 Some(Self {
+                     major,
+                     minor,
+diff --git a/src/target.rs b/src/target.rs
+index dc7df0cf..84bae559 100644
+--- a/src/target.rs
++++ b/src/target.rs
+@@ -1,4 +1,5 @@
+ use crate::cross_compile::is_cross_compiling;
++use crate::python_interpreter::InterpreterKind;
+ use crate::PlatformTag;
+ use anyhow::{anyhow, bail, format_err, Result};
+ use platform_info::*;
+@@ -368,6 +369,21 @@ impl Target {
+         }
+     }
+
++    /// Returns the extension architecture name python uses in `ext_suffix` for this architecture.
++    pub fn get_python_ext_arch(&self, python_impl: InterpreterKind) -> &str {
++        if matches!(self.target_arch(), Arch::Armv6L | Arch::Armv7L) {
++            "arm"
++        } else if matches!(self.target_arch(), Arch::Powerpc64Le)
++            && python_impl == InterpreterKind::PyPy
++        {
++            "ppc_64"
++        } else if matches!(self.target_arch(), Arch::X86) && python_impl == InterpreterKind::PyPy {
++            "x86"
++        } else {
++            self.get_python_arch()
++        }
++    }
++
+     /// Returns the name python uses in `sys.platform` for this os
+     pub fn get_python_os(&self) -> &str {
+         match self.os {
+-- 
+2.34.1
+
diff --git a/...ls/python/python3-maturin/0002-Fix-cross-compilation-issue-with-linux-armv7l-archit.patch b/...ls/python/python3-maturin/0002-Fix-cross-compilation-issue-with-linux-armv7l-archit.patch
@@ -0,0 +1,76 @@
+From 0c6b8cc84eff72ed21098029aaba079b899dbee2 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
+ <[email protected]>
+Date: Sun, 1 Sep 2024 09:23:40 +0300
+Subject: [PATCH 2/5] Fix cross compilation issue with linux-armv7l
+ architecture
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+When compiling under Yocto project for linux-armv7l target architecture
+.so files were generated incorrectly as:
+
+  rpds.cpython-312-armv7l-linux-gnueabihf.so
+
+Where as platform and EXT_SUFFIX are defined as:
+
+  >>> sysconfig.get_platform()
+  'linux-armv7l'
+  >>> sysconfig.get_config_vars()['EXT_SUFFIX']
+  '.cpython-312-arm-linux-gnueabihf.so'
+
+Which should have caused the .so files as:
+
+  rpds.cpython-312-arm-linux-gnueabihf.so
+
+Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/0c6b8cc84eff72ed21098029aaba079b899dbee2]
+
+Signed-off-by: Vesa Jääskeläinen <[email protected]>
+---
+ src/python_interpreter/config.rs | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
+index d76606f2..5736aedc 100644
+--- a/src/python_interpreter/config.rs
++++ b/src/python_interpreter/config.rs
+@@ -306,7 +306,7 @@ impl InterpreterConfig {
+                     format!(
+                         ".cpython-{}-{}-{}-{}.{}",
+                         abi_tag,
+-                        target.get_python_arch(),
++                        target.get_python_ext_arch(interpreter_kind),
+                         target.get_python_os(),
+                         target_env,
+                         file_ext,
+@@ -319,7 +319,7 @@ impl InterpreterConfig {
+                         major,
+                         minor,
+                         abi_tag,
+-                        target.get_python_arch(),
++                        target.get_python_ext_arch(interpreter_kind),
+                         target.get_python_os(),
+                         target_env,
+                         file_ext,
+@@ -330,7 +330,7 @@ impl InterpreterConfig {
+                     format!(
+                         ".{}-{}-{}.{}",
+                         abi_tag.replace('_', "-"),
+-                        target.get_python_arch(),
++                        target.get_python_ext_arch(interpreter_kind),
+                         target.get_python_os(),
+                         file_ext,
+                     )
+@@ -341,7 +341,7 @@ impl InterpreterConfig {
+                 format!(
+                     ".cpython-{}-{}-{}.{}",
+                     abi_tag,
+-                    target.get_python_arch(),
++                    target.get_python_ext_arch(interpreter_kind),
+                     target.get_python_os(),
+                     file_ext
+                 )
+-- 
+2.34.1
+
diff --git a/...ls/python/python3-maturin/0003-Extract-extension-ABI-name-resolvation-code-as-helpe.patch b/...ls/python/python3-maturin/0003-Extract-extension-ABI-name-resolvation-code-as-helpe.patch
@@ -0,0 +1,98 @@
+From fa64426f3a98a0455721c23ec86bd2240708b45e Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
+ <[email protected]>
+Date: Sun, 1 Sep 2024 15:55:07 +0300
+Subject: [PATCH 3/5] Extract extension ABI name resolvation code as helper
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This commit introduces helper InterpreterConfig.get_python_target_env()
+that can be used to determine the extension ABI python uses in
+`ext_suffix` for this architecture.
+
+Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/fa64426f3a98a0455721c23ec86bd2240708b45e]
+
+Signed-off-by: Vesa Jääskeläinen <[email protected]>
+---
+ src/python_interpreter/config.rs | 19 ++-----------------
+ src/target.rs                    | 20 ++++++++++++++++++++
+ 2 files changed, 22 insertions(+), 17 deletions(-)
+
+diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
+index 5736aedc..938e9955 100644
+--- a/src/python_interpreter/config.rs
++++ b/src/python_interpreter/config.rs
+@@ -48,17 +48,7 @@ impl InterpreterConfig {
+             return None;
+         }
+         let python_ext_arch = target.get_python_ext_arch(python_impl);
+-        // See https://github.com/pypa/auditwheel/issues/349
+-        let target_env = match python_impl {
+-            CPython => {
+-                if python_version >= (3, 11) {
+-                    target.target_env().to_string()
+-                } else {
+-                    target.target_env().to_string().replace("musl", "gnu")
+-                }
+-            }
+-            PyPy | GraalPy => "gnu".to_string(),
+-        };
++        let target_env = target.get_python_target_env(python_impl, python_version);
+         match (target.target_os(), python_impl) {
+             (Os::Linux, CPython) => {
+                 let abiflags = if python_version < (3, 8) {
+@@ -294,12 +284,7 @@ impl InterpreterConfig {
+         };
+         let file_ext = if target.is_windows() { "pyd" } else { "so" };
+         let ext_suffix = if target.is_linux() || target.is_macos() {
+-            // See https://github.com/pypa/auditwheel/issues/349
+-            let target_env = if (major, minor) >= (3, 11) {
+-                target.target_env().to_string()
+-            } else {
+-                target.target_env().to_string().replace("musl", "gnu")
+-            };
++            let target_env = target.get_python_target_env(interpreter_kind, (major, minor));
+             match interpreter_kind {
+                 InterpreterKind::CPython => ext_suffix.unwrap_or_else(|| {
+                     // Eg: .cpython-38-x86_64-linux-gnu.so
+diff --git a/src/target.rs b/src/target.rs
+index 84bae559..ad8ebaba 100644
+--- a/src/target.rs
++++ b/src/target.rs
+@@ -1,5 +1,6 @@
+ use crate::cross_compile::is_cross_compiling;
+ use crate::python_interpreter::InterpreterKind;
++use crate::python_interpreter::InterpreterKind::{CPython, GraalPy, PyPy};
+ use crate::PlatformTag;
+ use anyhow::{anyhow, bail, format_err, Result};
+ use platform_info::*;
+@@ -384,6 +385,25 @@ impl Target {
+         }
+     }
+
++    /// Returns the environment python uses in `ext_suffix` for this architecture.
++    pub fn get_python_target_env(
++        &self,
++        python_impl: InterpreterKind,
++        python_version: (usize, usize),
++    ) -> String {
++        match python_impl {
++            CPython => {
++                // For musl handling see https://github.com/pypa/auditwheel/issues/349
++                if python_version >= (3, 11) {
++                    self.target_env().to_string()
++                } else {
++                    self.target_env().to_string().replace("musl", "gnu")
++                }
++            }
++            PyPy | GraalPy => "gnu".to_string(),
++        }
++    }
++
+     /// Returns the name python uses in `sys.platform` for this os
+     pub fn get_python_os(&self) -> &str {
+         match self.os {
+-- 
+2.34.1
+
diff --git a/...ls/python/python3-maturin/0004-Fix-cross-compilation-issue-with-linux-ppc-architect.patch b/...ls/python/python3-maturin/0004-Fix-cross-compilation-issue-with-linux-ppc-architect.patch
@@ -0,0 +1,68 @@
+From f2c892109a05db144e8b18bcbcf9c24fe8d977c4 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Vesa=20J=C3=A4=C3=A4skel=C3=A4inen?=
+ <[email protected]>
+Date: Sun, 1 Sep 2024 15:55:16 +0300
+Subject: [PATCH 4/5] Fix cross compilation issue with linux-ppc architecture
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+When compiling under Yocto project for linux-ppc target architecture
+.so files were generated incorrectly as:
+
+  rpds.cpython-312-ppc-linux-gnu.so
+
+Where as platform and EXT_SUFFIX are defined as:
+
+  >>> sysconfig.get_platform()
+  'linux-ppc'
+  >>> sysconfig.get_config_vars()['EXT_SUFFIX']
+  '.cpython-312-powerpc-linux-gnu.so'
+
+Which should have caused the .so files as:
+
+  rpds.cpython-312-powerpc-linux-gnu.so
+
+Upstream-Status: Backport [https://github.com/PyO3/maturin/commit/f2c892109a05db144e8b18bcbcf9c24fe8d977c4]
+
+Signed-off-by: Vesa Jääskeläinen <[email protected]>
+---
+ src/python_interpreter/config.rs | 8 ++++++++
+ src/target.rs                    | 2 ++
+ 2 files changed, 10 insertions(+)
+
+diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
+index 938e9955..8f883887 100644
+--- a/src/python_interpreter/config.rs
++++ b/src/python_interpreter/config.rs
+@@ -424,6 +424,14 @@ mod test {
+             ".cpython-310-powerpc64le-linux-gnu.so"
+         );
+
++        let sysconfig = InterpreterConfig::lookup_one(
++            &Target::from_target_triple(Some("powerpc-unknown-linux-gnu".to_string())).unwrap(),
++            InterpreterKind::CPython,
++            (3, 10),
++        )
++        .unwrap();
++        assert_eq!(sysconfig.ext_suffix, ".cpython-310-powerpc-linux-gnu.so");
++
+         let sysconfig = InterpreterConfig::lookup_one(
+             &Target::from_target_triple(Some("s390x-unknown-linux-gnu".to_string())).unwrap(),
+             InterpreterKind::CPython,
+diff --git a/src/target.rs b/src/target.rs
+index ad8ebaba..93afd9bb 100644
+--- a/src/target.rs
++++ b/src/target.rs
+@@ -380,6 +380,8 @@ impl Target {
+             "ppc_64"
+         } else if matches!(self.target_arch(), Arch::X86) && python_impl == InterpreterKind::PyPy {
+             "x86"
++        } else if matches!(self.target_arch(), Arch::Powerpc) {
++            "powerpc"
+         } else {
+             self.get_python_arch()
+         }
+-- 
+2.34.1
+