diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
index d38d41ed552..2d815d78011 100644
--- a/pyo3-build-config/src/impl_.rs
+++ b/pyo3-build-config/src/impl_.rs
@@ -1404,7 +1404,10 @@ fn cross_compile_from_sysconfigdata(
 ) -> Result<Option<InterpreterConfig>> {
     if let Some(path) = find_sysconfigdata(cross_compile_config)? {
         let data = parse_sysconfigdata(path)?;
-        let config = InterpreterConfig::from_sysconfigdata(&data)?;
+        let mut config = InterpreterConfig::from_sysconfigdata(&data)?;
+        if let Some(cross_lib_dir) = cross_compile_config.lib_dir_string() {
+            config.lib_dir = Some(cross_lib_dir)
+        }
 
         Ok(Some(config))
     } else {