From c467d9eca35fc1404a2c73022df1f7bc2e23747f Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 1 Jan 2025 21:56:14 +0800 Subject: [PATCH] Add an API to set rpath when using macOS system Python --- build.rs | 7 ++- guide/src/building-and-distribution.md | 22 +++---- newsfragments/4833.added.md | 1 + pyo3-build-config/src/impl_.rs | 37 ++++++++++++ pyo3-build-config/src/lib.rs | 83 ++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 newsfragments/4833.added.md diff --git a/build.rs b/build.rs index 5d638291f3b..68a658bf285 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,9 @@ use std::env; use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; -use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; +use pyo3_build_config::{ + add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, +}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared { @@ -42,6 +44,9 @@ fn configure_pyo3() -> Result<()> { // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); + // Make `cargo test` etc work on macOS with Xcode bundled Python + add_python_framework_link_args(); + Ok(()) } diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 1a806304d22..d3474fedaf7 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -144,7 +144,17 @@ rustflags = [ ] ``` -Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. These can be resolved with another addition to `.cargo/config.toml`: +Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. + +The easiest way to set the correct linker arguments is to add a `build.rs` with the following content: + +```rust,ignore +fn main() { + pyo3_build_config::add_python_framework_link_args(); +} +``` + +Alternatively it can be resolved with another addition to `.cargo/config.toml`: ```toml [build] @@ -153,16 +163,6 @@ rustflags = [ ] ``` -Alternatively, one can include in `build.rs`: - -```rust -fn main() { - println!( - "cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks" - ); -} -``` - For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649). Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). diff --git a/newsfragments/4833.added.md b/newsfragments/4833.added.md new file mode 100644 index 00000000000..4e1e0005305 --- /dev/null +++ b/newsfragments/4833.added.md @@ -0,0 +1 @@ +Add `pyo3_build_config::add_python_framework_link_args` build script API to set rpath when using macOS system Python. diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 6a7bb03fcb2..05bb58ac7aa 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -167,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -245,6 +247,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -253,6 +256,7 @@ print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -289,6 +293,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) }; let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -359,6 +364,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -396,6 +402,9 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { Some(value) => value == "1", @@ -424,6 +433,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -500,6 +510,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -528,6 +539,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -558,6 +570,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } @@ -650,6 +663,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { writeln!(writer, "extra_build_script_line={}", line) @@ -1587,6 +1601,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2039,6 +2056,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2060,6 +2078,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2086,6 +2105,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2108,6 +2128,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2210,6 +2231,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2239,6 +2261,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2265,6 +2288,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2288,6 +2312,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2311,6 +2336,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2345,6 +2371,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2379,6 +2406,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2413,6 +2441,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2449,6 +2478,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2796,6 +2826,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2818,6 +2849,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2882,6 +2914,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -3006,6 +3039,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), @@ -3045,6 +3079,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3092,6 +3127,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3125,6 +3161,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 420b81c738b..9070f6d7401 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -74,6 +74,44 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap(); + _add_python_framework_link_args( + &interpreter_config, + &impl_::target_triple_from_env(), + impl_::is_linking_libpython(), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!( + writer, + "cargo:rustc-link-arg=-Wl,-rpath,{}", + framework_prefix + ) + .unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. @@ -306,4 +344,49 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } }