From ed994ca44f8cc7894e760b89281328f8f1d2693e Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 3 Aug 2021 21:17:17 +0100 Subject: [PATCH 1/5] pyo3-build-config: improve config file format --- build.rs | 183 +++++++--------- pyo3-build-config/src/impl_.rs | 375 +++++++++++++++++++++------------ pyo3-build-config/src/lib.rs | 13 +- 3 files changed, 331 insertions(+), 240 deletions(-) diff --git a/build.rs b/build.rs index c4ac6bf9623..c7f8ed05cbf 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,9 @@ -use std::{env, process::Command}; +use std::{env, ffi::OsString, path::Path, process::Command}; use pyo3_build_config::{ bail, cargo_env_var, ensure, env_var, errors::{Context, Result}, - InterpreterConfig, PythonImplementation, PythonVersion, + InterpreterConfig, PythonVersion, }; /// Minimum Python version PyO3 supports. @@ -20,80 +20,27 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { Ok(()) } -fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { +fn ensure_target_pointer_width(pointer_width: u32) -> Result<()> { // Try to check whether the target architecture matches the python library let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") .unwrap() .as_str() { - "64" => "64-bit", - "32" => "32-bit", + "64" => 64, + "32" => 32, x => bail!("unexpected Rust target pointer width: {}", x), }; - // The reason we don't use platform.architecture() here is that it's not - // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. - // Similarly, sys.maxsize is not reliable on Windows. See - // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 - // and https://stackoverflow.com/a/3411134/823869. - let python_target = match interpreter_config.calcsize_pointer { - Some(8) => "64-bit", - Some(4) => "32-bit", - None => { - // Unset, e.g. because we're cross-compiling. Don't check anything - // in this case. - return Ok(()); - } - Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), - }; - ensure!( - rust_target == python_target, - "Your Rust target architecture ({}) does not match your python interpreter ({})", + rust_target == pointer_width, + "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", rust_target, - python_target + pointer_width ); Ok(()) } -fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { - let link_name = if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { - if config.abi3 { - // Link against python3.lib for the stable ABI on Windows. - // See https://www.python.org/dev/peps/pep-0384/#linkage - // - // This contains only the limited ABI symbols. - "pythonXY:python3".to_owned() - } else if cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" { - // https://packages.msys2.org/base/mingw-w64-python - format!( - "pythonXY:python{}.{}", - config.version.major, config.version.minor - ) - } else { - format!( - "pythonXY:python{}{}", - config.version.major, config.version.minor - ) - } - } else { - match config.implementation { - PythonImplementation::CPython => match &config.ld_version { - Some(ld_version) => format!("python{}", ld_version), - None => bail!("failed to configure `ld_version` when compiling for unix"), - }, - PythonImplementation::PyPy => format!("pypy{}-c", config.version.major), - } - }; - - Ok(format!( - "cargo:rustc-link-lib={link_model}{link_name}", - link_model = if config.shared { "" } else { "static=" }, - link_name = link_name - )) -} - fn rustc_minor_version() -> Option { let rustc = env::var_os("RUSTC")?; let output = Command::new(rustc).arg("--version").output().ok()?; @@ -108,28 +55,29 @@ fn rustc_minor_version() -> Option { fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()> { let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); - match (is_extension_module, target_os.as_str()) { - (_, "windows") => { - // always link on windows, even with extension module - println!("{}", get_rustc_link_lib(interpreter_config)?); - // Set during cross-compiling. - if let Some(libdir) = &interpreter_config.libdir { - println!("cargo:rustc-link-search=native={}", libdir); - } - // Set if we have an interpreter to use. - if let Some(base_prefix) = &interpreter_config.base_prefix { - println!("cargo:rustc-link-search=native={}\\libs", base_prefix); - } - } - (false, _) | (_, "android") => { - // other systems, only link libs if not extension module - // android always link. - println!("{}", get_rustc_link_lib(interpreter_config)?); - if let Some(libdir) = &interpreter_config.libdir { - println!("cargo:rustc-link-search=native={}", libdir); - } + if target_os == "windows" || target_os == "android" || !is_extension_module { + // windows and android - always link + // other systems - only link if not extension module + println!( + "cargo:rustc-link-lib={link_model}{alias}{lib_name}", + link_model = if interpreter_config.shared { + "" + } else { + "static=" + }, + alias = if target_os == "windows" { + "pythonXY:" + } else { + "" + }, + lib_name = interpreter_config + .lib_name + .as_ref() + .ok_or("config does not contain lib_name")?, + ); + if let Some(lib_dir) = &interpreter_config.lib_dir { + println!("cargo:rustc-link-search=native={}", lib_dir); } - _ => {} } if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { @@ -152,9 +100,9 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies` // currently cause `auto-initialize` to be enabled in CI. - // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this. + // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead. if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() { - bail!("The `auto-initialize` feature is not supported with PyPy."); + bail!("the `auto-initialize` feature is not supported with PyPy"); } } @@ -166,21 +114,56 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() /// The result is written to pyo3_build_config::PATH, which downstream scripts can read from /// (including `pyo3-macros-backend` during macro expansion). fn configure_pyo3() -> Result<()> { - let interpreter_config = pyo3_build_config::make_interpreter_config()?; + let write_config_file = env_var("PYO3_WRITE_CONFIG_FILE").map_or(false, |os_str| os_str == "1"); + let custom_config_file_path = env_var("PYO3_CONFIG_FILE"); + if let Some(path) = &custom_config_file_path { + ensure!( + Path::new(path).is_absolute(), + "PYO3_CONFIG_FILE must be absolute" + ); + } + let (interpreter_config, path_to_write) = match (write_config_file, custom_config_file_path) { + (true, Some(path)) => { + // Create new interpreter config and write it to config file + (pyo3_build_config::make_interpreter_config()?, Some(path)) + } + (true, None) => bail!("PYO3_CONFIG_FILE must be set when PYO3_WRITE_CONFIG_FILE is set"), + (false, Some(path)) => { + // Read custom config file + let path = Path::new(&path); + println!("cargo:rerun-if-changed={}", path.display()); + let config_file = std::fs::File::open(path) + .with_context(|| format!("failed to read config file at {}", path.display()))?; + let reader = std::io::BufReader::new(config_file); + ( + pyo3_build_config::InterpreterConfig::from_reader(reader)?, + None, + ) + } + (false, None) => ( + // Create new interpreter config and write it to the default location + pyo3_build_config::make_interpreter_config()?, + Some(OsString::from(pyo3_build_config::DEFAULT_CONFIG_PATH)), + ), + }; + + if let Some(path) = path_to_write { + interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context(|| { + format!( + "failed to create config file at {}", + Path::new(&path).display() + ) + })?)?; + } if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { print_config_and_exit(&interpreter_config); } + ensure_python_version(&interpreter_config)?; - ensure_target_architecture(&interpreter_config)?; + if let Some(pointer_width) = interpreter_config.pointer_width { + ensure_target_pointer_width(pointer_width)?; + } emit_cargo_configuration(&interpreter_config)?; - interpreter_config.to_writer( - &mut std::fs::File::create(pyo3_build_config::PATH).with_context(|| { - format!( - "failed to create config file at {}", - pyo3_build_config::PATH - ) - })?, - )?; interpreter_config.emit_pyo3_cfgs(); let rustc_minor_version = rustc_minor_version().unwrap_or(0); @@ -200,15 +183,9 @@ fn configure_pyo3() -> Result<()> { fn print_config_and_exit(config: &InterpreterConfig) { println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --"); - println!("implementation: {}", config.implementation); - println!("interpreter version: {}", config.version); - println!("interpreter path: {:?}", config.executable); - println!("libdir: {:?}", config.libdir); - println!("shared: {}", config.shared); - println!("base prefix: {:?}", config.base_prefix); - println!("ld_version: {:?}", config.ld_version); - println!("pointer width: {:?}", config.calcsize_pointer); - + config + .to_writer(&mut std::io::stdout()) + .expect("failed to print config to stdout"); std::process::exit(101); } diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index a8ac9972cab..7b6594d8548 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -43,15 +43,14 @@ pub fn env_var(var: &str) -> Option { /// this type. #[cfg_attr(test, derive(Debug, PartialEq))] pub struct InterpreterConfig { + pub implementation: PythonImplementation, pub version: PythonVersion, - pub libdir: Option, pub shared: bool, pub abi3: bool, - pub ld_version: Option, - pub base_prefix: Option, + pub lib_name: Option, + pub lib_dir: Option, pub executable: Option, - pub calcsize_pointer: Option, - pub implementation: PythonImplementation, + pub pointer_width: Option, pub build_flags: BuildFlags, } @@ -89,66 +88,85 @@ impl InterpreterConfig { #[doc(hidden)] pub fn from_reader(reader: impl Read) -> Result { let reader = BufReader::new(reader); - let mut lines = reader.lines(); - - macro_rules! parse_line { - ($value:literal) => { - lines - .next() - .ok_or(concat!("reached end of config when reading ", $value))? - .context(concat!("failed to read ", $value, " from config"))? - .parse() - .context(concat!("failed to parse ", $value, " from config")) + let lines = reader.lines(); + + macro_rules! parse_value { + ($variable:ident, $value:ident) => { + $variable = Some($value.parse().context(format!( + concat!( + "failed to parse ", + stringify!($variable), + " from config value '{}'" + ), + $value + ))?) }; } - macro_rules! parse_option_line { - ($value:literal) => { - parse_option_string( - lines - .next() - .ok_or(concat!("reached end of config when reading ", $value))? - .context(concat!("failed to read ", $value, " from config"))?, - ) - .context(concat!("failed to parse ", $value, "from config")) - }; + let mut implementation = None; + let mut version = None; + let mut shared = None; + let mut abi3 = None; + let mut lib_name = None; + let mut lib_dir = None; + let mut executable = None; + let mut pointer_width = None; + let mut build_flags = None; + + for (i, line) in lines.enumerate() { + let line = line.context("failed to read line from config")?; + let mut split = line.splitn(2, '='); + let (key, value) = ( + split + .next() + .expect("first splitn value should always be present"), + split + .next() + .ok_or_else(|| format!("expected key=value pair on line {}", i + 1))?, + ); + match key { + "implementation" => parse_value!(implementation, value), + "version" => parse_value!(version, value), + "shared" => parse_value!(shared, value), + "abi3" => parse_value!(abi3, value), + "lib_name" => parse_value!(lib_name, value), + "lib_dir" => parse_value!(lib_dir, value), + "executable" => parse_value!(executable, value), + "pointer_width" => parse_value!(pointer_width, value), + "build_flags" => parse_value!(build_flags, value), + unknown => bail!("unknown config key `{}`", unknown), + } } - let major = parse_line!("major version")?; - let minor = parse_line!("minor version")?; - let libdir = parse_option_line!("libdir")?; - let shared = parse_line!("shared")?; - let abi3 = parse_line!("abi3")?; - let ld_version = parse_option_line!("ld_version")?; - let base_prefix = parse_option_line!("base_prefix")?; - let executable = parse_option_line!("executable")?; - let calcsize_pointer = parse_option_line!("calcsize_pointer")?; - let implementation = parse_line!("implementation")?; - let mut build_flags = BuildFlags(HashSet::new()); - for line in lines { - build_flags - .0 - .insert(line.context("failed to read flag from config")?.parse()?); - } + let version = version.ok_or("missing value for version")?; + let implementation = implementation.unwrap_or(PythonImplementation::CPython); + let abi3 = abi3.unwrap_or(false); + Ok(InterpreterConfig { - version: PythonVersion { major, minor }, - libdir, - shared, + implementation, + version, + shared: shared.unwrap_or(true), abi3, - ld_version, - base_prefix, + lib_name, + lib_dir, executable, - calcsize_pointer, - implementation, - build_flags, + pointer_width, + build_flags: build_flags.unwrap_or_else(|| { + if abi3 { + BuildFlags::abi3() + } else { + BuildFlags(HashSet::new()) + } + .fixup(version, implementation) + }), }) } #[doc(hidden)] pub fn to_writer(&self, mut writer: impl Write) -> Result<()> { macro_rules! write_line { - ($value:expr) => { - writeln!(writer, "{}", $value).context(concat!( + ($value:ident) => { + writeln!(writer, "{}={}", stringify!($value), self.$value).context(concat!( "failed to write ", stringify!($value), " to config" @@ -157,44 +175,32 @@ impl InterpreterConfig { } macro_rules! write_option_line { - ($opt:expr) => { - match &$opt { - Some(value) => writeln!(writer, "{}", value), - None => writeln!(writer, "null"), + ($value:ident) => { + if let Some(value) = &self.$value { + writeln!(writer, "{}={}", stringify!($value), value).context(concat!( + "failed to write ", + stringify!($value), + " to config" + )) + } else { + Ok(()) } - .context(concat!( - "failed to write ", - stringify!($value), - " to config" - )) }; } - write_line!(self.version.major)?; - write_line!(self.version.minor)?; - write_option_line!(self.libdir)?; - write_line!(self.shared)?; - write_line!(self.abi3)?; - write_option_line!(self.ld_version)?; - write_option_line!(self.base_prefix)?; - write_option_line!(self.executable)?; - write_option_line!(self.calcsize_pointer)?; - write_line!(self.implementation)?; - for flag in &self.build_flags.0 { - write_line!(flag)?; - } + write_line!(implementation)?; + write_line!(version)?; + write_line!(shared)?; + write_line!(abi3)?; + write_option_line!(lib_name)?; + write_option_line!(lib_dir)?; + write_option_line!(executable)?; + write_option_line!(pointer_width)?; + write_line!(build_flags)?; Ok(()) } } -fn parse_option_string(string: String) -> Result, ::Err> { - if string == "null" { - Ok(None) - } else { - string.parse().map(Some) - } -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct PythonVersion { pub major: u8, @@ -211,6 +217,24 @@ impl Display for PythonVersion { } } +impl FromStr for PythonVersion { + type Err = crate::errors::Error; + + fn from_str(value: &str) -> Result { + let mut split = value.splitn(2, '.'); + let (major, minor) = ( + split + .next() + .expect("first splitn value should always be present"), + split.next().ok_or("expected major.minor version")?, + ); + Ok(Self { + major: major.parse().context("failed to parse major version")?, + minor: minor.parse().context("failed to parse minor version")?, + }) + } +} + #[derive(Debug, Copy, Clone, PartialEq)] pub enum PythonImplementation { CPython, @@ -453,14 +477,6 @@ impl BuildFlags { // query the interpreter directly for its build flags. let mut flags = HashSet::new(); flags.insert(BuildFlag::WITH_THREAD); - - // Uncomment these manually if your python was built with these and you want - // the cfg flags to be set in rust. - // - // flags.insert(BuildFlag::Py_DEBUG); - // flags.insert(BuildFlag::Py_REF_DEBUG); - // flags.insert(BuildFlag::Py_TRACE_REFS); - // flags.insert(BuildFlag::COUNT_ALLOCS; Self(flags) } @@ -488,6 +504,33 @@ impl BuildFlags { } } +impl Display for BuildFlags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut first = true; + for flag in &self.0 { + if !first { + write!(f, ",")?; + } else { + first = false; + } + write!(f, "{}", flag)?; + } + Ok(()) + } +} + +impl FromStr for BuildFlags { + type Err = std::convert::Infallible; + + fn from_str(value: &str) -> Result { + let mut flags = HashSet::new(); + for flag in value.split(',') { + flags.insert(flag.parse().unwrap()); + } + Ok(BuildFlags(flags)) + } +} + fn parse_script_output(output: &str) -> HashMap { output .lines() @@ -675,11 +718,10 @@ fn load_cross_compile_from_sysconfigdata( let major = sysconfig_data.get_numeric("version_major")?; let minor = sysconfig_data.get_numeric("version_minor")?; - let ld_version = match sysconfig_data.get("LDVERSION") { - Some(s) => s.clone(), - None => format!("{}.{}", major, minor), - }; - let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok(); + let pointer_width = sysconfig_data + .get_numeric("SIZEOF_VOID_P") + .map(|bytes_width: u32| bytes_width * 8) + .ok(); let soabi = match sysconfig_data.get("SOABI") { Some(s) => s, None => bail!("sysconfigdata did not define SOABI"), @@ -695,15 +737,18 @@ fn load_cross_compile_from_sysconfigdata( }; Ok(InterpreterConfig { + implementation, version, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?, abi3: is_abi3(), - ld_version: Some(ld_version), - base_prefix: None, + lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), + lib_name: Some(default_lib_name_unix( + &version, + implementation, + sysconfig_data.get("LDVERSION").map(String::as_str), + )?), executable: None, - calcsize_pointer, - implementation, + pointer_width, build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation), }) } @@ -730,16 +775,17 @@ fn windows_hardcoded_cross_compile( bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.") }; + let version = PythonVersion { major, minor }; + Ok(InterpreterConfig { - version: PythonVersion { major, minor }, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), + implementation: PythonImplementation::CPython, + version, shared: true, abi3: is_abi3(), - ld_version: None, - base_prefix: None, + lib_name: Some(default_lib_name_windows(&version, false, "msvc")), + lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), executable: None, - calcsize_pointer: None, - implementation: PythonImplementation::CPython, + pointer_width: None, build_flags: BuildFlags::windows_hardcoded(), }) } @@ -762,6 +808,37 @@ fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result String { + if abi3 { + WINDOWS_ABI3_LIB_NAME.to_owned() + } else if target_env == "gnu" { + // https://packages.msys2.org/base/mingw-w64-python + format!("python{}.{}", version.major, version.minor) + } else { + format!("python{}{}", version.major, version.minor) + } +} + +fn default_lib_name_unix( + version: &PythonVersion, + implementation: PythonImplementation, + ld_version: Option<&str>, +) -> Result { + match implementation { + PythonImplementation::CPython => match &ld_version { + Some(ld_version) => Ok(format!("python{}", ld_version)), + None => bail!("failed to configure `ld_version` when compiling for unix"), + }, + PythonImplementation::PyPy => Ok(format!("pypy{}-c", version.major)), + } +} + /// Run a python script using the specified interpreter binary. fn run_python_script(interpreter: &Path, script: &str) -> Result { let out = Command::new(interpreter) @@ -878,13 +955,13 @@ FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) +print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) -print("implementation", platform.python_implementation()) -print_if_set("libdir", get_config_var("LIBDIR")) +print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) print_if_set("ld_version", get_config_var("LDVERSION")) +print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) -print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) "#; @@ -901,22 +978,48 @@ print("calcsize_pointer", struct.calcsize("P")) .context("failed to parse minor version")?, }; + let abi3 = is_abi3(); let implementation = map["implementation"].parse()?; + let lib_name = if cfg!(windows) { + default_lib_name_windows( + &version, + abi3, + &cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(), + ) + } else { + default_lib_name_unix( + &version, + implementation, + map.get("ld_version").map(String::as_str), + )? + }; + + let lib_dir = if cfg!(windows) { + map.get("base_prefix") + .map(|base_prefix| format!("cargo:rustc-link-search=native={}\\libs", base_prefix)) + } else { + map.get("libdir").cloned() + }; + + // The reason we don't use platform.architecture() here is that it's not + // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. + // Similarly, sys.maxsize is not reliable on Windows. See + // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 + // and https://stackoverflow.com/a/3411134/823869. + let calcsize_pointer: u32 = map["calcsize_pointer"] + .parse() + .context("failed to parse calcsize_pointer")?; + Ok(InterpreterConfig { version, implementation, - libdir: map.get("libdir").cloned(), shared, - abi3: is_abi3(), - ld_version: map.get("ld_version").cloned(), - base_prefix: map.get("base_prefix").cloned(), + abi3, + lib_name: Some(lib_name), + lib_dir, executable: map.get("executable").cloned(), - calcsize_pointer: Some( - map["calcsize_pointer"] - .parse() - .context("failed to parse calcsize_pointer")?, - ), + pointer_width: Some(calcsize_pointer * 8), build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), }) } @@ -932,19 +1035,25 @@ pub fn make_interpreter_config() -> Result { // If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python. if let Some(abi3_minor_version) = abi3_version { if env_var("PYO3_NO_PYTHON").is_some() { + let version = PythonVersion { + major: 3, + minor: abi3_minor_version, + }; + let implementation = PythonImplementation::CPython; + let lib_name = if cfg!(windows) { + Some(WINDOWS_ABI3_LIB_NAME.to_owned()) + } else { + None + }; return Ok(InterpreterConfig { - version: PythonVersion { - major: 3, - minor: abi3_minor_version, - }, - implementation: PythonImplementation::CPython, + version, + implementation, abi3: true, - libdir: None, + lib_name, + lib_dir: None, build_flags: BuildFlags::abi3(), - base_prefix: None, - calcsize_pointer: None, + pointer_width: None, executable: None, - ld_version: None, shared: true, }); } @@ -981,13 +1090,12 @@ mod tests { fn test_read_write_roundtrip() { let config = InterpreterConfig { abi3: true, - base_prefix: Some("base_prefix".into()), build_flags: BuildFlags::abi3(), - calcsize_pointer: Some(32), + pointer_width: Some(32), executable: Some("executable".into()), implementation: PythonImplementation::CPython, - ld_version: Some("ld_version".into()), - libdir: Some("libdir".into()), + lib_name: Some("lib_name".into()), + lib_dir: Some("lib_dir".into()), shared: true, version: MINIMUM_SUPPORTED_VERSION, }; @@ -1003,18 +1111,17 @@ mod tests { let config = InterpreterConfig { abi3: false, - base_prefix: None, build_flags: { let mut flags = HashSet::new(); flags.insert(BuildFlag::Py_DEBUG); flags.insert(BuildFlag::Other(String::from("Py_SOME_FLAG"))); BuildFlags(flags) }, - calcsize_pointer: None, + pointer_width: None, executable: None, implementation: PythonImplementation::PyPy, - ld_version: None, - libdir: None, + lib_dir: None, + lib_name: None, shared: true, version: PythonVersion { major: 3, diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 0bc947aabed..5a974923d92 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -7,6 +7,8 @@ pub mod errors; mod impl_; +use std::{ffi::OsString, path::Path}; + use once_cell::sync::OnceCell; // Used in PyO3's build.rs @@ -23,15 +25,20 @@ pub use impl_::{ pub fn get() -> &'static InterpreterConfig { static CONFIG: OnceCell = OnceCell::new(); CONFIG.get_or_init(|| { - let config_file = std::fs::File::open(PATH).expect("config file missing"); + let config_path = std::env::var_os("PYO3_CONFIG_FILE") + .unwrap_or_else(|| OsString::from(DEFAULT_CONFIG_PATH)); + let config_file = std::fs::File::open(DEFAULT_CONFIG_PATH).expect(&format!( + "failed to open PyO3 config file at {}", + Path::new(&config_path).display() + )); let reader = std::io::BufReader::new(config_file); InterpreterConfig::from_reader(reader).expect("failed to parse config file") }) } -/// Path where PyO3's build.rs will write configuration. +/// Path where PyO3's build.rs will write configuration by default. #[doc(hidden)] -pub const PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"); +pub const DEFAULT_CONFIG_PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"); /// Adds all the [`#[cfg]` flags](index.html) to the current compilation. /// From 20b34a5528aa1bb79d7a61c3e5d296e3773d37bc Mon Sep 17 00:00:00 2001 From: Tom Milligan Date: Mon, 2 Aug 2021 17:38:18 +0100 Subject: [PATCH 2/5] create pyo3 config file directory if not exist --- build.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/build.rs b/build.rs index c7f8ed05cbf..9674e217dd4 100644 --- a/build.rs +++ b/build.rs @@ -148,12 +148,23 @@ fn configure_pyo3() -> Result<()> { }; if let Some(path) = path_to_write { - interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context(|| { + let path = Path::new(&path); + let parent_dir = path.parent().ok_or_else(|| { format!( - "failed to create config file at {}", - Path::new(&path).display() + "failed to resolve parent directory of config file {}", + path.display() ) - })?)?; + })?; + std::fs::create_dir_all(&parent_dir).with_context(|| { + format!( + "failed to create config file directory {}", + parent_dir.display() + ) + })?; + interpreter_config + .to_writer(&mut std::fs::File::create(&path).with_context(|| { + format!("failed to create config file at {}", path.display()) + })?)?; } if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { print_config_and_exit(&interpreter_config); From 9507979d934b7ca3b46a0da30674dcf81126c92f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 4 Aug 2021 12:54:45 +0100 Subject: [PATCH 3/5] pyo3-build-config: inline config when not cross compiling --- build.rs | 67 +++++++++------------------------ pyo3-build-config/build.rs | 31 ++++++++++++++- pyo3-build-config/src/errors.rs | 28 ++++++++++++++ pyo3-build-config/src/impl_.rs | 50 +++++++++++++++++++++--- pyo3-build-config/src/lib.rs | 26 +++++++------ 5 files changed, 135 insertions(+), 67 deletions(-) diff --git a/build.rs b/build.rs index 9674e217dd4..714ca0c3803 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,9 @@ -use std::{env, ffi::OsString, path::Path, process::Command}; +use std::{env, io::Cursor, path::Path, process::Command}; use pyo3_build_config::{ bail, cargo_env_var, ensure, env_var, errors::{Context, Result}, - InterpreterConfig, PythonVersion, + make_cross_compile_config, InterpreterConfig, PythonVersion, }; /// Minimum Python version PyO3 supports. @@ -114,41 +114,16 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() /// The result is written to pyo3_build_config::PATH, which downstream scripts can read from /// (including `pyo3-macros-backend` during macro expansion). fn configure_pyo3() -> Result<()> { - let write_config_file = env_var("PYO3_WRITE_CONFIG_FILE").map_or(false, |os_str| os_str == "1"); - let custom_config_file_path = env_var("PYO3_CONFIG_FILE"); - if let Some(path) = &custom_config_file_path { - ensure!( - Path::new(path).is_absolute(), - "PYO3_CONFIG_FILE must be absolute" - ); - } - let (interpreter_config, path_to_write) = match (write_config_file, custom_config_file_path) { - (true, Some(path)) => { - // Create new interpreter config and write it to config file - (pyo3_build_config::make_interpreter_config()?, Some(path)) - } - (true, None) => bail!("PYO3_CONFIG_FILE must be set when PYO3_WRITE_CONFIG_FILE is set"), - (false, Some(path)) => { - // Read custom config file - let path = Path::new(&path); - println!("cargo:rerun-if-changed={}", path.display()); - let config_file = std::fs::File::open(path) - .with_context(|| format!("failed to read config file at {}", path.display()))?; - let reader = std::io::BufReader::new(config_file); - ( - pyo3_build_config::InterpreterConfig::from_reader(reader)?, - None, - ) - } - (false, None) => ( - // Create new interpreter config and write it to the default location - pyo3_build_config::make_interpreter_config()?, - Some(OsString::from(pyo3_build_config::DEFAULT_CONFIG_PATH)), - ), - }; - - if let Some(path) = path_to_write { + let interpreter_config = if let Some(path) = env_var("PYO3_CONFIG_FILE") { let path = Path::new(&path); + // This is necessary because the compilations that access PYO3_CONFIG_FILE (build scripts, + // proc macros) have many different working directories, so a relative path is no good. + ensure!(path.is_absolute(), "PYO3_CONFIG_FILE must be an absolute path"); + println!("cargo:rerun-if-changed={}", path.display()); + InterpreterConfig::from_path(path)? + } else if let Some(interpreter_config) = make_cross_compile_config()? { + // This is a cross compile, need to write the config file. + let path = Path::new(&pyo3_build_config::DEFAULT_CROSS_COMPILE_CONFIG_PATH); let parent_dir = path.parent().ok_or_else(|| { format!( "failed to resolve parent directory of config file {}", @@ -165,7 +140,11 @@ fn configure_pyo3() -> Result<()> { .to_writer(&mut std::fs::File::create(&path).with_context(|| { format!("failed to create config file at {}", path.display()) })?)?; - } + interpreter_config + } else { + InterpreterConfig::from_reader(Cursor::new(pyo3_build_config::HOST_CONFIG))? + }; + if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { print_config_and_exit(&interpreter_config); } @@ -201,20 +180,8 @@ fn print_config_and_exit(config: &InterpreterConfig) { } fn main() { - // Print out error messages using display, to get nicer formatting. if let Err(e) = configure_pyo3() { - use std::error::Error; - eprintln!("error: {}", e); - let mut source = e.source(); - if source.is_some() { - eprintln!("caused by:"); - let mut index = 0; - while let Some(some_source) = source { - eprintln!(" - {}: {}", index, some_source); - source = some_source.source(); - index += 1; - } - } + eprintln!("error: {}", e.report()); std::process::exit(1) } } diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs index b94555bb929..ae23362e2fa 100644 --- a/pyo3-build-config/build.rs +++ b/pyo3-build-config/build.rs @@ -1,3 +1,32 @@ +// Import some modules from this crate inline to generate the build config. +// Allow dead code because not all code in the modules is used in this build script. + +#[path = "src/impl_.rs"] +#[allow(dead_code)] +mod impl_; + +#[path = "src/errors.rs"] +#[allow(dead_code)] +mod errors; + +use std::{env, path::Path}; + +use errors::{Result, Context}; + +fn generate_build_config() -> Result<()> { + // Create new interpreter config and write it to the default location + let interpreter_config = impl_::make_interpreter_config()?; + + let path = Path::new(&env::var_os("OUT_DIR").unwrap()).join("pyo3-build-config.txt"); + interpreter_config + .to_writer(&mut std::fs::File::create(&path).with_context(|| { + format!("failed to create config file at {}", path.display()) + })?) +} + fn main() { - // Empty build script to force cargo to produce the "OUT_DIR" environment variable. + if let Err(e) = generate_build_config() { + eprintln!("error: {}", e.report()); + std::process::exit(1) + } } diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 8d2903a3ef4..dfa3f837377 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -26,6 +26,16 @@ pub struct Error { source: Option>, } +/// Error report inspired by +/// https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter +pub struct ErrorReport<'a>(&'a Error); + +impl Error { + pub fn report(&self) -> ErrorReport<'_> { + ErrorReport(self) + } +} + impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.value) @@ -38,6 +48,24 @@ impl std::error::Error for Error { } } +impl std::fmt::Display for ErrorReport<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use std::error::Error; + self.0.fmt(f)?; + let mut source = self.0.source(); + if source.is_some() { + writeln!(f, "\ncaused by:")?; + let mut index = 0; + while let Some(some_source) = source { + writeln!(f, " - {}: {}", index, some_source)?; + source = some_source.source(); + index += 1; + } + } + Ok(()) + } +} + impl From for Error { fn from(value: String) -> Self { Self { diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 7b6594d8548..e1834269388 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -85,6 +85,20 @@ impl InterpreterConfig { self.implementation == PythonImplementation::PyPy } + #[doc(hidden)] + pub fn from_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + let config_file = + std::fs::File::open(path).with_context(|| { + format!( + "failed to open PyO3 config file at {}", + path.display() + ) + })?; + let reader = std::io::BufReader::new(config_file); + InterpreterConfig::from_reader(reader) + } + #[doc(hidden)] pub fn from_reader(reader: impl Read) -> Result { let reader = BufReader::new(reader); @@ -303,6 +317,12 @@ struct CrossCompileConfig { arch: String, } +pub fn any_cross_compiling_env_vars_set() -> bool { + env::var_os("PYO3_CROSS").is_some() + || env::var_os("PYO3_CROSS_LIB_DIR").is_some() + || env::var_os("PYO3_CROSS_PYTHON_VERSION").is_some() +} + fn cross_compiling() -> Result> { let cross = env_var("PYO3_CROSS"); let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR"); @@ -1029,6 +1049,30 @@ fn get_abi3_minor_version() -> Option { .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) } +pub fn make_cross_compile_config() -> Result> { + let abi3_version = get_abi3_minor_version(); + + let mut interpreter_config = if let Some(paths) = cross_compiling()? { + load_cross_compile_info(paths)? + } else { + return Ok(None); + }; + + // Fixup minor version if abi3-pyXX feature set + if let Some(abi3_minor_version) = abi3_version { + ensure!( + abi3_minor_version <= interpreter_config.version.minor, + "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", + abi3_minor_version, + interpreter_config.version.minor + ); + + interpreter_config.version.minor = abi3_minor_version; + } + + Ok(Some(interpreter_config)) +} + pub fn make_interpreter_config() -> Result { let abi3_version = get_abi3_minor_version(); @@ -1059,11 +1103,7 @@ pub fn make_interpreter_config() -> Result { } } - let mut interpreter_config = if let Some(paths) = cross_compiling()? { - load_cross_compile_info(paths)? - } else { - get_config_from_interpreter(&find_interpreter()?)? - }; + let mut interpreter_config = get_config_from_interpreter(&find_interpreter()?)?; // Fixup minor version if abi3-pyXX feature set if let Some(abi3_minor_version) = abi3_version { diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 5a974923d92..7d77f11f4ec 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -7,14 +7,14 @@ pub mod errors; mod impl_; -use std::{ffi::OsString, path::Path}; +use std::io::Cursor; use once_cell::sync::OnceCell; // Used in PyO3's build.rs #[doc(hidden)] pub use impl_::{ - cargo_env_var, env_var, find_interpreter, get_config_from_interpreter, make_interpreter_config, + cargo_env_var, env_var, find_interpreter, get_config_from_interpreter, make_interpreter_config, make_cross_compile_config, InterpreterConfig, PythonImplementation, PythonVersion, }; @@ -25,20 +25,24 @@ pub use impl_::{ pub fn get() -> &'static InterpreterConfig { static CONFIG: OnceCell = OnceCell::new(); CONFIG.get_or_init(|| { - let config_path = std::env::var_os("PYO3_CONFIG_FILE") - .unwrap_or_else(|| OsString::from(DEFAULT_CONFIG_PATH)); - let config_file = std::fs::File::open(DEFAULT_CONFIG_PATH).expect(&format!( - "failed to open PyO3 config file at {}", - Path::new(&config_path).display() - )); - let reader = std::io::BufReader::new(config_file); - InterpreterConfig::from_reader(reader).expect("failed to parse config file") + if let Some(path) = std::env::var_os("PYO3_CONFIG_FILE") { + // Config file set - use that + InterpreterConfig::from_path(path) + } else if impl_::any_cross_compiling_env_vars_set() { + InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) + } else { + InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + }.expect("failed to parse PyO3 config file") }) } /// Path where PyO3's build.rs will write configuration by default. #[doc(hidden)] -pub const DEFAULT_CONFIG_PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"); +pub const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt"); + +/// Build configuration discovered by `pyo3-build-config` build script. Not aware of +/// cross-compilation settings. +pub const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt")); /// Adds all the [`#[cfg]` flags](index.html) to the current compilation. /// From 49387e9a70496bc818801621a8f3f4f7fda9bc94 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 5 Aug 2021 22:43:26 +0100 Subject: [PATCH 4/5] pyo3-build-config: many tidy ups --- build.rs | 151 ++++------ guide/src/building_and_distribution.md | 23 +- pyo3-build-config/build.rs | 84 +++++- pyo3-build-config/src/errors.rs | 5 +- pyo3-build-config/src/impl_.rs | 395 +++++++++++-------------- pyo3-build-config/src/lib.rs | 149 +++++++--- 6 files changed, 448 insertions(+), 359 deletions(-) diff --git a/build.rs b/build.rs index 714ca0c3803..4b5101289fe 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,11 @@ -use std::{env, io::Cursor, path::Path, process::Command}; +use std::{env, process::Command}; use pyo3_build_config::{ - bail, cargo_env_var, ensure, env_var, - errors::{Context, Result}, - make_cross_compile_config, InterpreterConfig, PythonVersion, + bail, ensure, + pyo3_build_script_impl::{ + cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig, + PythonVersion, + }, }; /// Minimum Python version PyO3 supports. @@ -20,24 +22,54 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { Ok(()) } -fn ensure_target_pointer_width(pointer_width: u32) -> Result<()> { - // Try to check whether the target architecture matches the python library - let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") - .unwrap() - .as_str() - { - "64" => 64, - "32" => 32, - x => bail!("unexpected Rust target pointer width: {}", x), - }; +fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> { + if let Some(pointer_width) = interpreter_config.pointer_width { + // Try to check whether the target architecture matches the python library + let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") + .unwrap() + .as_str() + { + "64" => 64, + "32" => 32, + x => bail!("unexpected Rust target pointer width: {}", x), + }; + + ensure!( + rust_target == pointer_width, + "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", + rust_target, + pointer_width + ); + } + Ok(()) +} - ensure!( - rust_target == pointer_width, - "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", - rust_target, - pointer_width - ); +fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { + if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { + if !interpreter_config.shared { + bail!( + "The `auto-initialize` feature is enabled, but your python installation only supports \ + embedding the Python interpreter statically. If you are attempting to run tests, or a \ + binary which is okay to link dynamically, install a Python distribution which ships \ + with the Python shared library.\n\ + \n\ + Embedding the Python interpreter statically does not yet have first-class support in \ + PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\ + \n\ + For more information, see \ + https://pyo3.rs/v{pyo3_version}/\ + building_and_distribution.html#embedding-python-in-rust", + pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() + ); + } + // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies` + // currently cause `auto-initialize` to be enabled in CI. + // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead. + if interpreter_config.implementation.is_pypy() && env::var_os("PYO3_CI").is_none() { + bail!("the `auto-initialize` feature is not supported with PyPy"); + } + } Ok(()) } @@ -70,89 +102,36 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() } else { "" }, - lib_name = interpreter_config - .lib_name - .as_ref() - .ok_or("config does not contain lib_name")?, + lib_name = interpreter_config.lib_name.as_ref().ok_or( + "attempted to link to Python shared library but config does not contain lib_name" + )?, ); if let Some(lib_dir) = &interpreter_config.lib_dir { println!("cargo:rustc-link-search=native={}", lib_dir); } } - if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { - if !interpreter_config.shared { - bail!( - "The `auto-initialize` feature is enabled, but your python installation only supports \ - embedding the Python interpreter statically. If you are attempting to run tests, or a \ - binary which is okay to link dynamically, install a Python distribution which ships \ - with the Python shared library.\n\ - \n\ - Embedding the Python interpreter statically does not yet have first-class support in \ - PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\ - \n\ - For more information, see \ - https://pyo3.rs/v{pyo3_version}/\ - building_and_distribution.html#embedding-python-in-rust", - pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() - ); - } - - // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies` - // currently cause `auto-initialize` to be enabled in CI. - // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead. - if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() { - bail!("the `auto-initialize` feature is not supported with PyPy"); - } - } - Ok(()) } -/// Generates the interpreter config suitable for the host / target / cross-compilation at hand. +/// Prepares the PyO3 crate for compilation. +/// +/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX +/// for users. /// -/// The result is written to pyo3_build_config::PATH, which downstream scripts can read from -/// (including `pyo3-macros-backend` during macro expansion). +/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler +/// version to enable features which aren't supported on MSRV. fn configure_pyo3() -> Result<()> { - let interpreter_config = if let Some(path) = env_var("PYO3_CONFIG_FILE") { - let path = Path::new(&path); - // This is necessary because the compilations that access PYO3_CONFIG_FILE (build scripts, - // proc macros) have many different working directories, so a relative path is no good. - ensure!(path.is_absolute(), "PYO3_CONFIG_FILE must be an absolute path"); - println!("cargo:rerun-if-changed={}", path.display()); - InterpreterConfig::from_path(path)? - } else if let Some(interpreter_config) = make_cross_compile_config()? { - // This is a cross compile, need to write the config file. - let path = Path::new(&pyo3_build_config::DEFAULT_CROSS_COMPILE_CONFIG_PATH); - let parent_dir = path.parent().ok_or_else(|| { - format!( - "failed to resolve parent directory of config file {}", - path.display() - ) - })?; - std::fs::create_dir_all(&parent_dir).with_context(|| { - format!( - "failed to create config file directory {}", - parent_dir.display() - ) - })?; - interpreter_config - .to_writer(&mut std::fs::File::create(&path).with_context(|| { - format!("failed to create config file at {}", path.display()) - })?)?; - interpreter_config - } else { - InterpreterConfig::from_reader(Cursor::new(pyo3_build_config::HOST_CONFIG))? - }; + let interpreter_config = resolve_interpreter_config()?; if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { print_config_and_exit(&interpreter_config); } ensure_python_version(&interpreter_config)?; - if let Some(pointer_width) = interpreter_config.pointer_width { - ensure_target_pointer_width(pointer_width)?; - } + ensure_target_pointer_width(&interpreter_config)?; + ensure_auto_initialize_ok(&interpreter_config)?; + emit_cargo_configuration(&interpreter_config)?; interpreter_config.emit_pyo3_cfgs(); diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 96cfc69838b..b7b6af134b9 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -30,23 +30,22 @@ Caused by: cargo:rerun-if-env-changed=PYO3_CROSS cargo:rerun-if-env-changed=PYO3_CROSS_LIB_DIR cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_VERSION - cargo:rerun-if-env-changed=PYO3_PYTHON - cargo:rerun-if-env-changed=VIRTUAL_ENV - cargo:rerun-if-env-changed=CONDA_PREFIX - cargo:rerun-if-env-changed=PATH cargo:rerun-if-env-changed=PYO3_PRINT_CONFIG -- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile -- - implementation: CPython - interpreter version: 3.8 - interpreter path: Some("/usr/bin/python") - libdir: Some("/usr/lib") - shared: true - base prefix: Some("/usr") - ld_version: Some("3.8") - pointer width: Some(8) + implementation=CPython + version=3.8 + shared=true + abi3=false + lib_name=python3.8 + lib_dir=/usr/lib + executable=/usr/bin/python + pointer_width=64 + build_flags=WITH_THREAD ``` +> Note: if you safe the output config to a file, it is possible to manually override the and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var. For now, this is an advanced feature that should not be needed for most users. The format of the config file and its contents are deliberately unstable and undocumented. If you have a production use-case for this config file, please file an issue and help us stabilize it! + ## Building Python extension modules Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs index ae23362e2fa..28d6057944c 100644 --- a/pyo3-build-config/build.rs +++ b/pyo3-build-config/build.rs @@ -11,21 +11,85 @@ mod errors; use std::{env, path::Path}; -use errors::{Result, Context}; +use errors::{Context, Result}; +use impl_::{ + env_var, get_abi3_version, make_interpreter_config, BuildFlags, InterpreterConfig, + PythonImplementation, +}; -fn generate_build_config() -> Result<()> { - // Create new interpreter config and write it to the default location - let interpreter_config = impl_::make_interpreter_config()?; +fn configure(interpreter_config: Option, name: &str) -> Result { + let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name); + if let Some(config) = interpreter_config { + config + .to_writer(&mut std::fs::File::create(&target).with_context(|| { + format!("failed to write config file at {}", target.display()) + })?)?; + Ok(true) + } else { + std::fs::File::create(&target) + .with_context(|| format!("failed to create new file at {}", target.display()))?; + Ok(false) + } +} + +/// If PYO3_CONFIG_FILE is set, copy it into the crate. +fn config_file() -> Result> { + if let Some(path) = env_var("PYO3_CONFIG_FILE") { + let path = Path::new(&path); + println!("cargo:rerun-if-changed={}", path.display()); + // Absolute path is necessary because this build script is run with a cwd different to the + // original `cargo build` instruction. + ensure!( + path.is_absolute(), + "PYO3_CONFIG_FILE must be an absolute path" + ); + + let interpreter_config = InterpreterConfig::from_path(path) + .context("failed to parse contents of PYO3_CONFIG_FILE")?; + Ok(Some(interpreter_config)) + } else { + Ok(None) + } +} - let path = Path::new(&env::var_os("OUT_DIR").unwrap()).join("pyo3-build-config.txt"); - interpreter_config - .to_writer(&mut std::fs::File::create(&path).with_context(|| { - format!("failed to create config file at {}", path.display()) - })?) +/// If PYO3_NO_PYTHON is set with abi3, use standard abi3 settings. +pub fn abi3_config() -> Option { + if let Some(version) = get_abi3_version() { + if env_var("PYO3_NO_PYTHON").is_some() { + return Some(InterpreterConfig { + version, + // NB PyPy doesn't support abi3 yet + implementation: PythonImplementation::CPython, + abi3: true, + lib_name: None, + lib_dir: None, + build_flags: BuildFlags::abi3(), + pointer_width: None, + executable: None, + shared: true, + }); + } + } + None +} + +fn generate_build_configs() -> Result<()> { + let mut configured = false; + configured |= configure(config_file()?, "pyo3-build-config-file.txt")?; + configured |= configure(abi3_config(), "pyo3-build-config-abi3.txt")?; + + if configured { + // Don't bother trying to find an interpreter on the host system if at least one of the + // config file or abi3 settings are present + configure(None, "pyo3-build-config.txt")?; + } else { + configure(Some(make_interpreter_config()?), "pyo3-build-config.txt")?; + } + Ok(()) } fn main() { - if let Err(e) = generate_build_config() { + if let Err(e) = generate_build_configs() { eprintln!("error: {}", e.report()); std::process::exit(1) } diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index dfa3f837377..ba740fe5e06 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -1,5 +1,6 @@ /// A simple macro for returning an error. Resembles anyhow::bail. #[macro_export] +#[doc(hidden)] macro_rules! bail { ($msg: expr) => { return Err($msg.into()); }; ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); }; @@ -7,12 +8,14 @@ macro_rules! bail { /// A simple macro for checking a condition. Resembles anyhow::ensure. #[macro_export] +#[doc(hidden)] macro_rules! ensure { ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } }; } /// Show warning. If needed, please extend this macro to support arguments. #[macro_export] +#[doc(hidden)] macro_rules! warn { ($msg: literal) => { println!(concat!("cargo:warning=", $msg)); @@ -27,7 +30,7 @@ pub struct Error { } /// Error report inspired by -/// https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter +/// pub struct ErrorReport<'a>(&'a Error); impl Error { diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index e1834269388..982a252d22a 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -38,9 +38,11 @@ pub fn env_var(var: &str) -> Option { /// Configuration needed by PyO3 to build for the correct Python implementation. /// -/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable -/// is set, or during cross compile situations, then alternative strategies are used to populate -/// this type. +/// Usually this is queried directly from the Python interpreter, or overridden using the +/// `PYO3_CONFIG_FILE` environment variable. +/// +/// When the `PYO3_NO_PYTHON` variable is set, or during cross compile situations, then alternative +/// strategies are used to populate this type. #[cfg_attr(test, derive(Debug, PartialEq))] pub struct InterpreterConfig { pub implementation: PythonImplementation, @@ -55,6 +57,7 @@ pub struct InterpreterConfig { } impl InterpreterConfig { + #[doc(hidden)] pub fn emit_pyo3_cfgs(&self) { // This should have been checked during pyo3-build-config build time. assert!(self.version >= MINIMUM_SUPPORTED_VERSION); @@ -66,7 +69,7 @@ impl InterpreterConfig { println!("cargo:rustc-cfg=Py_LIMITED_API"); } - if self.is_pypy() { + if self.implementation.is_pypy() { println!("cargo:rustc-cfg=PyPy"); if self.abi3 { warn!( @@ -81,20 +84,119 @@ impl InterpreterConfig { } } - pub fn is_pypy(&self) -> bool { - self.implementation == PythonImplementation::PyPy + #[doc(hidden)] + pub fn from_interpreter(interpreter: impl AsRef) -> Result { + let script = r#" +# Allow the script to run on Python 2, so that nicer error can be printed later. +from __future__ import print_function + +import os.path +import platform +import struct +import sys +from sysconfig import get_config_var + +PYPY = platform.python_implementation() == "PyPy" + +# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue +# so that the version mismatch can be reported in a nicer way later. +base_prefix = getattr(sys, "base_prefix", None) + +if base_prefix: + # Anaconda based python distributions have a static python executable, but include + # the shared library. Use the shared library for embedding to avoid rust trying to + # LTO the static library (and failing with newer gcc's, because it is old). + ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) +else: + ANACONDA = False + +def print_if_set(varname, value): + if value is not None: + print(varname, value) + +# Windows always uses shared linking +WINDOWS = hasattr(platform, "win32_ver") + +# macOS framework packages use shared linking +FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) + +# unix-style shared library enabled +SHARED = bool(get_config_var("Py_ENABLE_SHARED")) + +print("implementation", platform.python_implementation()) +print("version_major", sys.version_info[0]) +print("version_minor", sys.version_info[1]) +print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print_if_set("ld_version", get_config_var("LDVERSION")) +print_if_set("libdir", get_config_var("LIBDIR")) +print_if_set("base_prefix", base_prefix) +print("executable", sys.executable) +print("calcsize_pointer", struct.calcsize("P")) +"#; + let output = run_python_script(interpreter.as_ref(), script)?; + let map: HashMap = parse_script_output(&output); + let shared = map["shared"].as_str() == "True"; + + let version = PythonVersion { + major: map["version_major"] + .parse() + .context("failed to parse major version")?, + minor: map["version_minor"] + .parse() + .context("failed to parse minor version")?, + }; + + let abi3 = is_abi3(); + let implementation = map["implementation"].parse()?; + + let lib_name = if cfg!(windows) { + default_lib_name_windows( + &version, + abi3, + &cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(), + ) + } else { + default_lib_name_unix( + &version, + implementation, + map.get("ld_version").map(String::as_str), + )? + }; + + let lib_dir = if cfg!(windows) { + map.get("base_prefix") + .map(|base_prefix| format!("{}\\libs", base_prefix)) + } else { + map.get("libdir").cloned() + }; + + // The reason we don't use platform.architecture() here is that it's not + // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. + // Similarly, sys.maxsize is not reliable on Windows. See + // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 + // and https://stackoverflow.com/a/3411134/823869. + let calcsize_pointer: u32 = map["calcsize_pointer"] + .parse() + .context("failed to parse calcsize_pointer")?; + + Ok(InterpreterConfig { + version, + implementation, + shared, + abi3, + lib_name: Some(lib_name), + lib_dir, + executable: map.get("executable").cloned(), + pointer_width: Some(calcsize_pointer * 8), + build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), + }) } #[doc(hidden)] pub fn from_path(path: impl AsRef) -> Result { let path = path.as_ref(); - let config_file = - std::fs::File::open(path).with_context(|| { - format!( - "failed to open PyO3 config file at {}", - path.display() - ) - })?; + let config_file = std::fs::File::open(path) + .with_context(|| format!("failed to open PyO3 config file at {}", path.display()))?; let reader = std::io::BufReader::new(config_file); InterpreterConfig::from_reader(reader) } @@ -255,6 +357,13 @@ pub enum PythonImplementation { PyPy, } +impl PythonImplementation { + #[doc(hidden)] + pub fn is_pypy(self) -> bool { + self == PythonImplementation::PyPy + } +} + impl Display for PythonImplementation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -312,7 +421,7 @@ impl GetPrimitive for HashMap { struct CrossCompileConfig { lib_dir: PathBuf, - version: Option, + version: Option, os: String, arch: String, } @@ -377,10 +486,12 @@ fn cross_compiling() -> Result> { arch: target_arch.unwrap(), version: cross_python_version .map(|os_string| { - os_string + let utf8_str = os_string .to_str() - .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.") - .map(str::to_owned) + .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")?; + utf8_str + .parse() + .context("failed to parse PYO3_CROSS_PYTHON_VERSION") }) .transpose()?, })) @@ -462,7 +573,7 @@ impl BuildFlags { /// Examine python's compile flags to pass to cfg by launching /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. - fn from_interpreter(interpreter: &Path) -> Result { + fn from_interpreter(interpreter: impl AsRef) -> Result { // If we're on a Windows host, then Python won't have any useful config vars if cfg!(windows) { return Ok(Self::windows_hardcoded()); @@ -475,7 +586,7 @@ impl BuildFlags { script.push_str(&format!("print(config.get('{}', '0'))\n", k)); } - let stdout = run_python_script(interpreter, &script)?; + let stdout = run_python_script(interpreter.as_ref(), &script)?; let split_stdout: Vec<&str> = stdout.trim_end().lines().collect(); ensure!( split_stdout.len() == BuildFlags::ALL.len(), @@ -500,7 +611,7 @@ impl BuildFlags { Self(flags) } - fn abi3() -> Self { + pub fn abi3() -> Self { let mut flags = HashSet::new(); flags.insert(BuildFlag::WITH_THREAD); Self(flags) @@ -776,26 +887,8 @@ fn load_cross_compile_from_sysconfigdata( fn windows_hardcoded_cross_compile( cross_compile_config: CrossCompileConfig, ) -> Result { - let (major, minor) = if let Some(version) = cross_compile_config.version { - let mut parts = version.split('.'); - match ( - parts.next().and_then(|major| major.parse().ok()), - parts.next().and_then(|minor| minor.parse().ok()), - parts.next(), - ) { - (Some(major), Some(minor), None) => (major, minor), - _ => bail!( - "Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`", - version - ), - } - } else if let Some(minor_version) = get_abi3_minor_version() { - (3, minor_version) - } else { - bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.") - }; - - let version = PythonVersion { major, minor }; + let version = cross_compile_config.version.or_else(get_abi3_version) + .ok_or("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")?; Ok(InterpreterConfig { implementation: PythonImplementation::CPython, @@ -810,7 +903,9 @@ fn windows_hardcoded_cross_compile( }) } -fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result { +fn load_cross_compile_config( + cross_compile_config: CrossCompileConfig, +) -> Result { match cargo_env_var("CARGO_CFG_TARGET_FAMILY") { // Configure for unix platforms using the sysconfigdata file Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config), @@ -820,7 +915,10 @@ fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result load_cross_compile_from_sysconfigdata(cross_compile_config), // Waiting for users to tell us what they expect on their target platform Some(os) => bail!( - "Unsupported target OS family for cross-compilation: {:?}", + "Unknown target OS family for cross-compilation: {:?}.\n\ + \n\ + Please set the PYO3_CONFIG_FILE environment variable to a config suitable for your \ + target interpreter.", os ), // Unknown os family - try to do something useful @@ -903,11 +1001,13 @@ fn get_venv_path() -> Option { } } -/// Attempts to locate a python interpreter. Locations are checked in the order listed: -/// 1. If `PYO3_PYTHON` is set, this intepreter is used. -/// 2. If in a virtualenv, that environment's interpreter is used. -/// 3. `python`, if this is functional a Python 3.x interpreter -/// 4. `python3`, as above +/// Attempts to locate a python interpreter. +/// +/// Locations are checked in the order listed: +/// 1. If `PYO3_PYTHON` is set, this intepreter is used. +/// 2. If in a virtualenv, that environment's interpreter is used. +/// 3. `python`, if this is functional a Python 3.x interpreter +/// 4. `python3`, as above pub fn find_interpreter() -> Result { if let Some(exe) = env_var("PYO3_PYTHON") { Ok(exe.into()) @@ -936,187 +1036,52 @@ pub fn find_interpreter() -> Result { } } -/// Extract compilation vars from the specified interpreter. -pub fn get_config_from_interpreter(interpreter: &Path) -> Result { - let script = r#" -# Allow the script to run on Python 2, so that nicer error can be printed later. -from __future__ import print_function - -import os.path -import platform -import struct -import sys -from sysconfig import get_config_var - -PYPY = platform.python_implementation() == "PyPy" - -# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue -# so that the version mismatch can be reported in a nicer way later. -base_prefix = getattr(sys, "base_prefix", None) - -if base_prefix: - # Anaconda based python distributions have a static python executable, but include - # the shared library. Use the shared library for embedding to avoid rust trying to - # LTO the static library (and failing with newer gcc's, because it is old). - ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) -else: - ANACONDA = False - -def print_if_set(varname, value): - if value is not None: - print(varname, value) - -# Windows always uses shared linking -WINDOWS = hasattr(platform, "win32_ver") - -# macOS framework packages use shared linking -FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) - -# unix-style shared library enabled -SHARED = bool(get_config_var("Py_ENABLE_SHARED")) - -print("implementation", platform.python_implementation()) -print("version_major", sys.version_info[0]) -print("version_minor", sys.version_info[1]) -print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) -print_if_set("ld_version", get_config_var("LDVERSION")) -print_if_set("libdir", get_config_var("LIBDIR")) -print_if_set("base_prefix", base_prefix) -print("executable", sys.executable) -print("calcsize_pointer", struct.calcsize("P")) -"#; - let output = run_python_script(interpreter, script)?; - let map: HashMap = parse_script_output(&output); - let shared = map["shared"].as_str() == "True"; - - let version = PythonVersion { - major: map["version_major"] - .parse() - .context("failed to parse major version")?, - minor: map["version_minor"] - .parse() - .context("failed to parse minor version")?, - }; - - let abi3 = is_abi3(); - let implementation = map["implementation"].parse()?; - - let lib_name = if cfg!(windows) { - default_lib_name_windows( - &version, - abi3, - &cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(), - ) - } else { - default_lib_name_unix( - &version, - implementation, - map.get("ld_version").map(String::as_str), - )? - }; - - let lib_dir = if cfg!(windows) { - map.get("base_prefix") - .map(|base_prefix| format!("cargo:rustc-link-search=native={}\\libs", base_prefix)) - } else { - map.get("libdir").cloned() - }; - - // The reason we don't use platform.architecture() here is that it's not - // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. - // Similarly, sys.maxsize is not reliable on Windows. See - // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 - // and https://stackoverflow.com/a/3411134/823869. - let calcsize_pointer: u32 = map["calcsize_pointer"] - .parse() - .context("failed to parse calcsize_pointer")?; - - Ok(InterpreterConfig { - version, - implementation, - shared, - abi3, - lib_name: Some(lib_name), - lib_dir, - executable: map.get("executable").cloned(), - pointer_width: Some(calcsize_pointer * 8), - build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), - }) +pub fn get_abi3_version() -> Option { + let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()); + minor_version.map(|minor| PythonVersion { major: 3, minor }) } -fn get_abi3_minor_version() -> Option { - (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) +/// Lowers the configured version to the abi3 version, if set. +fn fixup_config_for_abi3( + config: &mut InterpreterConfig, + abi3_version: Option, +) -> Result<()> { + if let Some(version) = abi3_version { + ensure!( + version <= config.version, + "cannot set a mininimum Python version {} higher than the interpreter version {} \ + (the minimum Python version is implied by the abi3-py3{} feature)", + version, + config.version, + version.minor, + ); + + config.version = version; + } + Ok(()) } +/// Generates an interpreter config suitable for cross-compilation. +/// +/// This must be called from PyO3's build script, because it relies on environment variables such as +/// CARGO_CFG_TARGET_OS which aren't available at any other time. pub fn make_cross_compile_config() -> Result> { - let abi3_version = get_abi3_minor_version(); - let mut interpreter_config = if let Some(paths) = cross_compiling()? { - load_cross_compile_info(paths)? + load_cross_compile_config(paths)? } else { return Ok(None); }; - - // Fixup minor version if abi3-pyXX feature set - if let Some(abi3_minor_version) = abi3_version { - ensure!( - abi3_minor_version <= interpreter_config.version.minor, - "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", - abi3_minor_version, - interpreter_config.version.minor - ); - - interpreter_config.version.minor = abi3_minor_version; - } - + fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?; Ok(Some(interpreter_config)) } +/// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate. +/// Only used by `pyo3-build-config` build script. +#[allow(dead_code)] pub fn make_interpreter_config() -> Result { - let abi3_version = get_abi3_minor_version(); - - // If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python. - if let Some(abi3_minor_version) = abi3_version { - if env_var("PYO3_NO_PYTHON").is_some() { - let version = PythonVersion { - major: 3, - minor: abi3_minor_version, - }; - let implementation = PythonImplementation::CPython; - let lib_name = if cfg!(windows) { - Some(WINDOWS_ABI3_LIB_NAME.to_owned()) - } else { - None - }; - return Ok(InterpreterConfig { - version, - implementation, - abi3: true, - lib_name, - lib_dir: None, - build_flags: BuildFlags::abi3(), - pointer_width: None, - executable: None, - shared: true, - }); - } - } - - let mut interpreter_config = get_config_from_interpreter(&find_interpreter()?)?; - - // Fixup minor version if abi3-pyXX feature set - if let Some(abi3_minor_version) = abi3_version { - ensure!( - abi3_minor_version <= interpreter_config.version.minor, - "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", - abi3_minor_version, - interpreter_config.version.minor - ); - - interpreter_config.version.minor = abi3_minor_version; - } - + let mut interpreter_config = InterpreterConfig::from_interpreter(&find_interpreter()?)?; + fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?; Ok(interpreter_config) } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 7d77f11f4ec..be3f4093d97 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -1,48 +1,23 @@ //! Configuration used by PyO3 for conditional support of varying Python versions. //! -//! The public APIs exposed, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`] are intended -//! to be called from build scripts to simplify building crates which depend on PyO3. +//! This crate exposes two functions, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`], +//! which are intended to be called from build scripts to simplify building crates which depend on +//! PyO3. +//! +//! It used internally by the PyO3 crate's build script to apply the same configuration. -#[doc(hidden)] -pub mod errors; +mod errors; mod impl_; use std::io::Cursor; use once_cell::sync::OnceCell; -// Used in PyO3's build.rs -#[doc(hidden)] -pub use impl_::{ - cargo_env_var, env_var, find_interpreter, get_config_from_interpreter, make_interpreter_config, make_cross_compile_config, - InterpreterConfig, PythonImplementation, PythonVersion, -}; +use impl_::InterpreterConfig; -/// Reads the configuration written by PyO3's build.rs -/// -/// Because this will never change in a given compilation run, this is cached in a `once_cell`. +// Used in `pyo3-macros-backend`; may expose this in a future release. #[doc(hidden)] -pub fn get() -> &'static InterpreterConfig { - static CONFIG: OnceCell = OnceCell::new(); - CONFIG.get_or_init(|| { - if let Some(path) = std::env::var_os("PYO3_CONFIG_FILE") { - // Config file set - use that - InterpreterConfig::from_path(path) - } else if impl_::any_cross_compiling_env_vars_set() { - InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) - } else { - InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) - }.expect("failed to parse PyO3 config file") - }) -} - -/// Path where PyO3's build.rs will write configuration by default. -#[doc(hidden)] -pub const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt"); - -/// Build configuration discovered by `pyo3-build-config` build script. Not aware of -/// cross-compilation settings. -pub const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt")); +pub use impl_::PythonVersion; /// Adds all the [`#[cfg]` flags](index.html) to the current compilation. /// @@ -68,8 +43,112 @@ pub fn use_pyo3_cfgs() { /// This is currently a no-op on non-macOS platforms, however may emit additional linker arguments /// in future if deemed necessarys. pub fn add_extension_module_link_args() { - if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "macos" { + if impl_::cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "macos" { println!("cargo:rustc-cdylib-link-arg=-undefined"); println!("cargo:rustc-cdylib-link-arg=dynamic_lookup"); } } + +/// Loads the configuration determined from the build environment. +/// +/// Because this will never change in a given compilation run, this is cached in a `once_cell`. +#[doc(hidden)] +pub fn get() -> &'static InterpreterConfig { + static CONFIG: OnceCell = OnceCell::new(); + CONFIG.get_or_init(|| { + if !CONFIG_FILE.is_empty() { + InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + } else if !ABI3_CONFIG.is_empty() { + Ok(abi3_config()) + } else if impl_::any_cross_compiling_env_vars_set() { + InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) + } else { + InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + } + .expect("failed to parse PyO3 config file") + }) +} + +/// Path where PyO3's build.rs will write configuration by default. +#[doc(hidden)] +const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str = + concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt"); + +/// Build configuration provided by `PYO3_CONFIG_FILE`. May be empty if env var not set. +#[doc(hidden)] +const CONFIG_FILE: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-file.txt")); + +/// Build configuration set if abi3 features enabled and `PYO3_NO_PYTHON` env var present. Empty if +/// not both present. +#[doc(hidden)] +const ABI3_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-abi3.txt")); + +/// Build configuration discovered by `pyo3-build-config` build script. Not aware of +/// cross-compilation settings. +#[doc(hidden)] +const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt")); + +fn abi3_config() -> InterpreterConfig { + let mut interpreter_config = InterpreterConfig::from_reader(Cursor::new(ABI3_CONFIG)) + .expect("failed to parse hardcoded PyO3 abi3 config"); + // If running from a build script on Windows, tweak the hardcoded abi3 config to contain + // the standard lib name (this is necessary so that abi3 extension modules using + // PYO3_NO_PYTHON on Windows can link) + if std::env::var("CARGO_CFG_TARGET_OS").map_or(false, |target_os| target_os == "windows") { + assert_eq!(interpreter_config.lib_name, None); + interpreter_config.lib_name = Some("python3".to_owned()) + } + interpreter_config +} + +/// Private exports used in PyO3's build.rs +/// +/// Please don't use these - they could change at any time. +#[doc(hidden)] +pub mod pyo3_build_script_impl { + use crate::errors::{Context, Result}; + use std::path::Path; + + use super::*; + + pub mod errors { + pub use crate::errors::*; + } + pub use crate::impl_::{ + cargo_env_var, env_var, make_cross_compile_config, InterpreterConfig, PythonVersion, + }; + + /// Gets the configuration for use from PyO3's build script. + /// + /// Differs from .get() above only in the cross-compile case, where PyO3's build script is + /// required to generate a new config (as it's the first build script which has access to the + /// correct value for CARGO_CFG_TARGET_OS). + pub fn resolve_interpreter_config() -> Result { + if !CONFIG_FILE.is_empty() { + InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + } else if !ABI3_CONFIG.is_empty() { + Ok(abi3_config()) + } else if let Some(interpreter_config) = impl_::make_cross_compile_config()? { + // This is a cross compile and need to write the config file. + let path = Path::new(DEFAULT_CROSS_COMPILE_CONFIG_PATH); + let parent_dir = path.parent().ok_or_else(|| { + format!( + "failed to resolve parent directory of config file {}", + path.display() + ) + })?; + std::fs::create_dir_all(&parent_dir).with_context(|| { + format!( + "failed to create config file directory {}", + parent_dir.display() + ) + })?; + interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context( + || format!("failed to create config file at {}", path.display()), + )?)?; + Ok(interpreter_config) + } else { + InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + } + } +} From 96fefc96166decb731ed0053eac513d24750bc55 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 5 Aug 2021 23:23:23 +0100 Subject: [PATCH 5/5] pyo3-build-config: fix build for windows gnu targets --- CHANGELOG.md | 1 + pyo3-build-config/src/impl_.rs | 15 ++++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87f50f595d6..ba7b18866ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix regression in 0.14.0 leading to incorrect code coverage being computed for `#[pyfunction]`s. [#1726](https://github.com/PyO3/pyo3/pull/1726) - Fix incorrect FFI definition of `Py_Buffer` on PyPy. [#1737](https://github.com/PyO3/pyo3/pull/1737) - Fix incorrect calculation of `dictoffset` on 32-bit Windows. [#1475](https://github.com/PyO3/pyo3/pull/1475) +- Fix regression in 0.13.2 leading to linking to incorrect Python library on Windows "gnu" targets. [#1759](https://github.com/PyO3/pyo3/pull/1759) ## [0.14.1] - 2021-07-04 diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 982a252d22a..350d4b543a4 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -94,7 +94,7 @@ import os.path import platform import struct import sys -from sysconfig import get_config_var +from sysconfig import get_config_var, get_platform PYPY = platform.python_implementation() == "PyPy" @@ -132,6 +132,7 @@ print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) +print("mingw", get_platform() == "mingw") "#; let output = run_python_script(interpreter.as_ref(), script)?; let map: HashMap = parse_script_output(&output); @@ -150,11 +151,7 @@ print("calcsize_pointer", struct.calcsize("P")) let implementation = map["implementation"].parse()?; let lib_name = if cfg!(windows) { - default_lib_name_windows( - &version, - abi3, - &cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(), - ) + default_lib_name_windows(&version, abi3, map["mingw"].as_str() == "True") } else { default_lib_name_unix( &version, @@ -895,7 +892,7 @@ fn windows_hardcoded_cross_compile( version, shared: true, abi3: is_abi3(), - lib_name: Some(default_lib_name_windows(&version, false, "msvc")), + lib_name: Some(default_lib_name_windows(&version, false, false)), lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), executable: None, pointer_width: None, @@ -932,10 +929,10 @@ fn load_cross_compile_config( // This contains only the limited ABI symbols. const WINDOWS_ABI3_LIB_NAME: &str = "python3"; -fn default_lib_name_windows(version: &PythonVersion, abi3: bool, target_env: &str) -> String { +fn default_lib_name_windows(version: &PythonVersion, abi3: bool, mingw: bool) -> String { if abi3 { WINDOWS_ABI3_LIB_NAME.to_owned() - } else if target_env == "gnu" { + } else if mingw { // https://packages.msys2.org/base/mingw-w64-python format!("python{}.{}", version.major, version.minor) } else {