merge master into abi3, resolving conflicts

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
-All notable changes to this project will be documented in this file.
+All notable changes to this project will be documented in this file. For help with updating to new
+PyO3 versions, please see the [migration guide](https://pyo3.rs/master/migration.html).
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
@@ -18,9 +19,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Add macro attribute to `#[pyfn]` and `#[pyfunction]` to pass the module of a Python function to the function
   body. [#1143](https://github.com/PyO3/pyo3/pull/1143)
 - Add `add_function()` and `add_submodule()` functions to `PyModule` [#1143](https://github.com/PyO3/pyo3/pull/1143)
+- Add native `PyCFunction` and `PyFunction` types, change `add_function` to take a wrapper returning
+  a `&PyCFunction`instead of `PyObject`. [#1163](https://github.com/PyO3/pyo3/pull/1163)
 
 ### Changed
-- Exception types have been renamed from e.g. `RuntimeError` to `PyRuntimeError`, and are now only accessible by `&T` or `Py<T>` similar to other Python-native types. The old names continue to exist but are deprecated. [#1024](https://github.com/PyO3/pyo3/pull/1024)
+- Exception types have been renamed from e.g. `RuntimeError` to `PyRuntimeError`, and are now accessible by `&T` or `Py<T>` similar to other Python-native types. The old names continue to exist but are deprecated. [#1024](https://github.com/PyO3/pyo3/pull/1024) [#1115](https://github.com/PyO3/pyo3/pull/1115)
+  - Rename `PyException::py_err()` to `PyException::new_err()`.
+  - Rename `PyUnicodeDecodeErr::new_err()` to `PyUnicodeDecodeErr::new()`.
+  - Remove `PyStopIteration::stop_iteration()`.
 - Correct FFI definitions `Py_SetProgramName` and `Py_SetPythonHome` to take `*const` argument instead of `*mut`. [#1021](https://github.com/PyO3/pyo3/pull/1021)
 - Rename `PyString::to_string` to `to_str`, change return type `Cow<str>` to `&str`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
 - Correct FFI definition `_PyLong_AsByteArray` `*mut c_uchar` argument instead of `*const c_uchar`. [#1029](https://github.com/PyO3/pyo3/pull/1029)
@@ -30,7 +36,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Change `PyIterator::from_object` to return `PyResult<PyIterator>` instead of `Result<PyIterator, PyDowncastError>`. [#1051](https://github.com/PyO3/pyo3/pull/1051)
 - `IntoPy` is no longer implied by `FromPy`. [#1063](https://github.com/PyO3/pyo3/pull/1063)
 - `PyObject` is now just a type alias for `Py<PyAny>`. [#1063](https://github.com/PyO3/pyo3/pull/1063)
-- Implement `Send + Sync` for `PyErr`. `PyErr::new`, `PyErr::from_type`, `PyException::py_err` and `PyException::into` have had these bounds added to their arguments. [#1067](https://github.com/PyO3/pyo3/pull/1067)
+- Rework PyErr to be compatible with the `std::error::Error` trait: [#1067](https://github.com/PyO3/pyo3/pull/1067) [#1115](https://github.com/PyO3/pyo3/pull/1115)
+  - Implement `Display`, `Error`, `Send` and `Sync` for `PyErr` and `PyErrArguments`.
+  - Add `PyErr::instance()` which returns `&PyBaseException`.
+  - `PyErr`'s fields are now an implementation detail. The equivalent values can be accessed with `PyErr::ptype()`, `PyErr::pvalue()` and `PyErr::ptraceback()`.
+  - Change `PyErr::print()` and `PyErr::print_and_set_sys_last_vars()` to take `&self` instead of `self`.
+  - Remove `PyErrValue`, `PyErr::from_value`, `PyErr::into_normalized()`, and `PyErr::normalize()`.
+  - Remove `PyException::into()` and `Into<PyResult<T>>` for `PyErr` and `PyException`.
 - Change `#[pyproto]` to return NotImplemented for operators for which Python can try a reversed operation. #[1072](https://github.com/PyO3/pyo3/pull/1072)
 - `PyModule::add` now uses `IntoPy<PyObject>` instead of `ToPyObject`. #[1124](https://github.com/PyO3/pyo3/pull/1124)
 - Add nested modules as `&PyModule` instead of using the wrapper generated by `#[pymodule]`. [#1143](https://github.com/PyO3/pyo3/pull/1143)

README.md

@@ -67,7 +67,7 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
 /// A Python module implemented in Rust.
 #[pymodule]
 fn string_sum(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(sum_as_string))?;
+    m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
 
     Ok(())
 }

examples/rustapi_module/src/datetime.rs

@@ -215,29 +215,29 @@ impl TzClass {
 
 #[pymodule]
 fn datetime(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(make_date))?;
-    m.add_function(wrap_pyfunction!(get_date_tuple))?;
-    m.add_function(wrap_pyfunction!(date_from_timestamp))?;
-    m.add_function(wrap_pyfunction!(make_time))?;
-    m.add_function(wrap_pyfunction!(get_time_tuple))?;
-    m.add_function(wrap_pyfunction!(make_delta))?;
-    m.add_function(wrap_pyfunction!(get_delta_tuple))?;
-    m.add_function(wrap_pyfunction!(make_datetime))?;
-    m.add_function(wrap_pyfunction!(get_datetime_tuple))?;
-    m.add_function(wrap_pyfunction!(datetime_from_timestamp))?;
+    m.add_function(wrap_pyfunction!(make_date, m)?)?;
+    m.add_function(wrap_pyfunction!(get_date_tuple, m)?)?;
+    m.add_function(wrap_pyfunction!(date_from_timestamp, m)?)?;
+    m.add_function(wrap_pyfunction!(make_time, m)?)?;
+    m.add_function(wrap_pyfunction!(get_time_tuple, m)?)?;
+    m.add_function(wrap_pyfunction!(make_delta, m)?)?;
+    m.add_function(wrap_pyfunction!(get_delta_tuple, m)?)?;
+    m.add_function(wrap_pyfunction!(make_datetime, m)?)?;
+    m.add_function(wrap_pyfunction!(get_datetime_tuple, m)?)?;
+    m.add_function(wrap_pyfunction!(datetime_from_timestamp, m)?)?;
 
     // Python 3.6+ functions
     #[cfg(Py_3_6)]
     {
-        m.add_function(wrap_pyfunction!(time_with_fold))?;
+        m.add_function(wrap_pyfunction!(time_with_fold, m)?)?;
         #[cfg(not(PyPy))]
         {
-            m.add_function(wrap_pyfunction!(get_time_tuple_fold))?;
-            m.add_function(wrap_pyfunction!(get_datetime_tuple_fold))?;
+            m.add_function(wrap_pyfunction!(get_time_tuple_fold, m)?)?;
+            m.add_function(wrap_pyfunction!(get_datetime_tuple_fold, m)?)?;
         }
     }
 
-    m.add_function(wrap_pyfunction!(issue_219))?;
+    m.add_function(wrap_pyfunction!(issue_219, m)?)?;
     m.add_class::<TzClass>()?;
 
     Ok(())

examples/rustapi_module/src/othermod.rs

@@ -31,7 +31,7 @@ fn double(x: i32) -> i32 {
 
 #[pymodule]
 fn othermod(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(double))?;
+    m.add_function(wrap_pyfunction!(double, m)?)?;
 
     m.add_class::<ModClass>()?;
 

examples/word-count/src/lib.rs

@@ -56,8 +56,8 @@ fn count_line(line: &str, needle: &str) -> usize {
 #[pymodule]
 fn word_count(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(search))?;
-    m.add_function(wrap_pyfunction!(search_sequential))?;
-    m.add_function(wrap_pyfunction!(search_sequential_allow_threads))?;
+    m.add_function(wrap_pyfunction!(search_sequential, m)?)?;
+    m.add_function(wrap_pyfunction!(search_sequential_allow_threads, m)?)?;
 
     Ok(())
 }

guide/src/exception.md

@@ -61,7 +61,7 @@ use pyo3::exceptions::PyTypeError;
 fn main() {
     let gil = Python::acquire_gil();
     let py = gil.python();
-    PyErr::new::<PyTypeError, _>("Error").restore(py);
+    PyTypeError::new_err("Error").restore(py);
     assert!(PyErr::occurred(py));
     drop(PyErr::fetch(py));
 }
@@ -76,7 +76,7 @@ If you already have a Python exception instance, you can simply call [`PyErr::fr
 PyErr::from_instance(py, err).restore(py);
 ```
 
-If a Rust type exists for the exception, then it is possible to use the `py_err` method.
+If a Rust type exists for the exception, then it is possible to use the `new_err` method.
 For example, each standard exception defined in the `pyo3::exceptions` module
 has a corresponding Rust type, exceptions defined by [`create_exception!`] and [`import_exception!`] macro
 have Rust types as well.
@@ -87,7 +87,7 @@ have Rust types as well.
 # fn check_for_error() -> bool {false}
 fn my_func(arg: PyObject) -> PyResult<()> {
     if check_for_error() {
-        Err(PyValueError::py_err("argument is wrong"))
+        Err(PyValueError::new_err("argument is wrong"))
     } else {
         Ok(())
     }
@@ -115,41 +115,32 @@ fn main() {
 [`Python::is_instance`] calls the underlying [`PyType::is_instance`](https://docs.rs/pyo3/latest/pyo3/types/struct.PyType.html#method.is_instance)
 method to do the actual work.
 
-To check the type of an exception, you can simply do:
+To check the type of an exception, you can similarly do:
 
 ```rust
 # use pyo3::exceptions::PyTypeError;
 # use pyo3::prelude::*;
-# fn main() {
 # let gil = Python::acquire_gil();
 # let py = gil.python();
-# let err = PyTypeError::py_err(());
+# let err = PyTypeError::new_err(());
 err.is_instance::<PyTypeError>(py);
-# }
 ```
 
 ## Handling Rust errors
 
-The vast majority of operations in this library will return [`PyResult<T>`](https://docs.rs/pyo3/latest/pyo3/prelude/type.PyResult.html),
+The vast majority of operations in this library will return
+[`PyResult<T>`](https://docs.rs/pyo3/latest/pyo3/prelude/type.PyResult.html),
 which is an alias for the type `Result<T, PyErr>`.
 
-A [`PyErr`] represents a Python exception.
-Errors within the PyO3 library are also exposed as Python exceptions.
-
-The PyO3 library handles Python exceptions in two stages. During the first stage, a [`PyErr`] instance is
-created. At this stage, holding Python's GIL is not required. During the second stage, an actual Python
-exception instance is created and set active in the Python interpreter.
+A [`PyErr`] represents a Python exception. Errors within the PyO3 library are also exposed as
+Python exceptions.
 
-In simple cases, for custom errors adding an implementation of `std::convert::From<T>` trait
-for this custom error is enough. `PyErr::new` accepts an argument in the form
-of `ToPyObject + 'static`. If the `'static` constraint can not be satisfied or
-more complex arguments are required, the
-[`PyErrArguments`](https://docs.rs/pyo3/latest/pyo3/trait.PyErrArguments.html)
-trait can be implemented. In that case, actual exception argument creation is delayed
-until a `Python` object is available.
+If your code has a custom error type e.g. `MyError`, adding an implementation of
+`std::convert::From<MyError> for PyErr` is usually enough. PyO3 will then automatically convert
+your error to a Python exception when needed.
 
 ```rust
-# use pyo3::{PyErr, PyResult};
+# use pyo3::prelude::*;
 # use pyo3::exceptions::PyOSError;
 # use std::error::Error;
 # use std::fmt;
@@ -170,11 +161,12 @@ until a `Python` object is available.
 # }
 impl std::convert::From<CustomIOError> for PyErr {
     fn from(err: CustomIOError) -> PyErr {
-        PyOSError::py_err(err.to_string())
+        PyOSError::new_err(err.to_string())
     }
 }
 
-fn connect(s: String) -> PyResult<bool> {
+#[pyfunction]
+fn connect(s: String) -> Result<bool, CustomIOError> {
     bind("127.0.0.1:80")?;
     Ok(true)
 }
@@ -195,6 +187,10 @@ fn parse_int(s: String) -> PyResult<usize> {
 
 The code snippet above will raise a `ValueError` in Python if `String::parse()` returns an error.
 
+If lazy construction of the Python exception instance is desired, the
+[`PyErrArguments`](https://docs.rs/pyo3/latest/pyo3/trait.PyErrArguments.html)
+trait can be implemented. In that case, actual exception argument creation is delayed
+until the `PyErr` is needed.
 
 ## Using exceptions defined in Python code
 
@@ -213,7 +209,7 @@ fn tell(file: &PyAny) -> PyResult<u64> {
     use pyo3::exceptions::*;
 
     match file.call_method0("tell") {
-        Err(_) => Err(io::UnsupportedOperation::py_err("not supported: tell")),
+        Err(_) => Err(io::UnsupportedOperation::new_err("not supported: tell")),
         Ok(x) => x.extract::<u64>(),
     }
 }

guide/src/function.md

@@ -36,7 +36,7 @@ fn double(x: usize) -> usize {
 
 #[pymodule]
 fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(double)).unwrap();
+    m.add_function(wrap_pyfunction!(double, m)?).unwrap();
 
     Ok(())
 }
@@ -65,7 +65,7 @@ fn num_kwds(kwds: Option<&PyDict>) -> usize {
 
 #[pymodule]
 fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(num_kwds)).unwrap();
+    m.add_function(wrap_pyfunction!(num_kwds, m)?).unwrap();
     Ok(())
 }
 
@@ -206,7 +206,7 @@ fn pyfunction_with_module(module: &PyModule) -> PyResult<&str> {
 
 #[pymodule]
 fn module_with_fn(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(pyfunction_with_module))
+    m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)
 }
 
 # fn main() {}

guide/src/migration.md

@@ -5,13 +5,88 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
 ## from 0.11.* to 0.12
 
+### `PyErr` has been reworked
+
+In PyO3 `0.12` the `PyErr` type has been re-implemented to be significantly more compatible with
+the standard Rust error handling ecosystem. Specificially `PyErr` now implements
+`Error + Send + Sync`, which are the standard traits used for error types.
+
+While this has necessitated the removal of a number of APIs, the resulting `PyErr` type should now
+be much more easier to work with. The following sections list the changes in detail and how to
+migrate to the new APIs.
+
+#### `PyErr::new` and `PyErr::from_type` now require `Send + Sync` for their argument
+
+For most uses no change will be needed. If you are trying to construct `PyErr` from a value that is
+not `Send + Sync`, you will need to first create the Python object and then use
+`PyErr::from_instance`.
+
+Similarly, any types which implemented `PyErrArguments` will now need to be `Send + Sync`.
+
+#### `PyErr`'s contents are now private
+
+It is no longer possible to access the fields `.ptype`, `.pvalue` and `.ptraceback` of a `PyErr`.
+You should instead now use the new methods `PyErr::ptype()`, `PyErr::pvalue()` and `PyErr::ptraceback()`.
+
+#### `PyErrValue` and `PyErr::from_value` have been removed
+
+As these were part the internals of `PyErr` which have been reworked, these APIs no longer exist.
+
+If you used this API, it is recommended to use `PyException::new_err` (see [the section on
+Exception types](#exception-types-have-been-reworked)).
+
+#### `Into<PyResult<T>>` for `PyErr` has been removed
+
+This implementation was redundant. Just construct the `Result::Err` variant directly.
+
+Before:
+```rust,ignore
+let result: PyResult<()> = PyErr::new::<TypeError, _>("error message").into();
+```
+
+After (also using the new reworked exception types; see the following section):
+```rust
+# use pyo3::{PyErr, PyResult, exceptions::PyTypeError};
+let result: PyResult<()> = Err(PyTypeError::new_err("error message"));
+```
+
+### Exception types have been reworked
+
+Previously exception types were zero-sized marker types purely used to construct `PyErr`. In PyO3
+0.12, these types have been replaced with full definitions and are usable in the same way as `PyAny`, `PyDict` etc. This
+makes it possible to interact with Python exception objects.
+
+The new types also have names starting with the "Py" prefix. For example, before:
+
+```rust,ignore
+let err: PyErr = TypeError::py_err("error message");
+```
+
+After:
+
+```
+# use pyo3::{PyErr, PyResult, Python, type_object::PyTypeObject};
+# use pyo3::exceptions::{PyBaseException, PyTypeError};
+# Python::with_gil(|py| -> PyResult<()> {
+let err: PyErr = PyTypeError::new_err("error message");
+
+// Uses Display for PyErr, new for PyO3 0.12
+assert_eq!(err.to_string(), "TypeError: error message");
+
+// Now possible to interact with exception instances, new for PyO3 0.12
+let instance: &PyBaseException = err.instance(py);
+assert_eq!(instance.getattr("__class__")?, PyTypeError::type_object(py).as_ref());
+# Ok(())
+# }).unwrap();
+```
+
 ### `FromPy` has been removed
-To simplify the PyO3 public conversion trait hierarchy, the `FromPy` has been removed. In PyO3
-`0.11` there were two ways to define the to-Python conversion for a type: `FromPy<T> for PyObject`,
-and `IntoPy<PyObject> for T`.
+To simplify the PyO3 conversion traits, the `FromPy` trait has been removed. Previously there were
+two ways to define the to-Python conversion for a type:
+`FromPy<T> for PyObject` and `IntoPy<PyObject> for T`.
 
-Now, the canonical implementation is always `IntoPy`, so downstream crates may need to adjust
-accordingly.
+Now there is only one way to define the conversion, `IntoPy`, so downstream crates may need to
+adjust accordingly.
 
 Before:
 ```rust,ignore
@@ -85,7 +160,6 @@ let list_ref: &PyList = list_py.as_ref(py);
 # })
 ```
 
-
 ## from 0.10.* to 0.11
 
 ### Stable Rust

guide/src/module.md

@@ -73,7 +73,7 @@ fn subfunction() -> String {
 }
 
 fn init_submodule(module: &PyModule) -> PyResult<()> {
-    module.add_function(wrap_pyfunction!(subfunction))?;
+    module.add_function(wrap_pyfunction!(subfunction, module)?)?;
     Ok(())
 }
 

guide/src/trait_bounds.md

@@ -488,7 +488,7 @@ pub struct UserModel {
 #[pymodule]
 fn trait_exposure(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<UserModel>()?;
-    m.add_function(wrap_pyfunction!(solve_wrapper))?;
+    m.add_function(wrap_pyfunction!(solve_wrapper, m)?)?;
     Ok(())
 }
 

pyo3-derive-backend/src/from_pyobject.rs

@@ -78,7 +78,7 @@ impl<'a> Enum<'a> {
                 .map(|s| format!("{} ({})", s.to_string_lossy(), type_name))
                 .unwrap_or_else(|_| type_name.to_string());
             let err_msg = format!("Can't convert {} to {}", from, #error_names);
-            Err(::pyo3::exceptions::PyTypeError::py_err(err_msg))
+            Err(::pyo3::exceptions::PyTypeError::new_err(err_msg))
         )
     }
 }
@@ -263,7 +263,7 @@ impl<'a> Container<'a> {
         quote!(
             let s = <::pyo3::types::PyTuple as ::pyo3::conversion::PyTryFrom>::try_from(obj)?;
             if s.len() != #len {
-                return Err(::pyo3::exceptions::PyValueError::py_err(#msg))
+                return Err(::pyo3::exceptions::PyValueError::new_err(#msg))
             }
             let slice = s.as_slice();
             Ok(#self_ty(#fields))