Merge pull request #332 from PyO3/inventory

 Allow 0..n pymethod blocks without specialization

Cargo.toml

@@ -20,20 +20,21 @@ appveyor = { repository = "fafhrd91/pyo3" }
 codecov = { repository = "PyO3/pyo3", branch = "master", service = "github" }
 
 [dependencies]
-libc = "0.2.43"
+libc = "0.2.48"
 spin = "0.5.0"
 num-traits = "0.2.6"
 pyo3cls = { path = "pyo3cls", version = "=0.6.0-alpha.2" }
 mashup = "0.1.9"
 num-complex = { version = "0.2.1", optional = true }
+inventory = "0.1.3"
 
 [dev-dependencies]
-assert_approx_eq = "1.0.0"
+assert_approx_eq = "1.1.0"
 docmatic = "0.1.2"
 indoc = "0.3.1"
 
 [build-dependencies]
-regex = "1.0.5"
+regex = "1.1.0"
 version_check = "0.1.5"
 
 [features]

Makefile

@@ -4,7 +4,11 @@ test:
 	cargo test
 	cargo clippy
 	tox
-	for example in examples/*; do tox -e py --workdir $$example; done
+	for example in examples/*; do tox -e py -c $$example/tox.ini; done
+
+test_py3:
+	tox -e py3
+	for example in examples/*; do tox -e py3 -c $$example/tox.ini; done
 
 publish:
 	cargo test

examples/rustapi_module/src/lib.rs

@@ -1,5 +1,3 @@
-#![feature(specialization)]
-
 pub mod datetime;
 pub mod dict_iter;
 pub mod othermod;

examples/rustapi_module/tests/test_datetime.py

@@ -296,4 +296,4 @@ def test_tz_class_introspection():
     tzi = rdt.TzClass()
 
     assert tzi.__class__ == rdt.TzClass
-    assert repr(tzi).startswith("<rustapi_module.datetime.TzClass object at")
+    assert repr(tzi).startswith("<TzClass object at")

examples/word-count/src/lib.rs

@@ -1,6 +1,5 @@
 // Source adopted from
 // https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs
-#![feature(specialization)]
 
 use pyo3::prelude::*;
 use pyo3::wrap_pyfunction;

guide/src/class.md

@@ -5,7 +5,6 @@
 To define python custom class, rust struct needs to be annotated with `#[pyclass]` attribute.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 
 #[pyclass]
@@ -42,11 +41,8 @@ To declare a constructor, you need to define a class method and annotate it with
 attribute. Only the python `__new__` method can be specified, `__init__` is not available.
 
 ```rust
-# #![feature(specialization)]
-#
 # use pyo3::prelude::*;
 # use pyo3::PyRawObject;
-
 #[pyclass]
 struct MyClass {
    num: i32,
@@ -86,7 +82,6 @@ By default `PyObject` is used as default base class. To override default base cl
 with value of custom class struct. Subclass must call parent's `__new__` method.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # use pyo3::PyRawObject;
 #[pyclass]
@@ -136,7 +131,6 @@ Descriptor methods can be defined in
 attributes. i.e.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -161,7 +155,6 @@ Descriptor name becomes function name with prefix removed. This is useful in cas
 rust's special keywords like `type`.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -190,7 +183,6 @@ Also both `#[getter]` and `#[setter]` attributes accepts one parameter.
 If parameter is specified, it is used and property name. i.e.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -218,7 +210,6 @@ In this case property `number` is defined. And it is available from python code
 For simple cases you can also define getters and setters in your Rust struct field definition, for example:
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 #[pyclass]
 struct MyClass {
@@ -237,7 +228,6 @@ wrappers for all functions in this block with some variations, like descriptors,
 class method static methods, etc.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -265,7 +255,6 @@ The return type must be `PyResult<T>` for some `T` that implements `IntoPyObject
 get injected by method wrapper. i.e
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -289,7 +278,6 @@ To specify class method for custom class, method needs to be annotated
 with`#[classmethod]` attribute.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -321,7 +309,6 @@ with `#[staticmethod]` attribute. The return type must be `PyResult<T>`
 for some `T` that implements `IntoPyObject`.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -344,7 +331,6 @@ To specify custom `__call__` method for custom class, call method needs to be an
 with `#[call]` attribute. Arguments of the method are specified same as for instance method.
 
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 # #[pyclass]
 # struct MyClass {
@@ -387,7 +373,6 @@ Each parameter could one of following type:
 
 Example:
 ```rust
-# #![feature(specialization)]
 # use pyo3::prelude::*;
 #
 # #[pyclass]
@@ -556,3 +541,13 @@ impl PyIterProtocol for MyIterator {
     }
 }
 ```
+
+## Manually implementing pyclass
+
+TODO: Which traits to implement (basically `PyTypeCreate: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol + Sized`) and what they mean.
+
+## How methods are implemented
+
+Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while pyo3 needs a trait with a function that returns all methods. Since it's impossible make the code generation in pyclass dependent on whether there is an impl block, we'd need to make to implement the trait on `#[pyclass]` and override the implementation in `#[pymethods]`, which is to my best knowledge only possible with the specialization feature, which is can't be used on stable.
+
+To escape this we use [inventory](https://github.com/dtolnay/inventory), which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_derive_backend::py_class::impl_inventory` for more details.

pyo3-derive-backend/src/py_class.rs

@@ -63,6 +63,40 @@ fn parse_descriptors(item: &mut syn::Field) -> Vec<FnType> {
     descs
 }
 
+/// The orphan rule disallows using a generic inventory struct, so we create the whole boilerplate
+/// once per class
+fn impl_inventory(cls: &syn::Ident) -> TokenStream {
+    // Try to build a unique type that gives a hint about it's function when
+    // it comes up in error messages
+    let name = cls.to_string() + "GeneratedPyo3Inventory";
+    let inventory_cls = syn::Ident::new(&name, Span::call_site());
+
+    quote! {
+        #[doc(hidden)]
+        pub struct #inventory_cls {
+            methods: &'static [::pyo3::class::PyMethodDefType],
+        }
+
+        impl ::pyo3::class::methods::PyMethodsInventory for #inventory_cls {
+            fn new(methods: &'static [::pyo3::class::PyMethodDefType]) -> Self {
+                Self {
+                    methods
+                }
+            }
+
+            fn get_methods(&self) -> &'static [::pyo3::class::PyMethodDefType] {
+                self.methods
+            }
+        }
+
+        impl ::pyo3::class::methods::PyMethodsInventoryDispatch for #cls {
+            type InventoryType = #inventory_cls;
+        }
+
+        ::pyo3::inventory::collect!(#inventory_cls);
+    }
+}
+
 fn impl_class(
     cls: &syn::Ident,
     base: &syn::TypePath,
@@ -136,6 +170,8 @@ fn impl_class(
         quote! {0}
     };
 
+    let inventory_impl = impl_inventory(&cls);
+
     quote! {
         impl ::pyo3::typeob::PyTypeInfo for #cls {
             type Type = #cls;
@@ -197,6 +233,8 @@ fn impl_class(
             }
         }
 
+        #inventory_impl
+
         #extra
     }
 }
@@ -287,12 +325,10 @@ fn impl_descriptors(cls: &syn::Type, descriptors: Vec<(syn::Field, Vec<FnType>)>
     quote! {
         #(#methods)*
 
-        impl ::pyo3::class::methods::PyPropMethodsProtocolImpl for #cls {
-            fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
-                static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
-                    #(#py_methods),*
-                ];
-                METHODS
+        ::pyo3::inventory::submit! {
+            #![crate = pyo3] {
+                type ClsInventory = <#cls as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
+                <ClsInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#py_methods),*])
             }
         }
     }

pyo3-derive-backend/src/py_impl.rs

@@ -31,12 +31,10 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec<syn::ImplItem>) -> TokenStre
     }
 
     quote! {
-        impl ::pyo3::class::methods::PyMethodsProtocolImpl for #ty {
-            fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
-                static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
-                    #(#methods),*
-                ];
-                METHODS
+       ::pyo3::inventory::submit! {
+            #![crate = pyo3] {
+                type TyInventory = <#ty as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
+                <TyInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#methods),*])
             }
         }
     }

src/class/methods.rs

@@ -58,10 +58,13 @@ pub struct PySetterDef {
 }
 
 unsafe impl Sync for PyMethodDef {}
+
 unsafe impl Sync for ffi::PyMethodDef {}
 
 unsafe impl Sync for PyGetterDef {}
+
 unsafe impl Sync for PySetterDef {}
+
 unsafe impl Sync for ffi::PyGetSetDef {}
 
 impl PyMethodDef {
@@ -110,21 +113,40 @@ impl PySetterDef {
     }
 }
 
-#[doc(hidden)]
-/// The pymethods macro implements this trait so the methods are added to the object
-pub trait PyMethodsProtocolImpl {
-    fn py_methods() -> &'static [PyMethodDefType] {
-        &[]
-    }
+#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
+/// This trait is implemented for all pyclass so to implement the [PyMethodsProtocol]
+/// through inventory
+pub trait PyMethodsInventoryDispatch {
+    /// This allows us to get the inventory type when only the pyclass is in scope
+    type InventoryType: PyMethodsInventory;
 }
 
-impl<T> PyMethodsProtocolImpl for T {}
+#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
+/// Allows arbitrary pymethod blocks to submit their methods, which are eventually collected by pyclass
+pub trait PyMethodsInventory: inventory::Collect {
+    /// Create a new instance
+    fn new(methods: &'static [PyMethodDefType]) -> Self;
 
-#[doc(hidden)]
-pub trait PyPropMethodsProtocolImpl {
-    fn py_methods() -> &'static [PyMethodDefType] {
-        &[]
-    }
+    /// Returns the methods for a single impl block
+    fn get_methods(&self) -> &'static [PyMethodDefType];
 }
 
-impl<T> PyPropMethodsProtocolImpl for T {}
+/// The implementation of tis trait defines which methods a python type has.
+///
+/// For pyclass derived structs this is implemented by collecting all impl blocks through inventory
+pub trait PyMethodsProtocol {
+    /// Returns all methods that are defined for a class
+    fn py_methods() -> Vec<&'static PyMethodDefType>;
+}
+
+impl<T> PyMethodsProtocol for T
+where
+    T: PyMethodsInventoryDispatch,
+{
+    fn py_methods() -> Vec<&'static PyMethodDefType> {
+        inventory::iter::<T::InventoryType>
+            .into_iter()
+            .flat_map(PyMethodsInventory::get_methods)
+            .collect()
+    }
+}

src/lib.rs

@@ -55,8 +55,6 @@
 //! **`src/lib.rs`**
 //!
 //! ```rust
-//! #![feature(specialization)]
-//!
 //! use pyo3::prelude::*;
 //! use pyo3::wrap_pyfunction;
 //!
@@ -101,8 +99,6 @@
 //! Example program displaying the value of `sys.version`:
 //!
 //! ```rust
-//! #![feature(specialization)]
-//!
 //! use pyo3::prelude::*;
 //! use pyo3::types::PyDict;
 //!
@@ -146,12 +142,12 @@ pub use crate::pythonrun::{init_once, prepare_freethreaded_python, GILGuard, GIL
 pub use crate::typeob::{PyObjectAlloc, PyRawObject, PyTypeInfo};
 pub use crate::types::exceptions;
 
-// We need those types in the macro exports
-#[doc(hidden)]
-pub use libc;
 // We need that reexport for wrap_function
 #[doc(hidden)]
 pub use mashup;
+// We need that reexport for pymethods
+#[doc(hidden)]
+pub use inventory;
 
 /// Rust FFI declarations for Python
 pub mod ffi;
@@ -207,7 +203,7 @@ pub mod proc_macro {
 macro_rules! wrap_pyfunction {
     ($function_name:ident) => {{
         // Get the mashup macro and its helpers into scope
-        use $crate::mashup::*;
+        use pyo3::mashup::*;
 
         mashup! {
             // Make sure this ident matches the one in function_wrapper_ident
@@ -227,7 +223,7 @@ macro_rules! wrap_pyfunction {
 #[macro_export]
 macro_rules! wrap_pymodule {
     ($module_name:ident) => {{
-        use $crate::mashup::*;
+        use pyo3::mashup::*;
 
         mashup! {
             m["method"] = PyInit_ $module_name;