From 7525d7e74045ed14b965733e5bc21b70977754de Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 22 Nov 2024 00:35:33 +0100 Subject: [PATCH] doc: add documentation for using extension modules --- docs/building-extension-modules.md | 300 +++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 docs/building-extension-modules.md diff --git a/docs/building-extension-modules.md b/docs/building-extension-modules.md new file mode 100644 index 00000000000..dca194df485 --- /dev/null +++ b/docs/building-extension-modules.md @@ -0,0 +1,300 @@ +--- +title: "Building extension modules" +draft: false +type: docs +layout: single + +menu: + docs: + weight: 125 +--- + +# Building Extension Modules + +{{% warning %}} +While this feature has been around since almost the beginning of the Poetry project and has needed minimal changes, +it is still considered unstable. You can participate in the discussions about stabilizing this feature +[here](https://github.com/python-poetry/poetry/issues/2740). + +And as always, your contributions towards the goal of improving this feature are also welcome. +{{% /warning %}} + +Poetry allows a project developer to introduce support for, build and distribute native extensions within their project. +In order to achieve this, at the highest level, the following steps are required. + +{{< steps >}} +{{< step >}} +**Add Build Dependencies** + +The build dependencies, in this context, refer to those Python packages that required in order to successfully execute +your build script. Common examples include `cpython`, `meson`, `maturin`, `setuptools` etc., depending on how your +extension is built. + +{{% note %}} +You must assume that only python built-ins are available by default in a build environment. This means, if you use need +even packages like `setuptools`, it must be explicitly declared. +{{% /note %}} + +The necessary build dependencies must be added to the `build-system.requires` section of your `pyproject.toml` file. + +```toml +[build-system] +requires = ["poetry-core", "setuptools", "cython"] +build-backend = "poetry.core.masonry.api" +``` + +{{% note %}} +If you wish to develop the build script within your project's virtual environment, then you must also add the +dependencies to your project explicitly to a dependency group - the name of which is not important. + +```sh +poetry add --group=build setuptools cython +``` +{{% /note %}} + +{{< /step >}} + +{{< step >}} +**Add Build Script** + +The build script can be free-form Python script that uses any dependency specified in the previous step. This can be +named as needed, but **must** be located in the project root and also **must** be included in your source distribution. +You can see the [example snippets section]({{< relref "#example-snippets" >}}) for inspiration. + +```toml +[tool.poetry.build] +script = "build-extension.py" +``` + +{{% note %}} +The name of the build script is arbitrary. Common practice has been to name it `build.py`, however this could have +undesired consequences. It is also recommended that the script be, if feasible placed inside a subdirectory like +`contrib` or `src`. +{{% /note %}} + +{{< /step >}} + +{{< step >}} +**Specify Distribution Files** + +{{% warning %}} +The following is an example, and should not be considered as complete. +{{% /warning %}} + +```toml +packages = [ + { include = "package", from = "src" } +] +include = [ + { path = "src/package/**/*.so", format = "wheel" }, + { path = "build.py", format = "sdist" } +] +exclude = [ + { path = "build.py", format = "wheel" }, + { path = "src/package/**/*.c", format = "wheel" } +] +``` + +The key takeaway here should be the following. You can refer to the [`pyproject.toml`]({{< relref "pyproject#exclude-and-include" >}}) +documentation for information on each of the relevant sections. + +1. Include your build outputs in your wheel. +2. Exclude your build inputs from your wheel. +3. Include your build inputs to your source distribution. + +{{< /step >}} + +{{< /steps >}} + +## Example Snippets + +### Cython + +{{< tabs tabTotal="2" tabID1="cython-pyproject" tabName1="pyproject.toml" tabID2="cython-build-script" tabName2="build-extension.py">}} + +{{< tab tabID="cython-pyproject" >}} + +```toml +[tool.poetry.build] +script = "build-extension.py" +``` + +{{< /tab >}} + +{{< tab tabID="cython-build-script" >}} + +```py +from __future__ import annotations + +import os +import shutil + +from Cython.Build import cythonize +from setuptools import Distribution +from setuptools import Extension +from setuptools.command.build_ext import build_ext + +COMPILE_ARGS = ["-march=native", "-O3", "-msse", "-msse2", "-mfma", "-mfpmath=sse"] +LINK_ARGS = [] +INCLUDE_DIRS = [] +LIBRARIES = ["m"] + + +def build(): + extensions = [ + Extension( + "*", + ["src/package/*.pyx"], + extra_compile_args=COMPILE_ARGS, + extra_link_args=LINK_ARGS, + include_dirs=INCLUDE_DIRS, + libraries=LIBRARIES, + ) + ] + ext_modules = cythonize( + extensions, + include_path=INCLUDE_DIRS, + compiler_directives={"binding": True, "language_level": 3}, + ) + + distribution = Distribution({ + "name": "extended", + "ext_modules": ext_modules + }) + + cmd = build_ext(distribution) + cmd.ensure_finalized() + cmd.run() + + # Copy built extensions back to the project + for output in cmd.get_outputs(): + relative_extension = os.path.relpath(output, cmd.build_lib) + shutil.copyfile(output, relative_extension) + mode = os.stat(relative_extension).st_mode + mode |= (mode & 0o444) >> 2 + os.chmod(relative_extension, mode) + + +if __name__ == "__main__": + build() +``` + +{{< /tab >}} + +{{< /tabs >}} + +### Meson + +{{< tabs tabTotal="2" tabID1="meson-pyproject" tabName1="pyproject.toml" tabID2="meson-build-script" tabName2="build-extension.py">}} + +{{< tab tabID="meson-pyproject" >}} + +```toml +[tool.poetry.build] +script = "build-extension.py" + +[build-system] +requires = ["poetry-core", "meson"] +build-backend = "poetry.core.masonry.api" +``` + +{{< /tab >}} + +{{< tab tabID="meson-build-script" >}} + +```py +from __future__ import annotations + +import subprocess + +from pathlib import Path + + +def meson(*args): + subprocess.call(["meson", *args]) + + +def build(): + build_dir = Path(__file__).parent.joinpath("build") + build_dir.mkdir(parents=True, exist_ok=True) + + meson("setup", build_dir.as_posix()) + meson("compile", "-C", build_dir.as_posix()) + meson("install", "-C", build_dir.as_posix()) + + +if __name__ == "__main__": + build() +``` + +{{< /tab >}} + +{{< /tabs >}} + +### Maturin + +{{< tabs tabTotal="2" tabID1="maturin-pyproject" tabName1="pyproject.toml" tabID2="maturin-build-script" tabName2="build-extension.py">}} + +{{< tab tabID="maturin-pyproject" >}} + +```toml +[tool.poetry.build] +script = "build-extension.py" + +[build-system] +requires = ["poetry-core", "maturin"] +build-backend = "poetry.core.masonry.api" +``` + +{{< /tab >}} + +{{< tab tabID="maturin-build-script" >}} + +```py +import os +import shlex +import shutil +import subprocess +import zipfile + +from pathlib import Path + + +def maturin(*args): + subprocess.call(["maturin", *list(args)]) + + +def build(): + build_dir = Path(__file__).parent.joinpath("build") + build_dir.mkdir(parents=True, exist_ok=True) + + wheels_dir = Path(__file__).parent.joinpath("target/wheels") + if wheels_dir.exists(): + shutil.rmtree(wheels_dir) + + cargo_args = [] + if os.getenv("MATURIN_BUILD_ARGS"): + cargo_args = shlex.split(os.getenv("MATURIN_BUILD_ARGS", "")) + + maturin("build", "-r", *cargo_args) + + # We won't use the wheel built by maturin directly since + # we want Poetry to build it but, we need to retrieve the + # compiled extensions from the maturin wheel. + wheel = next(iter(wheels_dir.glob("*.whl"))) + with zipfile.ZipFile(wheel.as_posix()) as whl: + whl.extractall(wheels_dir.as_posix()) + + for extension in wheels_dir.rglob("**/*.so"): + shutil.copyfile(extension, Path(__file__).parent.joinpath(extension.name)) + + shutil.rmtree(wheels_dir) + + +if __name__ == "__main__": + build() +``` + +{{< /tab >}} + +{{< /tabs >}}