feat: package table (#841)

This is an implementation of #669.

---------

Signed-off-by: Henry Schreiner <[email protected]>

README.md

@@ -212,7 +212,8 @@ sdist.cmake = false
 # A list of packages to auto-copy into the wheel. If this is not set, it will
 # default to the first of ``src/<package>``, ``python/<package>``, or
 # ``<package>`` if they exist.  The prefix(s) will be stripped from the package
-# name inside the wheel.
+# name inside the wheel. If a dict, provides a mapping of package name to source
+# directory.
 wheel.packages = ["src/<package>", "python/<package>", "<package>"]
 
 # The Python tags. The default (empty string) will use the default Python

docs/configuration.md

@@ -249,8 +249,24 @@ list packages explicitly, you can. The final path element is the package.
 wheel.packages = ["python/src/mypackage"]
 ```
 
-Or you can disable Python file inclusion entirely, and rely only on CMake's
-install mechanism, you can do that instead:
+This can also be a table, allowing full customization of where a source package
+maps to a wheel directory. The final components of both paths must match due to the
+way editable installs work. The equivalent of the above is:
+
+```toml
+[tool.scikit-build.wheel.packages]
+mypackage = "python/src/mypackage"
+```
+
+But you can also do more complex moves:
+
+```toml
+[tool.scikit-build.wheel.packages]
+"mypackage/subpackage" = "python/src/subpackage"
+```
+
+You can disable Python file inclusion entirely, and rely only on CMake's
+install mechanism:
 
 ```toml
 [tool.scikit-build]

src/scikit_build_core/build/_pathutil.py

@@ -9,7 +9,7 @@
 from ._file_processor import each_unignored_file
 
 if TYPE_CHECKING:
-    from collections.abc import Generator, Sequence
+    from collections.abc import Generator, Mapping, Sequence
 
 __all__ = ["is_valid_module", "packages_to_file_mapping", "path_to_module", "scantree"]
 
@@ -36,24 +36,28 @@ def path_to_module(path: Path) -> str:
 
 def packages_to_file_mapping(
     *,
-    packages: Sequence[str],
+    packages: Mapping[str, str],
     platlib_dir: Path,
     include: Sequence[str],
     src_exclude: Sequence[str],
     target_exclude: Sequence[str],
 ) -> dict[str, str]:
+    """
+    This will output a mapping of source files to target files.
+    """
     mapping = {}
     exclude_spec = pathspec.GitIgnoreSpec.from_lines(target_exclude)
-    for package in packages:
-        source_package = Path(package)
-        base_path = source_package.parent
+    for package_str, source_str in packages.items():
+        package_dir = Path(package_str)
+        source_dir = Path(source_str)
+
         for filepath in each_unignored_file(
-            source_package,
+            source_dir,
             include=include,
             exclude=src_exclude,
         ):
-            rel_path = filepath.relative_to(base_path)
-            target_path = platlib_dir / rel_path
+            rel_path = filepath.relative_to(source_dir)
+            target_path = platlib_dir / package_dir / rel_path
             if not exclude_spec.match_file(rel_path) and not target_path.is_file():
                 mapping[str(filepath)] = str(target_path)
 

src/scikit_build_core/build/wheel.py

@@ -6,6 +6,7 @@
 import sys
 import sysconfig
 import tempfile
+from collections.abc import Mapping
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -86,21 +87,23 @@ def _make_editable(
 
 def _get_packages(
     *,
-    packages: Sequence[str] | None,
+    packages: Sequence[str] | Mapping[str, str] | None,
     name: str,
-) -> list[str]:
+) -> dict[str, str]:
     if packages is not None:
-        return list(packages)
+        if isinstance(packages, Mapping):
+            return dict(packages)
+        return {str(Path(p).name): p for p in packages}
 
     # Auto package discovery
-    packages = []
+    packages = {}
     for base_path in (Path("src"), Path("python"), Path()):
         path = base_path / name
         if path.is_dir() and (
             (path / "__init__.py").is_file() or (path / "__init__.pyi").is_file()
         ):
             logger.info("Discovered Python package at {}", path)
-            packages += [str(path)]
+            packages[name] = str(path)
             break
     else:
         logger.debug("Didn't find a Python package for {}", name)
@@ -457,7 +460,9 @@ def _build_wheel_impl_impl(
         ) as wheel:
             wheel.build(wheel_dirs, exclude=settings.wheel.exclude)
 
-            str_pkgs = (str(Path.cwd().joinpath(p).parent.resolve()) for p in packages)
+            str_pkgs = (
+                str(Path.cwd().joinpath(p).parent.resolve()) for p in packages.values()
+            )
             if editable and settings.editable.mode == "redirect":
                 reload_dir = build_dir.resolve() if settings.build_dir else None
 

src/scikit_build_core/resources/scikit-build.schema.json

@@ -167,11 +167,23 @@
       "additionalProperties": false,
       "properties": {
         "packages": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "description": "A list of packages to auto-copy into the wheel. If this is not set, it will default to the first of ``src/<package>``, ``python/<package>``, or ``<package>`` if they exist.  The prefix(s) will be stripped from the package name inside the wheel."
+          "oneOf": [
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            {
+              "type": "object",
+              "patternProperties": {
+                ".+": {
+                  "type": "string"
+                }
+              }
+            }
+          ],
+          "description": "A list of packages to auto-copy into the wheel. If this is not set, it will default to the first of ``src/<package>``, ``python/<package>``, or ``<package>`` if they exist.  The prefix(s) will be stripped from the package name inside the wheel. If a dict, provides a mapping of package name to source directory."
         },
         "py-api": {
           "type": "string",

src/scikit_build_core/settings/json_schema.py

@@ -142,7 +142,13 @@ def convert_type(t: Any, *, normalize_keys: bool) -> dict[str, Any]:
                 next(iter(a for a in args if a is not type(None))),
                 normalize_keys=normalize_keys,
             )
-        return {"oneOf": [convert_type(a, normalize_keys=normalize_keys) for a in args]}
+        return {
+            "oneOf": [
+                convert_type(a, normalize_keys=normalize_keys)
+                for a in args
+                if a is not type(None)
+            ]
+        }
     if origin is Literal:
         return {"enum": list(args)}
 

src/scikit_build_core/settings/skbuild_model.py

@@ -142,12 +142,13 @@ class SDistSettings:
 
 @dataclasses.dataclass
 class WheelSettings:
-    packages: Optional[List[str]] = None
+    packages: Optional[Union[List[str], Dict[str, str]]] = None
     """
     A list of packages to auto-copy into the wheel. If this is not set, it will
     default to the first of ``src/<package>``, ``python/<package>``, or
     ``<package>`` if they exist.  The prefix(s) will be stripped from the
-    package name inside the wheel.
+    package name inside the wheel. If a dict, provides a mapping of package
+    name to source directory.
     """
 
     py_api: str = ""

src/scikit_build_core/settings/skbuild_read_settings.py

@@ -205,6 +205,13 @@ def __init__(
                 )
                 raise CMakeConfigError(msg)
 
+        if isinstance(self.settings.wheel.packages, dict):
+            for key, value in self.settings.wheel.packages.items():
+                if Path(key).name != Path(value).name:
+                    rich_error(
+                        "wheel.packages table must match in the last component of the paths"
+                    )
+
         if self.settings.editable.rebuild:
             if self.settings.editable.mode == "inplace":
                 rich_error("editable rebuild is incompatible with inplace mode")

src/scikit_build_core/settings/skbuild_schema.py

@@ -157,6 +157,10 @@ def generate_skbuild_schema(tool_name: str = "scikit-build") -> dict[str, Any]:
             kk: {"$ref": "#/$defs/inherit"}
             for kk, vv in v["properties"].items()
             if vv.get("type", "") in {"object", "array"}
+            or any(
+                vvv.get("type", "") in {"object", "array"}
+                for vvv in vv.get("oneOf", {})
+            )
         }
         for k, v in schema["properties"].items()
         if v.get("type", "") == "object"

src/scikit_build_core/settings/sources.py

@@ -36,6 +36,7 @@
 - ``Union[str, ...]``: Supports other input types in TOML form (bool currently). Otherwise a string.
 - ``List[T]``: A list of items. `;` separated supported in EnvVar/config forms. T can be a dataclass (TOML only).
 - ``Dict[str, T]``: A table of items. TOML supports a layer of nesting. Any is supported as an item type.
+- ``Union[list[T], Dict[str, T]]`` (TOML only): A list or dict of items.
 - ``Literal[...]``: A list of strings, the result must be in the list.
 - ``Annotated[Dict[...], "EnvVar"]``: A dict of items, where each item can be a string or a dict with "env" and "default" keys.
 
@@ -246,6 +247,9 @@ def get_item(
 
     @classmethod
     def convert(cls, item: str, target: type[Any]) -> object:
+        """
+        Convert an item from the environment (always a string) into a target type.
+        """
         target, _ = _process_annotated(target)
         raw_target = _get_target_raw_type(target)
         if dataclasses.is_dataclass(raw_target):
@@ -265,6 +269,23 @@ def convert(cls, item: str, target: type[Any]) -> object:
         if raw_target is Union and str in get_args(target):
             return item
 
+        if raw_target is Union:
+            args = {_get_target_raw_type(t): t for t in get_args(target)}
+            if str in args:
+                return item
+            if dict in args and "=" in item:
+                items = (i.strip().split("=") for i in item.split(";"))
+                return {
+                    k: cls.convert(v, _get_inner_type(args[dict])) for k, v in items
+                }
+            if list in args:
+                return [
+                    cls.convert(i.strip(), _get_inner_type(args[list]))
+                    for i in item.split(";")
+                ]
+            msg = f"Can't convert into {target}"
+            raise TypeError(msg)
+
         if raw_target is Literal:
             if item not in get_args(_process_union(target)):
                 msg = f"{item!r} not in {get_args(_process_union(target))!r}"
@@ -376,21 +397,32 @@ def convert(
             if isinstance(item, list):
                 return [cls.convert(i, _get_inner_type(target)) for i in item]
             if isinstance(item, (dict, bool)):
-                msg = f"Expected {target}, got {type(item).__name__}"
+                msg = f"Expected {target}, got {type(item)}"
                 raise TypeError(msg)
             return [
                 cls.convert(i.strip(), _get_inner_type(target)) for i in item.split(";")
             ]
         if raw_target is dict:
             assert not isinstance(item, (str, list, bool))
             return {k: cls.convert(v, _get_inner_type(target)) for k, v in item.items()}
+        if raw_target is Union:
+            args = {_get_target_raw_type(t): t for t in get_args(target)}
+            if str in args:
+                return item
+            if dict in args and isinstance(item, dict):
+                return {
+                    k: cls.convert(v, _get_inner_type(args[dict]))
+                    for k, v in item.items()
+                }
+            if list in args and isinstance(item, list):
+                return [cls.convert(i, _get_inner_type(args[list])) for i in item]
+            msg = f"Can't convert into {target}"
+            raise TypeError(msg)
         if isinstance(item, (list, dict)):
             msg = f"Expected {target}, got {type(item).__name__}"
             raise TypeError(msg)
         if raw_target is bool:
             return item if isinstance(item, bool) else _process_bool(item)
-        if raw_target is Union and str in get_args(target):
-            return item
         if raw_target is Literal:
             if item not in get_args(_process_union(target)):
                 msg = f"{item!r} not in {get_args(_process_union(target))!r}"
@@ -453,6 +485,9 @@ def get_item(self, *fields: str, is_dict: bool) -> Any:  # noqa: ARG002
 
     @classmethod
     def convert(cls, item: Any, target: type[Any]) -> object:
+        """
+        Convert an ``item`` from TOML into a ``target`` type.
+        """
         target, annotations = _process_annotated(target)
         raw_target = _get_target_raw_type(target)
         if dataclasses.is_dataclass(raw_target):
@@ -482,8 +517,17 @@ def convert(cls, item: Any, target: type[Any]) -> object:
             return {k: cls.convert(v, _get_inner_type(target)) for k, v in item.items()}
         if raw_target is Any:
             return item
-        if raw_target is Union and type(item) in get_args(target):
-            return item
+        if raw_target is Union:
+            args = {_get_target_raw_type(t): t for t in get_args(target)}
+            if type(item) in args:
+                if isinstance(item, dict):
+                    return {
+                        k: cls.convert(v, _get_inner_type(args[dict]))
+                        for k, v in item.items()
+                    }
+                if isinstance(item, list):
+                    return [cls.convert(i, _get_inner_type(args[list])) for i in item]
+                return item
         if raw_target is Literal:
             if item not in get_args(_process_union(target)):
                 msg = f"{item!r} not in {get_args(_process_union(target))!r}"

tests/test_editable_unit.py

@@ -134,7 +134,7 @@ def test_navigate_editable_pkg(editable_package: EditablePackage, virtualenv: VE
     site_packages, pkg_dir, src_pkg_dir = editable_package
 
     # Create a fake editable install
-    packages = ["pkg"]
+    packages = {"pkg": "pkg"}
     mapping = packages_to_file_mapping(
         packages=packages,
         platlib_dir=site_packages,