fix(puzzle): handle self-referential extras

Co-authored-by: David Hotham <[email protected]>

src/poetry/puzzle/provider.py

@@ -460,18 +460,29 @@ def complete_package(
             dependency = dependency_package.dependency
             requires = package.requires
 
-        optional_dependencies = []
+        found_extras = set()
+        optional_dependencies = set()
         _dependencies = []
 
-        # If some extras/features were required, we need to
-        # add a special dependency representing the base package
-        # to the current package
         if dependency.extras:
-            for extra in dependency.extras:
-                if extra not in package.extras:
+            # Find all the optional dependencies that are wanted - taking care to allow
+            # for self-referential extras.
+            stack = list(dependency.extras)
+            while stack:
+                extra = stack.pop()
+                if extra in found_extras:
                     continue
+                found_extras.add(extra)
 
-                optional_dependencies += [d.name for d in package.extras[extra]]
+                extra_dependencies = package.extras.get(extra, [])
+                for extra_dependency in extra_dependencies:
+                    if extra_dependency.name == dependency.name:
+                        stack += list(extra_dependency.extras)
+                    else:
+                        optional_dependencies.add(extra_dependency.name)
+
+            # If some extras/features were required, we need to add a special dependency
+            # representing the base package to the current package.
 
             dependency_package = dependency_package.with_features(
                 list(dependency.extras)
@@ -507,10 +518,7 @@ def complete_package(
 
             if not package.is_root() and (
                 (dep.is_optional() and dep.name not in optional_dependencies)
-                or (
-                    dep.in_extras
-                    and not set(dep.in_extras).intersection(dependency.extras)
-                )
+                or (dep.in_extras and not set(dep.in_extras).intersection(found_extras))
             ):
                 continue
 

tests/conftest.py

@@ -21,6 +21,8 @@
 from keyring.credentials import SimpleCredential
 from keyring.errors import KeyringError
 from keyring.errors import KeyringLocked
+from packaging.utils import canonicalize_name
+from poetry.core.version.markers import parse_marker
 from pytest import FixtureRequest
 
 from poetry.config.config import Config as BaseConfig
@@ -29,7 +31,9 @@
 from poetry.factory import Factory
 from poetry.layouts import layout
 from poetry.packages.direct_origin import _get_package_from_git
+from poetry.repositories import Repository
 from poetry.repositories import RepositoryPool
+from poetry.repositories.exceptions import PackageNotFoundError
 from poetry.repositories.installed_repository import InstalledRepository
 from poetry.utils.cache import ArtifactCache
 from poetry.utils.env import EnvManager
@@ -39,6 +43,7 @@
 from tests.helpers import MOCK_DEFAULT_GIT_REVISION
 from tests.helpers import TestLocker
 from tests.helpers import TestRepository
+from tests.helpers import get_dependency
 from tests.helpers import get_package
 from tests.helpers import http_setup_redirect
 from tests.helpers import isolated_environment
@@ -57,6 +62,9 @@
     from cleo.io.inputs.argument import Argument
     from cleo.io.inputs.option import Option
     from keyring.credentials import Credential
+    from packaging.utils import NormalizedName
+    from poetry.core.packages.dependency import Dependency
+    from poetry.core.packages.package import Package
     from pytest import Config as PyTestConfig
     from pytest import Parser
     from pytest import TempPathFactory
@@ -66,6 +74,7 @@
     from tests.types import CommandFactory
     from tests.types import FixtureCopier
     from tests.types import FixtureDirGetter
+    from tests.types import PackageFactory
     from tests.types import ProjectFactory
     from tests.types import SetProjectContext
 
@@ -500,6 +509,74 @@ def _factory(
     return _factory
 
 
+@pytest.fixture
+def create_package(repo: Repository) -> PackageFactory:
+    """
+    This function is a pytest fixture that creates a factory function to generate
+    and customize package objects. These packages are added to the default repository
+    fixture and configured with specific versions, optional extras, and self-referenced
+    extras. This helps in setting up package dependencies for testing purposes.
+
+    :return: A factory function that can be used to create and configure packages.
+    """
+
+    def create_new_package(
+        name: str,
+        version: str | None = None,
+        dependencies: list[Dependency] | None = None,
+        extras: dict[str, list[str]] | None = None,
+        self_referenced_extras: dict[str, list[str]] | None = None,
+    ) -> Package:
+        version = version or "1.0"
+        package = get_package(name, version)
+
+        package_extras: dict[NormalizedName, list[Dependency]] = {}
+
+        for extra, dep_names in (extras or {}).items():
+            extra = canonicalize_name(extra)
+
+            if extra not in package_extras:
+                package_extras[extra] = []
+
+            for dep_name in dep_names:
+                try:
+                    pkg = repo.package(dep_name, package.version)
+                except PackageNotFoundError:
+                    pkg = get_package(dep_name, version)
+                    repo.add_package(pkg)
+
+                dep = get_dependency(pkg.name, f"^{pkg.version}", optional=True)
+                dep.marker = parse_marker(f"extra == '{extra}'")
+                package_extras[extra].append(dep)
+                package.add_dependency(dep)
+
+        # add self-referencing extras
+        for extra, target_names in (self_referenced_extras or {}).items():
+            extra = canonicalize_name(extra)
+            package_extras[extra] = [
+                Factory.create_dependency(
+                    package.name,
+                    {
+                        "version": "*",
+                        "extras": [
+                            canonicalize_name(target) for target in target_names
+                        ],
+                    },
+                )
+            ]
+
+        package.extras = package_extras
+
+        for dependency in dependencies or []:
+            package.add_dependency(dependency)
+
+        repo.add_package(package)
+
+        return package
+
+    return create_new_package
+
+
 @pytest.fixture(autouse=True)
 def set_simple_log_formatter() -> None:
     """

tests/installation/fixtures/with-self-referencing-extras-all-deep.test

@@ -0,0 +1,55 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = "^1.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = "^1.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+a = {version = "1.0", extras = ["all"]}
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-all-top.test

@@ -0,0 +1,43 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = "^1.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = "^1.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-b-markers.test

@@ -0,0 +1,44 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = "^1.0", optional = true, markers = "extra == \"download\""}
+install-package = {version = "^1.0", optional = true, markers = "extra == \"install\""}
+
+[package.extras]
+all = ["a[download,install]"]
+b = ["a[all] ; python_version == \"0\""]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[[package]]
+name = "install-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-deep.test

@@ -0,0 +1,33 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+a = "1.0"
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"

tests/installation/fixtures/with-self-referencing-extras-download-deep.test

@@ -0,0 +1,45 @@
+# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+download-package = {version = "^1.0", optional = true, markers = "extra == \"download\""}
+
+[package.extras]
+all = ["a[download,install]"]
+download = ["download-package (>=1.0,<2.0)"]
+install = ["install-package (>=1.0,<2.0)"]
+nested = ["a[all]"]
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[package.dependencies]
+a = {version = "1.0", extras = ["download"]}
+
+[[package]]
+name = "download-package"
+version = "1.0"
+description = ""
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = []
+
+[metadata]
+lock-version = "2.1"
+python-versions = "*"
+content-hash = "123456789"