[WIP] support poetry sources for private pypi repositories

README.md

@@ -156,7 +156,33 @@ The default category is `main`.
 [environment.yml][envyaml], using a vendored copy of [Poetry's][poetry] dependency solver.
 
 ### private pip repositories
-Right now `conda-lock` only supports [legacy](https://warehouse.pypa.io/api-reference/legacy.html) pypi repos with basic auth. Most self-hosted repositories like Nexus, Artifactory etc. use this. To use this feature, add your private repo into Poetry's config _including_ the basic auth in the url:
+
+`conda-lock` currently only supports [legacy](https://warehouse.pypa.io/api-reference/legacy.html) pypi repos with basic auth. Most self-hosted repositories like Nexus, Artifactory etc. use this.
+
+#### pyproject.toml: via poetry sources
+
+conda-lock can fetch packages from (private) python indices using [poetry sources](https://python-poetry.org/docs/repositories/#project-configuration) in the pyproject.toml:
+
+```toml
+[[tool.poetry.source]]
+name = "my_pypi"
+url = "https://pypi.python.org/simple"
+default = false
+secondary = false
+```
+
+You need to manually assign the source to the dependent with the `source` keyword.
+
+```toml
+[tool.poetry.dependencies]
+numpy = {version="*", source="my_pypi"}
+```
+
+Should your source require login credentials you can provide them via poetry commandline: `poetry config http-basic.<source-name> <username> <password>`.
+
+#### private pypi repos without pyproject.toml
+
+For sources other than `pyprojoect.toml` add your private repo into Poetry's config _including_ the basic auth in the url:
 
 ```bash
 poetry config repositories.foo https://username:[email protected]/simple/

conda_lock/conda_lock.py

@@ -733,6 +733,7 @@ def _solve_for_arch(
             conda_locked={dep.name: dep for dep in conda_deps.values()},
             python_version=conda_deps["python"].version,
             platform=platform,
+            pypi_channels=spec.pypi_channels,
             allow_pypi_requests=spec.allow_pypi_requests,
         )
     else:

conda_lock/conda_solver.py

@@ -143,7 +143,7 @@ def solve_conda(
     """
 
     conda_specs = [
-        _to_match_spec(dep.name, dep.version, dep.build, dep.conda_channel)
+        _to_match_spec(dep.name, dep.version, dep.build, dep.channel)
         for dep in specs.values()
         if isinstance(dep, VersionedDependency) and dep.manager == "conda"
     ]

conda_lock/pypi_solver.py

@@ -16,13 +16,15 @@
     ProjectPackage as PoetryProjectPackage,
 )
 from conda_lock._vendor.poetry.core.packages import URLDependency as PoetryURLDependency
+from conda_lock._vendor.poetry.core.toml.file import TOMLFile
 from conda_lock._vendor.poetry.factory import Factory
 from conda_lock._vendor.poetry.installation.chooser import Chooser
 from conda_lock._vendor.poetry.installation.operations.uninstall import Uninstall
 from conda_lock._vendor.poetry.puzzle import Solver as PoetrySolver
 from conda_lock._vendor.poetry.repositories.pool import Pool
 from conda_lock._vendor.poetry.repositories.pypi_repository import PyPiRepository
 from conda_lock._vendor.poetry.repositories.repository import Repository
+from conda_lock._vendor.poetry.utils.appdirs import user_config_dir
 from conda_lock._vendor.poetry.utils.env import Env
 from conda_lock.lookup import conda_name_to_pypi_name
 
@@ -146,9 +148,11 @@ def get_dependency(dep: src_parser.Dependency) -> PoetryDependency:
     # FIXME: how do deal with extras?
     extras: List[str] = []
     if isinstance(dep, src_parser.VersionedDependency):
-        return PoetryDependency(
+        dependency = PoetryDependency(
             name=dep.name, constraint=dep.version or "*", extras=dep.extras
         )
+        dependency.source_name = dep.channel  # type: ignore
+        return dependency
     elif isinstance(dep, src_parser.URLDependency):
         return PoetryURLDependency(
             name=dep.name,
@@ -178,6 +182,7 @@ def solve_pypi(
     conda_locked: Dict[str, lockfile.LockedDependency],
     python_version: str,
     platform: str,
+    pypi_channels: Optional[List[src_parser.PyPIChannel]] = None,
     allow_pypi_requests: bool = True,
     verbose: bool = False,
 ) -> Dict[str, lockfile.LockedDependency]:
@@ -213,7 +218,7 @@ def solve_pypi(
     for dep in dependencies:
         dummy_package.add_dependency(dep)
 
-    pool = _prepare_repositories_pool(allow_pypi_requests)
+    pool = _prepare_repositories_pool(allow_pypi_requests, pypi_channels)
 
     installed = Repository()
     locked = Repository()
@@ -322,7 +327,10 @@ def solve_pypi(
     return {dep.name: dep for dep in requirements}
 
 
-def _prepare_repositories_pool(allow_pypi_requests: bool) -> Pool:
+def _prepare_repositories_pool(
+    allow_pypi_requests: bool,
+    repositories: Optional[List[src_parser.PyPIChannel]] = None,
+) -> Pool:
     """
     Prepare the pool of repositories to solve pip dependencies
 
@@ -333,12 +341,34 @@ def _prepare_repositories_pool(allow_pypi_requests: bool) -> Pool:
     """
     factory = Factory()
     config = factory.create_config()
-    repos = [
-        factory.create_legacy_repository(
-            {"name": source[0], "url": source[1]["url"]}, config
-        )
-        for source in config.get("repositories", {}).items()
-    ]
+
+    # read auth from global poetry config
+    auth_config_file = TOMLFile(Path(user_config_dir("pypoetry")) / "auth.toml")
+    if auth_config_file.exists():
+        config.merge(auth_config_file.read())
+
+    if repositories is not None:
+        repo_config = {}
+        for repo in repositories:
+            repo_config[repo.name] = {"url": repo.url}
+        config.merge(dict(repositories=repo_config))
+        repos = [
+            factory.create_legacy_repository(
+                dict(
+                    name=poetry_repo.name,
+                    url=poetry_repo.url,
+                ),
+                config,
+            )
+            for poetry_repo in repositories
+        ]
+    else:
+        repos = [
+            factory.create_legacy_repository(
+                {"name": source[0], "url": source[1]["url"]}, config
+            )
+            for source in config.get("repositories", {}).items()
+        ]
     if allow_pypi_requests:
         repos.append(PyPiRepository())
     return Pool(repositories=[*repos])

conda_lock/src_parser/__init__.py

@@ -45,10 +45,20 @@ class _BaseDependency(StrictModel):
     selectors: Selectors = Selectors()
 
 
+class PyPIChannel(StrictModel):
+    class Config:
+        frozen = True
+
+    name: str
+    url: str
+    default: bool = False
+    secondary: bool = False
+
+
 class VersionedDependency(_BaseDependency):
     version: str
     build: Optional[str] = None
-    conda_channel: Optional[str] = None
+    channel: Optional[str] = None
 
 
 class URLDependency(_BaseDependency):
@@ -72,6 +82,7 @@ class LockSpecification(BaseModel):
     sources: List[pathlib.Path]
     virtual_package_repo: Optional[FakeRepoData] = None
     allow_pypi_requests: bool = True
+    pypi_channels: Optional[List[PyPIChannel]] = None
 
     def content_hash(self) -> Dict[str, str]:
         return {
@@ -135,4 +146,8 @@ def aggregate_lock_specs(
         # uniquify metadata, preserving order
         platforms=ordered_union(lock_spec.platforms or [] for lock_spec in lock_specs),
         sources=ordered_union(lock_spec.sources or [] for lock_spec in lock_specs),
+        pypi_channels=ordered_union(
+            lock_spec.pypi_channels or [] for lock_spec in lock_specs
+        )
+        or None,
     )

conda_lock/src_parser/conda_common.py

@@ -29,5 +29,5 @@ def conda_spec_to_versioned_dep(spec: str, category: str) -> VersionedDependency
         category=category,
         extras=[],
         build=ms.get("build"),
-        conda_channel=channel_str,
+        channel=channel_str,
     )

conda_lock/src_parser/pyproject_toml.py

@@ -31,6 +31,7 @@
 from conda_lock.src_parser import (
     Dependency,
     LockSpecification,
+    PyPIChannel,
     URLDependency,
     VersionedDependency,
 )
@@ -120,6 +121,11 @@ def parse_poetry_pyproject_toml(
         group_key = tuple(["group", group_name, "dependencies"])
         categories[group_key] = group_name
 
+    poetry_repositories = []
+    for repository in get_in(["tool", "poetry", "source"], contents, {}):
+        source = PyPIChannel(**repository)
+        poetry_repositories.append(source)
+
     for section, default_category in categories.items():
         for depname, depattrs in get_in(
             ["tool", "poetry", *section], contents, {}
@@ -129,18 +135,19 @@ def parse_poetry_pyproject_toml(
             manager: Literal["conda", "pip"] = "conda"
             url = None
             extras = []
+            poetry_source = None
             if isinstance(depattrs, collections.abc.Mapping):
                 poetry_version_spec = depattrs.get("version", None)
                 url = depattrs.get("url", None)
                 optional = depattrs.get("optional", False)
                 extras = depattrs.get("extras", [])
                 # If a dependency is explicitly marked as sourced from pypi,
                 # or is a URL dependency, delegate to the pip section
-                if (
-                    depattrs.get("source", None) == "pypi"
-                    or poetry_version_spec is None
-                ):
+
+                poetry_source = depattrs.get("source", None)
+                if poetry_source is not None or poetry_version_spec is None:
                     manager = "pip"
+                    poetry_source = poetry_source
                 # TODO: support additional features such as markers for things like sys_platform, platform_system
             elif isinstance(depattrs, str):
                 poetry_version_spec = depattrs
@@ -180,14 +187,20 @@ def parse_poetry_pyproject_toml(
                         optional=optional,
                         category=category,
                         extras=extras,
+                        channel=poetry_source,
                     )
                 )
 
-    return specification_with_dependencies(path, contents, dependencies)
+    return specification_with_dependencies(
+        path, contents, dependencies, poetry_repositories
+    )
 
 
 def specification_with_dependencies(
-    path: pathlib.Path, toml_contents: Mapping[str, Any], dependencies: List[Dependency]
+    path: pathlib.Path,
+    toml_contents: Mapping[str, Any],
+    dependencies: List[Dependency],
+    pypi_channels: Optional[List[PyPIChannel]] = None,
 ) -> LockSpecification:
     force_pypi = set()
     for depname, depattrs in get_in(
@@ -206,7 +219,7 @@ def specification_with_dependencies(
                 )
             )
         elif isinstance(depattrs, collections.abc.Mapping):
-            if depattrs.get("source", None) == "pypi":
+            if depattrs.get("source") is not None:
                 force_pypi.add(depname)
         else:
             raise TypeError(f"Unsupported type for dependency: {depname}: {depattrs:r}")
@@ -221,6 +234,7 @@ def specification_with_dependencies(
         channels=get_in(["tool", "conda-lock", "channels"], toml_contents, []),
         platforms=get_in(["tool", "conda-lock", "platforms"], toml_contents, []),
         sources=[path],
+        pypi_channels=pypi_channels,
         allow_pypi_requests=get_in(
             ["tool", "conda-lock", "allow-pypi-requests"], toml_contents, True
         ),