diff --git a/src/poetry/inspection/info.py b/src/poetry/inspection/info.py index 7e851223b5a..1495bc7c62c 100644 --- a/src/poetry/inspection/info.py +++ b/src/poetry/inspection/info.py @@ -69,6 +69,7 @@ def __init__(self, path: Path | str, *reasons: BaseException | str) -> None: class PackageInfo: def __init__( self, + *, name: str | None = None, version: str | None = None, summary: str | None = None, @@ -76,6 +77,7 @@ def __init__( requires_dist: list[str] | None = None, requires_python: str | None = None, files: list[dict[str, str]] | None = None, + yanked: str | bool = False, cache_version: str | None = None, ) -> None: self.name = name @@ -85,6 +87,7 @@ def __init__( self.requires_dist = requires_dist self.requires_python = requires_python self.files = files or [] + self.yanked = yanked self._cache_version = cache_version self._source_type: str | None = None self._source_url: str | None = None @@ -117,6 +120,7 @@ def asdict(self) -> dict[str, Any]: "requires_dist": self.requires_dist, "requires_python": self.requires_python, "files": self.files, + "yanked": self.yanked, "_cache_version": self._cache_version, } @@ -163,6 +167,7 @@ def to_package( source_type=self._source_type, source_url=self._source_url, source_reference=self._source_reference, + yanked=self.yanked, ) if self.summary is not None: package.description = self.summary @@ -450,6 +455,7 @@ def from_package(cls, package: Package) -> PackageInfo: requires_dist=list(requires), requires_python=package.python_versions, files=package.files, + yanked=package.yanked_reason if package.yanked else False, ) @staticmethod diff --git a/src/poetry/installation/chooser.py b/src/poetry/installation/chooser.py index 600a46306ce..53e746fdfd2 100644 --- a/src/poetry/installation/chooser.py +++ b/src/poetry/installation/chooser.py @@ -198,8 +198,7 @@ def _sort_key( has_allowed_hash = int(self._is_link_hash_allowed_for_package(link, package)) - # TODO: Proper yank value - yank_value = 0 + yank_value = int(not link.yanked) return ( has_allowed_hash, diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 4990ea26e4f..e3b56cf2a13 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -78,6 +78,7 @@ def __init__( self._executed = {"install": 0, "update": 0, "uninstall": 0} self._skipped = {"install": 0, "update": 0, "uninstall": 0} self._sections: dict[int, SectionOutput] = {} + self._yanked_warnings: list[str] = [] self._lock = threading.Lock() self._shutdown = False self._hashes: dict[str, str] = {} @@ -140,6 +141,7 @@ def execute(self, operations: list[Operation]) -> int: # We group operations by priority groups = itertools.groupby(operations, key=lambda o: -o.priority) self._sections = {} + self._yanked_warnings = [] for _, group in groups: tasks = [] serial_operations = [] @@ -179,6 +181,9 @@ def execute(self, operations: list[Operation]) -> int: break + for warning in self._yanked_warnings: + self._io.write_error_line(f"Warning: {warning}") + return 1 if self._shutdown else 0 @staticmethod @@ -611,6 +616,18 @@ def _install_git(self, operation: Install | Update) -> int: def _download(self, operation: Install | Update) -> Path: link = self._chooser.choose_for(operation.package) + if link.yanked: + # Store yanked warnings in a list and print after installing, so they can't + # be overlooked. Further, printing them in the concerning section would have + # the risk of overwriting the warning, so it is only briefly visible. + message = ( + f"The file chosen for install of {operation.package.pretty_name} " + f"{operation.package.pretty_version} ({link.show_url}) is yanked." + ) + if link.yanked_reason: + message += f" Reason for being yanked: {link.yanked_reason}" + self._yanked_warnings.append(message) + return self._download_link(operation, link) def _download_link(self, operation: Install | Update, link: Link) -> Path: diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index fa6bedd98e5..b1fd6d10e19 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -290,6 +290,7 @@ def search_for(self, dependency: Dependency) -> list[DependencyPackage]: packages.sort( key=lambda p: ( + not p.yanked, not p.is_prerelease() and not dependency.allows_prereleases(), p.version, ), diff --git a/src/poetry/puzzle/solver.py b/src/poetry/puzzle/solver.py index 815e5d54766..fb3c389f979 100644 --- a/src/poetry/puzzle/solver.py +++ b/src/poetry/puzzle/solver.py @@ -83,6 +83,16 @@ def solve(self, use_latest: list[str] | None = None) -> Transaction: f" {', '.join(f'({b})' for b in self._overrides)}" ) + for p in packages: + if p.yanked: + message = ( + f"The locked version {p.pretty_version} for {p.pretty_name} is a" + " yanked version." + ) + if p.yanked_reason: + message += f" Reason for being yanked: {p.yanked_reason}" + self._io.write_error_line(f"Warning: {message}") + return Transaction( self._locked_packages, list(zip(packages, depths)), diff --git a/src/poetry/repositories/cached.py b/src/poetry/repositories/cached.py index 17f18c37a46..a4b6b6ad089 100644 --- a/src/poetry/repositories/cached.py +++ b/src/poetry/repositories/cached.py @@ -21,7 +21,7 @@ class CachedRepository(Repository, ABC): - CACHE_VERSION = parse_constraint("1.0.0") + CACHE_VERSION = parse_constraint("1.1.0") def __init__( self, name: str, disable_cache: bool = False, config: Config | None = None diff --git a/src/poetry/repositories/http.py b/src/poetry/repositories/http.py index a3eef5f1643..c63f2b19281 100644 --- a/src/poetry/repositories/http.py +++ b/src/poetry/repositories/http.py @@ -210,6 +210,9 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] urls = defaultdict(list) files: list[dict[str, Any]] = [] for link in links: + if link.yanked and not data.yanked: + # drop yanked files unless the entire release is yanked + continue if link.is_wheel: urls["bdist_wheel"].append(link.url) elif link.filename.endswith( diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index 5c115c979f4..51aece929d0 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -72,7 +72,7 @@ def _find_packages( """ Find packages on the remote server. """ - versions: list[Version] + versions: list[tuple[Version, str | bool]] key: str = name if not constraint.is_any(): @@ -90,7 +90,9 @@ def _find_packages( return [] versions = [ - version for version in page.versions(name) if constraint.allows(version) + (version, page.yanked(name, version)) + for version in page.versions(name) + if constraint.allows(version) ] self._cache.store("matches").put(key, versions, 5) @@ -101,8 +103,9 @@ def _find_packages( source_type="legacy", source_reference=self.name, source_url=self._url, + yanked=yanked, ) - for version in versions + for version, yanked in versions ] def _get_release_info( @@ -113,6 +116,7 @@ def _get_release_info( raise PackageNotFound(f'No package named "{name}"') links = list(page.links_for_version(name, version)) + yanked = page.yanked(name, version) return self._links_to_data( links, @@ -124,6 +128,7 @@ def _get_release_info( requires_dist=[], requires_python=None, files=[], + yanked=yanked, cache_version=str(self.CACHE_VERSION), ), ) diff --git a/src/poetry/repositories/link_sources/base.py b/src/poetry/repositories/link_sources/base.py index 824f536c1c3..6e07de3cfdd 100644 --- a/src/poetry/repositories/link_sources/base.py +++ b/src/poetry/repositories/link_sources/base.py @@ -113,3 +113,17 @@ def clean_link(self, url: str) -> str: the link, it will be rewritten to %20 (while not over-quoting % or other characters).""" return self.CLEAN_REGEX.sub(lambda match: f"%{ord(match.group(0)):02x}", url) + + def yanked(self, name: NormalizedName, version: Version) -> str | bool: + reasons = set() + for link in self.links_for_version(name, version): + if link.yanked: + if link.yanked_reason: + reasons.add(link.yanked_reason) + else: + # release is not yanked if at least one file is not yanked + return False + # if all files are yanked (or there are no files) the release is yanked + if reasons: + return "\n".join(sorted(reasons)) + return True diff --git a/src/poetry/repositories/link_sources/html.py b/src/poetry/repositories/link_sources/html.py index 9f07740642a..d72a8e06ded 100644 --- a/src/poetry/repositories/link_sources/html.py +++ b/src/poetry/repositories/link_sources/html.py @@ -33,7 +33,13 @@ def links(self) -> Iterator[Link]: url = self.clean_link(urllib.parse.urljoin(self._url, href)) pyrequire = anchor.get("data-requires-python") pyrequire = unescape(pyrequire) if pyrequire else None - link = Link(url, requires_python=pyrequire) + yanked_value = anchor.get("data-yanked") + yanked: str | bool + if yanked_value: + yanked = unescape(yanked_value) + else: + yanked = "data-yanked" in anchor.attrib + link = Link(url, requires_python=pyrequire, yanked=yanked) if link.ext not in self.SUPPORTED_FORMATS: continue diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py index 3413aedde45..277df3cf1ae 100644 --- a/src/poetry/repositories/pypi_repository.py +++ b/src/poetry/repositories/pypi_repository.py @@ -144,7 +144,10 @@ def _find_packages( continue if constraint.allows(version): - packages.append(Package(info["info"]["name"], version)) + # PEP 592: PyPI always yanks entire releases, not individual files, + # so we just have to look for the first file + yanked = self._get_yanked(release[0]) + packages.append(Package(info["info"]["name"], version, yanked=yanked)) return packages @@ -163,7 +166,7 @@ def find_links_for_package(self, package: Package) -> list[Link]: links = [] for url in json_data["urls"]: h = f"sha256={url['digests']['sha256']}" - links.append(Link(url["url"] + "#" + h)) + links.append(Link(url["url"] + "#" + h, yanked=self._get_yanked(url))) return links @@ -188,6 +191,7 @@ def _get_release_info( requires_dist=info["requires_dist"], requires_python=info["requires_python"], files=info.get("files", []), + yanked=self._get_yanked(info), cache_version=str(self.CACHE_VERSION), ) @@ -254,3 +258,9 @@ def _get(self, endpoint: str) -> dict[str, Any] | None: json: dict[str, Any] = json_response.json() return json + + @staticmethod + def _get_yanked(json_data: dict[str, Any]) -> str | bool: + if json_data.get("yanked", False): + return json_data.get("yanked_reason") or True # noqa: SIM222 + return False diff --git a/src/poetry/repositories/repository.py b/src/poetry/repositories/repository.py index b5933a23d66..2d823bf7276 100644 --- a/src/poetry/repositories/repository.py +++ b/src/poetry/repositories/repository.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from poetry.core.semver.helpers import parse_constraint +from poetry.core.semver.version import Version from poetry.core.semver.version_constraint import VersionConstraint from poetry.core.semver.version_range import VersionRange @@ -16,7 +17,6 @@ from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package from poetry.core.packages.utils.link import Link - from poetry.core.semver.version import Version class Repository: @@ -43,6 +43,11 @@ def find_packages(self, dependency: Dependency) -> list[Package]: ignored_pre_release_packages = [] for package in self._find_packages(dependency.name, constraint): + if package.yanked and not isinstance(constraint, Version): + # PEP 592: yanked files are always ignored, unless they are the only + # file that matches a version specifier that "pins" to an exact + # version + continue if ( package.is_prerelease() and not allow_prereleases diff --git a/tests/console/commands/test_add.py b/tests/console/commands/test_add.py index 27f803d3915..72dc8836825 100644 --- a/tests/console/commands/test_add.py +++ b/tests/console/commands/test_add.py @@ -823,7 +823,9 @@ def test_add_constraint_with_source( ): repo = LegacyRepository(name="my-index", url="https://my-index.fake") repo.add_package(get_package("cachy", "0.2.0")) - repo._cache.store("matches").put("cachy:0.2.0", [Version.parse("0.2.0")], 5) + repo._cache.store("matches").put( + "cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5 + ) poetry.pool.add_repository(repo) @@ -1810,7 +1812,9 @@ def test_add_constraint_with_source_old_installer( ): repo = LegacyRepository(name="my-index", url="https://my-index.fake") repo.add_package(get_package("cachy", "0.2.0")) - repo._cache.store("matches").put("cachy:0.2.0", [Version.parse("0.2.0")], 5) + repo._cache.store("matches").put( + "cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5 + ) poetry.pool.add_repository(repo) diff --git a/tests/helpers.py b/tests/helpers.py index b9cd9af5392..05c0ea7bdd9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -45,8 +45,10 @@ MOCK_DEFAULT_GIT_REVISION = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" -def get_package(name: str, version: str | Version) -> Package: - return Package(name, version) +def get_package( + name: str, version: str | Version, yanked: str | bool = False +) -> Package: + return Package(name, version, yanked=yanked) def get_dependency( diff --git a/tests/installation/test_chooser.py b/tests/installation/test_chooser.py index 51c8b66c2ab..b4a8d36bc26 100644 --- a/tests/installation/test_chooser.py +++ b/tests/installation/test_chooser.py @@ -90,6 +90,26 @@ def callback( ) +@pytest.fixture() +def mock_legacy_partial_yank(http: type[httpretty.httpretty]) -> None: + def callback( + request: HTTPrettyRequest, uri: str, headers: dict[str, Any] + ) -> list[int | dict[str, Any] | str]: + parts = uri.rsplit("/") + name = parts[-2] + + fixture = LEGACY_FIXTURES / (name + "_partial_yank" + ".html") + + with fixture.open(encoding="utf-8") as f: + return [200, headers, f.read()] + + http.register_uri( + http.GET, + re.compile("^https://foo2.bar/simple/(.+?)$"), + body=callback, + ) + + @pytest.fixture() def pool() -> Pool: pool = Pool() @@ -98,6 +118,9 @@ def pool() -> Pool: pool.add_repository( LegacyRepository("foo", "https://foo.bar/simple/", disable_cache=True) ) + pool.add_repository( + LegacyRepository("foo2", "https://foo2.bar/simple/", disable_cache=True) + ) return pool @@ -266,6 +289,83 @@ def test_chooser_chooses_distributions_that_match_the_package_hashes( assert link.filename == "isort-4.3.4.tar.gz" +@pytest.mark.parametrize("source_type", ["", "legacy"]) +def test_chooser_chooses_yanked_if_no_others( + env: MockEnv, + mock_pypi: None, + mock_legacy: None, + source_type: str, + pool: Pool, +) -> None: + chooser = Chooser(pool, env) + + package = Package("black", "21.11b0") + files = [ + { + "filename": "black-21.11b0-py3-none-any.whl", + "hash": "sha256:0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117", # noqa: E501 + } + ] + if source_type == "legacy": + package = Package( + package.name, + package.version.text, + source_type="legacy", + source_reference="foo", + source_url="https://foo.bar/simple/", + ) + + package.files = files + + link = chooser.choose_for(package) + + assert link.filename == "black-21.11b0-py3-none-any.whl" + assert link.yanked + + +def test_chooser_does_not_choose_yanked_if_others( + mock_legacy: None, + mock_legacy_partial_yank: None, + pool: Pool, +) -> None: + chooser = Chooser(pool, MockEnv(supported_tags=[Tag("py2", "none", "any")])) + + package = Package("futures", "3.2.0") + files = [ + { + "filename": "futures-3.2.0-py2-none-any.whl", + "hash": "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1", # noqa: E501 + }, + { + "filename": "futures-3.2.0.tar.gz", + "hash": "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", # noqa: E501 + }, + ] + package = Package( + package.name, + package.version.text, + source_type="legacy", + source_reference="foo", + source_url="https://foo.bar/simple/", + ) + package_partial_yank = Package( + package.name, + package.version.text, + source_type="legacy", + source_reference="foo2", + source_url="https://foo2.bar/simple/", + ) + + package.files = files + package_partial_yank.files = files + + link = chooser.choose_for(package) + link_partial_yank = chooser.choose_for(package_partial_yank) + + assert link.filename == "futures-3.2.0-py2-none-any.whl" + assert link_partial_yank.filename == "futures-3.2.0.tar.gz" + + @pytest.mark.parametrize("source_type", ["", "legacy"]) def test_chooser_throws_an_error_if_package_hashes_do_not_match( env: MockEnv, diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 7f60905a33c..edfb6c27f98 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -32,6 +32,7 @@ from pytest_mock import MockerFixture from poetry.config.config import Config + from poetry.installation.operations.operation import Operation from poetry.utils.env import VirtualEnv from tests.types import FixtureDirGetter @@ -177,6 +178,61 @@ def test_execute_executes_a_batch_of_operations( assert pip_install.call_args.kwargs.get("editable", False) +@pytest.mark.parametrize( + "operations, has_warning", + [ + ( + [Install(Package("black", "21.11b0")), Install(Package("pytest", "3.5.2"))], + True, + ), + ( + [ + Uninstall(Package("black", "21.11b0")), + Uninstall(Package("pytest", "3.5.2")), + ], + False, + ), + ( + [ + Update(Package("black", "19.10b0"), Package("black", "21.11b0")), + Update(Package("pytest", "3.5.1"), Package("pytest", "3.5.2")), + ], + True, + ), + ], +) +def test_execute_prints_warning_for_yanked_package( + config: Config, + pool: Pool, + io: BufferedIO, + tmp_dir: str, + mock_file_downloads: None, + env: MockEnv, + operations: list[Operation], + has_warning: bool, +): + config.merge({"cache-dir": tmp_dir}) + + executor = Executor(env, pool, config, io) + + return_code = executor.execute(operations) + + expected = ( + "Warning: The file chosen for install of black 21.11b0 " + "(black-21.11b0-py3-none-any.whl) is yanked. Reason for being yanked: " + "Broken regex dependency. Use 21.11b1 instead." + ) + error = io.fetch_error() + assert return_code == 0 + assert "pytest" not in error + if has_warning: + assert expected in error + assert error.count("is yanked") == 1 + else: + assert expected not in error + assert error.count("yanked") == 0 + + def test_execute_shows_skipped_operations_if_verbose( config: Config, pool: Pool, io: BufferedIO, config_cache_dir: Path, env: MockEnv ): diff --git a/tests/mixology/helpers.py b/tests/mixology/helpers.py index 0120bdb72ff..225807e2208 100644 --- a/tests/mixology/helpers.py +++ b/tests/mixology/helpers.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from poetry.core.packages.project_package import ProjectPackage + from poetry.mixology import SolverResult from poetry.repositories import Repository from tests.mixology.version_solver.conftest import Provider @@ -23,8 +24,9 @@ def add_to_repo( version: str, deps: dict[str, str] | None = None, python: str | None = None, + yanked: bool = False, ) -> None: - package = Package(name, version) + package = Package(name, version, yanked=yanked) if python: package.python_versions = python @@ -43,7 +45,7 @@ def check_solver_result( tries: int | None = None, locked: dict[str, Package] | None = None, use_latest: list[str] | None = None, -) -> None: +) -> SolverResult | None: if locked is not None: locked = { k: [DependencyPackage(l.to_dependency(), l)] for k, l in locked.items() @@ -59,20 +61,22 @@ def check_solver_result( if tries is not None: assert solver.solution.attempted_solutions == tries - return + return None raise except AssertionError as e: if error: assert str(e) == error - return + return None raise packages = {} for package in solution.packages: packages[package.name] = str(package.version) - assert result == packages + assert packages == result if tries is not None: assert solution.attempted_solutions == tries + + return solution diff --git a/tests/mixology/version_solver/test_basic_graph.py b/tests/mixology/version_solver/test_basic_graph.py index f8758f00c3e..210abc371e4 100644 --- a/tests/mixology/version_solver/test_basic_graph.py +++ b/tests/mixology/version_solver/test_basic_graph.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +import pytest + from poetry.factory import Factory from tests.mixology.helpers import add_to_repo from tests.mixology.helpers import check_solver_result @@ -87,3 +89,44 @@ def test_circular_dependency( add_to_repo(repo, "bar", "1.0.0", deps={"foo": "1.0.0"}) check_solver_result(root, provider, {"foo": "1.0.0", "bar": "1.0.0"}) + + +@pytest.mark.parametrize( + "constraint, versions, yanked_versions, expected", + [ + (">=1", ["1", "2"], [], "2"), + (">=1", ["1", "2"], ["2"], "1"), + (">=1", ["1", "2", "3"], ["2"], "3"), + (">=1", ["1", "2", "3"], ["2", "3"], "1"), + (">1", ["1", "2"], ["2"], "error"), + (">1", ["2"], ["2"], "error"), + (">=2", ["2"], ["2"], "error"), + ("==2", ["2"], ["2"], "2"), + ("==2", ["2", "2+local"], [], "2+local"), + ("==2", ["2", "2+local"], ["2+local"], "2"), + ], +) +def test_yanked_release( + root: ProjectPackage, + provider: Provider, + repo: Repository, + constraint: str, + versions: list[str], + yanked_versions: list[str], + expected: str, +) -> None: + root.add_dependency(Factory.create_dependency("foo", constraint)) + + for version in versions: + add_to_repo(repo, "foo", version, yanked=version in yanked_versions) + + if expected == "error": + result = None + error = ( + f"Because myapp depends on foo ({constraint}) which doesn't match any " + "versions, version solving failed." + ) + else: + result = {"foo": expected} + error = None + check_solver_result(root, provider, result, error) diff --git a/tests/mixology/version_solver/test_with_lock.py b/tests/mixology/version_solver/test_with_lock.py index e9d07d21d7b..fa0e536c3ba 100644 --- a/tests/mixology/version_solver/test_with_lock.py +++ b/tests/mixology/version_solver/test_with_lock.py @@ -169,3 +169,31 @@ def test_with_compatible_locked_dependencies_with_extras( "baz": get_package("baz", "1.0.0"), }, ) + + +def test_with_yanked_package_in_lock( + root: ProjectPackage, provider: Provider, repo: Repository +): + root.add_dependency(Factory.create_dependency("foo", "*")) + + add_to_repo(repo, "foo", "1") + add_to_repo(repo, "foo", "2", yanked=True) + + # yanked version is kept in lock file + locked_foo = get_package("foo", "2") + assert not locked_foo.yanked + result = check_solver_result( + root, + provider, + result={"foo": "2"}, + locked={"foo": locked_foo}, + ) + foo = result.packages[0] + assert foo.yanked + + # without considering the lock file, the other version is chosen + check_solver_result( + root, + provider, + result={"foo": "1"}, + ) diff --git a/tests/puzzle/test_solver.py b/tests/puzzle/test_solver.py index 7af0fcaf540..12a8fdb2062 100644 --- a/tests/puzzle/test_solver.py +++ b/tests/puzzle/test_solver.py @@ -6,6 +6,7 @@ import pytest +from cleo.io.buffered_io import BufferedIO from cleo.io.null_io import NullIO from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package @@ -3680,3 +3681,44 @@ def test_update_with_prerelease_and_no_solution( with pytest.raises(SolverProblemError): solver.solve() + + +def test_solver_yanked_warning( + package: ProjectPackage, + installed: InstalledRepository, + locked: Repository, + pool: Pool, + repo: Repository, +) -> None: + package.add_dependency(Factory.create_dependency("foo", "==1")) + package.add_dependency(Factory.create_dependency("bar", "==2")) + package.add_dependency(Factory.create_dependency("baz", "==3")) + foo = get_package("foo", "1", yanked=False) + bar = get_package("bar", "2", yanked=True) + baz = get_package("baz", "3", yanked="just wrong") + repo.add_package(foo) + repo.add_package(bar) + repo.add_package(baz) + + io = BufferedIO(decorated=False) + solver = Solver(package, pool, installed.packages, locked.packages, io) + transaction = solver.solve() + + check_solver_result( + transaction, + [ + {"job": "install", "package": bar}, + {"job": "install", "package": baz}, + {"job": "install", "package": foo}, + ], + ) + error = io.fetch_error() + assert "foo" not in error + assert "The locked version 2 for bar is a yanked version." in error + assert ( + "The locked version 3 for baz is a yanked version. Reason for being yanked:" + " just wrong" + in error + ) + assert error.count("is a yanked version") == 2 + assert error.count("Reason for being yanked") == 1 diff --git a/tests/repositories/fixtures/legacy/black.html b/tests/repositories/fixtures/legacy/black.html index 9d3fa08c633..ea050662b83 100644 --- a/tests/repositories/fixtures/legacy/black.html +++ b/tests/repositories/fixtures/legacy/black.html @@ -4,7 +4,8 @@ Links for black

Links for black

- black-19.10b0.tar.gz + black-19.10b0-py36-none-any.whl + black-21.11b0-py3-none-any.whl diff --git a/tests/repositories/fixtures/legacy/futures_partial_yank.html b/tests/repositories/fixtures/legacy/futures_partial_yank.html new file mode 100644 index 00000000000..41fa4ac3d38 --- /dev/null +++ b/tests/repositories/fixtures/legacy/futures_partial_yank.html @@ -0,0 +1,12 @@ + + + + Links for futures + + +

Links for futures

+ futures-3.2.0-py2-none-any.whl
+ futures-3.2.0.tar.gz
+ + + diff --git a/tests/repositories/fixtures/pypi.org/dists/black-21.11b0-py3-none-any.whl b/tests/repositories/fixtures/pypi.org/dists/black-21.11b0-py3-none-any.whl new file mode 100644 index 00000000000..f0e3956e20f Binary files /dev/null and b/tests/repositories/fixtures/pypi.org/dists/black-21.11b0-py3-none-any.whl differ diff --git a/tests/repositories/fixtures/pypi.org/json/black.json b/tests/repositories/fixtures/pypi.org/json/black.json index 17c61d29de4..38db6d8c89c 100644 --- a/tests/repositories/fixtures/pypi.org/json/black.json +++ b/tests/repositories/fixtures/pypi.org/json/black.json @@ -99,6 +99,48 @@ "yanked": false, "yanked_reason": null } + ], + "21.11b0": [ + { + "comment_text": "", + "digests": { + "md5": "945da11b34c11738560fc6698cffa425", + "sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117" + }, + "downloads": -1, + "filename": "black-21.11b0-py3-none-any.whl", + "has_sig": false, + "md5_digest": "945da11b34c11738560fc6698cffa425", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": ">=3.6.2", + "size": 155131, + "upload_time": "2021-11-17T02:32:14", + "upload_time_iso_8601": "2021-11-17T02:32:14.551680Z", + "url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + }, + { + "comment_text": "", + "digests": { + "md5": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa" + }, + "downloads": -1, + "filename": "black-21.11b0.tar.gz", + "has_sig": false, + "md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=3.6.2", + "size": 593164, + "upload_time": "2021-11-17T02:32:16", + "upload_time_iso_8601": "2021-11-17T02:32:16.396821Z", + "url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + } ] }, "urls": [ diff --git a/tests/repositories/fixtures/pypi.org/json/black/21.11b0.json b/tests/repositories/fixtures/pypi.org/json/black/21.11b0.json new file mode 100644 index 00000000000..f816ef3b96b --- /dev/null +++ b/tests/repositories/fixtures/pypi.org/json/black/21.11b0.json @@ -0,0 +1,155 @@ +{ + "info": { + "author": "Łukasz Langa", + "author_email": "lukasz@langa.pl", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance" + ], + "description": "", + "description_content_type": "text/markdown", + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://github.com/psf/black", + "keywords": "automation formatter yapf autopep8 pyfmt gofmt rustfmt", + "license": "MIT", + "maintainer": "", + "maintainer_email": "", + "name": "black", + "package_url": "https://pypi.org/project/black/", + "platform": "", + "project_url": "https://pypi.org/project/black/", + "project_urls": { + "Changelog": "https://github.com/psf/black/blob/main/CHANGES.md", + "Homepage": "https://github.com/psf/black" + }, + "release_url": "https://pypi.org/project/black/21.11b0/", + "requires_dist": [ + "click (>=7.1.2)", + "platformdirs (>=2)", + "tomli (<2.0.0,>=0.2.6)", + "regex (>=2020.1.8)", + "pathspec (<1,>=0.9.0)", + "typing-extensions (>=3.10.0.0)", + "mypy-extensions (>=0.4.3)", + "dataclasses (>=0.6) ; python_version < \"3.7\"", + "typed-ast (>=1.4.2) ; python_version < \"3.8\" and implementation_name == \"cpython\"", + "typing-extensions (!=3.10.0.1) ; python_version >= \"3.10\"", + "colorama (>=0.4.3) ; extra == 'colorama'", + "aiohttp (>=3.7.4) ; extra == 'd'", + "ipython (>=7.8.0) ; extra == 'jupyter'", + "tokenize-rt (>=3.2.0) ; extra == 'jupyter'", + "typed-ast (>=1.4.3) ; extra == 'python2'", + "uvloop (>=0.15.2) ; extra == 'uvloop'" + ], + "requires_python": ">=3.6.2", + "summary": "The uncompromising code formatter.", + "version": "21.11b0", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + }, + "last_serial": 13326107, + "releases": { + "21.11b0": [ + { + "comment_text": "", + "digests": { + "md5": "945da11b34c11738560fc6698cffa425", + "sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117" + }, + "downloads": -1, + "filename": "black-21.11b0-py3-none-any.whl", + "has_sig": false, + "md5_digest": "945da11b34c11738560fc6698cffa425", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": ">=3.6.2", + "size": 155131, + "upload_time": "2021-11-17T02:32:14", + "upload_time_iso_8601": "2021-11-17T02:32:14.551680Z", + "url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + }, + { + "comment_text": "", + "digests": { + "md5": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa" + }, + "downloads": -1, + "filename": "black-21.11b0.tar.gz", + "has_sig": false, + "md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=3.6.2", + "size": 593164, + "upload_time": "2021-11-17T02:32:16", + "upload_time_iso_8601": "2021-11-17T02:32:16.396821Z", + "url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "945da11b34c11738560fc6698cffa425", + "sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117" + }, + "downloads": -1, + "filename": "black-21.11b0-py3-none-any.whl", + "has_sig": false, + "md5_digest": "945da11b34c11738560fc6698cffa425", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": ">=3.6.2", + "size": 155131, + "upload_time": "2021-11-17T02:32:14", + "upload_time_iso_8601": "2021-11-17T02:32:14.551680Z", + "url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + }, + { + "comment_text": "", + "digests": { + "md5": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa" + }, + "downloads": -1, + "filename": "black-21.11b0.tar.gz", + "has_sig": false, + "md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=3.6.2", + "size": 593164, + "upload_time": "2021-11-17T02:32:16", + "upload_time_iso_8601": "2021-11-17T02:32:16.396821Z", + "url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz", + "yanked": true, + "yanked_reason": "Broken regex dependency. Use 21.11b1 instead." + } + ], + "vulnerabilities": [] +} diff --git a/tests/repositories/link_sources/test_base.py b/tests/repositories/link_sources/test_base.py index 2870499854b..c949f90a5eb 100644 --- a/tests/repositories/link_sources/test_base.py +++ b/tests/repositories/link_sources/test_base.py @@ -5,6 +5,7 @@ import pytest +from packaging.utils import canonicalize_name from poetry.core.packages.package import Package from poetry.core.packages.utils.link import Link from poetry.core.semver.version import Version @@ -87,4 +88,7 @@ def test_links_for_version( ) -> None: version = Version.parse(version_string) expected = {Link(f"{link_source.url}/{name}") for name in filenames} - assert set(link_source.links_for_version("demo", version)) == expected + assert ( + set(link_source.links_for_version(canonicalize_name("demo"), version)) + == expected + ) diff --git a/tests/repositories/link_sources/test_html.py b/tests/repositories/link_sources/test_html.py new file mode 100644 index 00000000000..35b9ebfaa88 --- /dev/null +++ b/tests/repositories/link_sources/test_html.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import pytest + +from poetry.core.packages.utils.link import Link +from poetry.core.semver.version import Version + +from poetry.repositories.link_sources.html import HTMLPage + + +DEMO_TEMPLATE = """ + + + + + Links for demo + + +

Links for demo

+ {} + + +""" + + +@pytest.mark.parametrize( + "attributes, expected_link", + [ + ("", Link("https://example.org/demo-0.1.whl")), + ( + 'data-requires-python=">=3.7"', + Link("https://example.org/demo-0.1.whl", requires_python=">=3.7"), + ), + ( + "data-yanked", + Link("https://example.org/demo-0.1.whl", yanked=True), + ), + ( + 'data-yanked=""', + Link("https://example.org/demo-0.1.whl", yanked=True), + ), + ( + 'data-yanked="<reason>"', + Link("https://example.org/demo-0.1.whl", yanked=""), + ), + ( + 'data-requires-python=">=3.7" data-yanked', + Link( + "https://example.org/demo-0.1.whl", requires_python=">=3.7", yanked=True + ), + ), + ], +) +def test_link_attributes(attributes: str, expected_link: Link) -> None: + anchor = ( + f'demo-0.1.whl
' + ) + content = DEMO_TEMPLATE.format(anchor) + page = HTMLPage("https://example.org", content) + + assert len(list(page.links)) == 1 + link = list(page.links)[0] + assert link.url == expected_link.url + assert link.requires_python == expected_link.requires_python + assert link.yanked == expected_link.yanked + assert link.yanked_reason == expected_link.yanked_reason + + +@pytest.mark.parametrize( + "yanked_attrs, expected", + [ + (("", ""), False), + (("data-yanked", ""), False), + (("", "data-yanked"), False), + (("data-yanked", "data-yanked"), True), + (("data-yanked='reason'", "data-yanked"), "reason"), + (("data-yanked", "data-yanked='reason'"), "reason"), + (("data-yanked='reason'", "data-yanked=''"), "reason"), + (("data-yanked=''", "data-yanked='reason'"), "reason"), + (("data-yanked='reason'", "data-yanked='reason'"), "reason"), + (("data-yanked='reason 1'", "data-yanked='reason 2'"), "reason 1\nreason 2"), + ], +) +def test_yanked(yanked_attrs: tuple[str, str], expected: bool | str) -> None: + anchors = ( + f'' + "demo-0.1.tar.gz" + f'demo-0.1.whl' + ) + content = DEMO_TEMPLATE.format(anchors) + page = HTMLPage("https://example.org", content) + + assert page.yanked("demo", Version.parse("0.1")) == expected diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index a02cff0b127..be1d646b50d 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -234,6 +234,24 @@ def test_find_packages_only_prereleases_empty_when_not_any() -> None: assert len(packages) == 0 +@pytest.mark.parametrize( + ["constraint", "expected"], + [ + # yanked 21.11b0 is ignored except for pinned version + ("*", ["19.10b0"]), + (">=19.0a0", ["19.10b0"]), + (">=20.0a0", []), + (">=21.11b0", []), + ("==21.11b0", ["21.11b0"]), + ], +) +def test_find_packages_yanked(constraint: str, expected: list[str]) -> None: + repo = MockRepository() + packages = repo.find_packages(Factory.create_dependency("black", constraint)) + + assert [str(p.version) for p in packages] == expected + + def test_get_package_information_chooses_correct_distribution() -> None: repo = MockRepository() @@ -402,6 +420,62 @@ def test_get_package_retrieves_packages_with_no_hashes() -> None: ] == package.files +@pytest.mark.parametrize( + "package_name, version, yanked, yanked_reason", + [ + ("black", "19.10b0", False, ""), + ("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."), + ], +) +def test_package_yanked( + package_name: str, version: str, yanked: bool, yanked_reason: str +) -> None: + repo = MockRepository() + + package = repo.package(canonicalize_name(package_name), Version.parse(version)) + + assert package.name == package_name + assert str(package.version) == version + assert package.yanked is yanked + assert package.yanked_reason == yanked_reason + + +def test_package_partial_yank(): + class SpecialMockRepository(MockRepository): + def _get_page(self, endpoint: str) -> SimpleRepositoryPage | None: + return super()._get_page(f"/{endpoint.strip('/')}_partial_yank/") + + repo = MockRepository() + package = repo.package(canonicalize_name("futures"), Version.parse("3.2.0")) + assert len(package.files) == 2 + + repo = SpecialMockRepository() + package = repo.package(canonicalize_name("futures"), Version.parse("3.2.0")) + assert len(package.files) == 1 + assert package.files[0]["file"].endswith(".tar.gz") + + +@pytest.mark.parametrize( + "package_name, version, yanked, yanked_reason", + [ + ("black", "19.10b0", False, ""), + ("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."), + ], +) +def test_find_links_for_package_yanked( + package_name: str, version: str, yanked: bool, yanked_reason: str +) -> None: + repo = MockRepository() + + package = repo.package(canonicalize_name(package_name), Version.parse(version)) + links = repo.find_links_for_package(package) + + assert len(links) == 1 + for link in links: + assert link.yanked == yanked + assert link.yanked_reason == yanked_reason + + class MockHttpRepository(LegacyRepository): def __init__( self, endpoint_responses: dict, http: type[httpretty.httpretty] diff --git a/tests/repositories/test_pypi_repository.py b/tests/repositories/test_pypi_repository.py index 9f7371f34c7..2e398de7272 100644 --- a/tests/repositories/test_pypi_repository.py +++ b/tests/repositories/test_pypi_repository.py @@ -96,6 +96,24 @@ def test_find_packages_only_prereleases(constraint: str, count: int) -> None: assert len(packages) == count +@pytest.mark.parametrize( + ["constraint", "expected"], + [ + # yanked 21.11b0 is ignored except for pinned version + ("*", ["19.10b0"]), + (">=19.0a0", ["19.10b0"]), + (">=20.0a0", []), + (">=21.11b0", []), + ("==21.11b0", ["21.11b0"]), + ], +) +def test_find_packages_yanked(constraint: str, expected: list[str]) -> None: + repo = MockRepository() + packages = repo.find_packages(Factory.create_dependency("black", constraint)) + + assert [str(p.version) for p in packages] == expected + + def test_package() -> None: repo = MockRepository() @@ -128,6 +146,47 @@ def test_package() -> None: ) +@pytest.mark.parametrize( + "package_name, version, yanked, yanked_reason", + [ + ("black", "19.10b0", False, ""), + ("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."), + ], +) +def test_package_yanked( + package_name: str, version: str, yanked: bool, yanked_reason: str +) -> None: + repo = MockRepository() + + package = repo.package(package_name, version) + + assert package.name == package_name + assert str(package.version) == version + assert package.yanked is yanked + assert package.yanked_reason == yanked_reason + + +@pytest.mark.parametrize( + "package_name, version, yanked, yanked_reason", + [ + ("black", "19.10b0", False, ""), + ("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."), + ], +) +def test_find_links_for_package_yanked( + package_name: str, version: str, yanked: bool, yanked_reason: str +) -> None: + repo = MockRepository() + + package = repo.package(package_name, version) + links = repo.find_links_for_package(package) + + assert len(links) == 2 + for link in links: + assert link.yanked == yanked + assert link.yanked_reason == yanked_reason + + def test_fallback_on_downloading_packages() -> None: repo = MockRepository(fallback=True) diff --git a/tests/repositories/test_repository.py b/tests/repositories/test_repository.py new file mode 100644 index 00000000000..4073d92e53a --- /dev/null +++ b/tests/repositories/test_repository.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import pytest + +from packaging.utils import canonicalize_name +from poetry.core.packages.package import Package +from poetry.core.semver.version import Version + +from poetry.factory import Factory +from poetry.repositories import Repository + + +@pytest.fixture(scope="module") +def black_repository() -> Repository: + repo = Repository("repo") + repo.add_package(Package("black", "19.10b0")) + repo.add_package(Package("black", "21.11b0", yanked="reason")) + return repo + + +@pytest.mark.parametrize( + ["constraint", "expected"], + [ + # yanked 21.11b0 is ignored except for pinned version + ("*", ["19.10b0"]), + (">=19.0a0", ["19.10b0"]), + (">=20.0a0", []), + (">=21.11b0", []), + ("==21.11b0", ["21.11b0"]), + ], +) +def test_find_packages_yanked( + black_repository: Repository, constraint: str, expected: list[str] +) -> None: + packages = black_repository.find_packages( + Factory.create_dependency("black", constraint) + ) + + assert [str(p.version) for p in packages] == expected + + +@pytest.mark.parametrize( + "package_name, version, yanked, yanked_reason", + [ + ("black", "19.10b0", False, ""), + ("black", "21.11b0", True, "reason"), + ], +) +def test_package_yanked( + black_repository: Repository, + package_name: str, + version: str, + yanked: bool, + yanked_reason: str, +) -> None: + package = black_repository.package( + canonicalize_name(package_name), Version.parse(version) + ) + + assert package.name == package_name + assert str(package.version) == version + assert package.yanked is yanked + assert package.yanked_reason == yanked_reason