Skip to content

Commit

Permalink
Reject VCS requirements in locks. (#1563)
Browse files Browse the repository at this point in the history 
The current pex lock infrastructure cannot handle VCS requirements in
the same way it can't handle local project requirements. Unlike local
project requirements though, the failure to handle these requirements
was uncontrolled. Add explicit parsing of VCS requirements to allow them
to be singled out and rejected along side local project requirements.

This addresses the UX for the failure noted in #1556 but may not be the
final word there. If there is a desire to actually handle VCS
requirements in locks, the `VCSRequirement` parsing added here can be
expanded to extract the needed extra data (commit id certainly) to build
that support.

Closes #1562
  • Loading branch information
@jsirois
jsirois authored Jan 11, 2022
1 parent 4215e8d commit 851666c
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 43 deletions.
31 changes: 20 additions & 11 deletions pex/cli/commands/lockfile/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
from pex import resolver
from pex.cli.commands.lockfile.lockfile import Lockfile as Lockfile # For re-export.
from pex.commands.command import Error
from pex.common import safe_open
from pex.requirements import LocalProjectRequirement
from pex.common import pluralize, safe_open
from pex.requirements import LocalProjectRequirement, VCSRequirement
from pex.resolve.locked_resolve import LockConfiguration
from pex.resolve.requirement_configuration import RequirementConfiguration
from pex.resolve.resolver_configuration import PipConfiguration
Expand Down Expand Up @@ -86,20 +86,29 @@ def create(

network_configuration = pip_configuration.network_configuration
requirements = [] # type: List[Requirement]
local_projects = [] # type: List[LocalProjectRequirement]
projects = [] # type: List[str]
for parsed_requirement in requirement_configuration.parse_requirements(network_configuration):
if isinstance(parsed_requirement, LocalProjectRequirement):
local_projects.append(parsed_requirement)
projects.append("local project at {path}".format(path=parsed_requirement.path))
elif isinstance(parsed_requirement, VCSRequirement):
projects.append(
"{vcs} project {project_name} at {url}".format(
vcs=parsed_requirement.vcs,
project_name=parsed_requirement.requirement.project_name,
url=parsed_requirement.url,
)
)
else:
requirements.append(parsed_requirement.requirement)
if local_projects:
if projects:
return Error(
"Cannot create a lock for local project requirements. Given {count}:\n"
"{projects}".format(
count=len(local_projects),
projects="\n".join(
"{index}.) {project}".format(index=index, project=project.path)
for index, project in enumerate(local_projects, start=1)
"Cannot create a lock for project requirements built from local or version "
"controlled sources. Given {count} such {projects}:\n{project_descriptions}".format(
count=len(projects),
projects=pluralize(projects, "project"),
project_descriptions="\n".join(
"{index}.) {project}".format(index=index, project=project)
for index, project in enumerate(projects, start=1)
),
)
)
Expand Down
97 changes: 75 additions & 22 deletions pex/requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@
from pex import attrs, dist_metadata
from pex.compatibility import urlparse
from pex.dist_metadata import MetadataError, ProjectNameAndVersion
from pex.enum import Enum
from pex.fetcher import URLFetcher
from pex.third_party.packaging.markers import Marker
from pex.third_party.packaging.specifiers import SpecifierSet
from pex.third_party.packaging.version import InvalidVersion, Version
from pex.third_party.pkg_resources import Requirement, RequirementParseError
from pex.typing import TYPE_CHECKING
from pex.typing import TYPE_CHECKING, cast

if TYPE_CHECKING:
import attr # vendor:skip
Expand Down Expand Up @@ -143,14 +144,35 @@ class PyPIRequirement(object):

@attr.s(frozen=True)
class URLRequirement(object):
"""A requirement realized through an distribution archive at a fixed URL."""
"""A requirement realized through a distribution archive at a fixed URL."""

line = attr.ib() # type: LogicalLine
url = attr.ib() # type: Text
requirement = attr.ib() # type: Requirement
editable = attr.ib(default=False) # type: bool


class VCS(Enum):
class Value(Enum.Value):
pass

Bazaar = Value("bzr")
Git = Value("git")
Mercurial = Value("hg")
Subversion = Value("svn")


@attr.s(frozen=True)
class VCSRequirement(object):
"""A requirement realized by building a distribution from sources retrieved from a VCS."""

line = attr.ib() # type: LogicalLine
vcs = attr.ib() # type: VCS.Value
url = attr.ib() # type: Text
requirement = attr.ib() # type: Requirement
editable = attr.ib(default=False) # type: bool


def parse_requirement_from_project_name_and_specifier(
project_name, # type: Text
extras=None, # type: Optional[Iterable[str]]
Expand Down Expand Up @@ -208,7 +230,9 @@ def as_requirement(self, dist):


if TYPE_CHECKING:
ParsedRequirement = Union[PyPIRequirement, URLRequirement, LocalProjectRequirement]
ParsedRequirement = Union[
PyPIRequirement, URLRequirement, VCSRequirement, LocalProjectRequirement
]


@attr.s(frozen=True)
Expand Down Expand Up @@ -243,29 +267,54 @@ def _strip_requirement_options(line):
return editable, re.sub(r"\s--(global-option|install-option|hash).*$", "", processed_text)


def _is_recognized_non_local_pip_url_scheme(scheme):
# type: (str) -> bool
return bool(
re.match(
r"""
(
class ArchiveScheme(Enum):
class Value(Enum.Value):
pass

FTP = Value("ftp")
HTTP = Value("http")
HTTPS = Value("https")


@attr.s(frozen=True)
class VCSScheme(object):
vcs = attr.ib() # type: VCS.Value
scheme = attr.ib() # type: str


def _parse_non_local_scheme(scheme):
# type: (str) -> Optional[Union[ArchiveScheme.Value, VCSScheme]]
match = re.match(
r"""
^
(?:
(?P<archive_scheme>
# Archives
ftp
| https?
# VCSs: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support
| (
bzr
| git
| hg
| svn
)\+
)
""",
scheme,
re.VERBOSE,
|
(?P<vcs_type>
# VCSs: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support
bzr
| git
| hg
| svn
)\+(?P<vcs_scheme>.+)
)
$
""",
scheme,
re.VERBOSE,
)
if not match:
return None

archive_scheme = match.group("archive_scheme")
if archive_scheme:
return cast(ArchiveScheme.Value, ArchiveScheme.for_value(archive_scheme))

return VCSScheme(vcs=VCS.for_value(match.group("vcs_type")), scheme=match.group("vcs_scheme"))


@attr.s(frozen=True)
Expand Down Expand Up @@ -416,8 +465,9 @@ def _parse_requirement_line(
project_name, direct_reference_url = _split_direct_references(processed_text)
parsed_url = urlparse.urlparse(direct_reference_url or processed_text)

# Handle non local URLs (Pip proprietary).
if _is_recognized_non_local_pip_url_scheme(parsed_url.scheme):
# Handle non local URLs.
non_local_scheme = _parse_non_local_scheme(parsed_url.scheme)
if non_local_scheme:
project_name_extras_and_marker = _try_parse_fragment_project_name_and_marker(
parsed_url.fragment
)
Expand Down Expand Up @@ -454,6 +504,9 @@ def _parse_requirement_line(
specifier=specifier,
marker=marker,
)
if isinstance(non_local_scheme, VCSScheme):
url = urlparse.urlparse(url)._replace(scheme=non_local_scheme.scheme).geturl()
return VCSRequirement(line, non_local_scheme.vcs, url, requirement, editable=editable)
return URLRequirement(line, url, requirement, editable=editable)

# Handle local archives and project directories via path or file URL (Pip proprietary).
Expand Down
30 changes: 30 additions & 0 deletions tests/integration/cli/commands/test_lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,36 @@ def create_lock(style):
assert 1 == len(create_lock("sources").additional_artifacts)


def test_create_local_unsupported(pex_project_dir):
# type: (str) -> None

result = run_pex3("lock", "create", pex_project_dir)
result.assert_failure()
assert (
"Cannot create a lock for project requirements built from local or version controlled "
"sources. Given 1 such project:\n"
"1.) local project at {path}\n".format(path=pex_project_dir)
) == result.error


def test_create_vcs_unsupported():
# type: () -> None

result = run_pex3(
"lock",
"create",
"pex @ git+https://github.com/pantsbuild/pex@473c6ac7",
"git+https://github.com/pypa/pip@f0f67af3#egg=pip",
)
result.assert_failure()
assert (
"Cannot create a lock for project requirements built from local or version controlled "
"sources. Given 2 such projects:\n"
"1.) git project pex at https://github.com/pantsbuild/pex@473c6ac7\n"
"2.) git project pip at https://github.com/pypa/pip@f0f67af3\n"
) == result.error


UPDATE_LOCKFILE_CONTENTS = """\
{
"allow_builds": true,
Expand Down
51 changes: 41 additions & 10 deletions tests/test_requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@
from pex.common import safe_open, temporary_dir, touch
from pex.fetcher import URLFetcher
from pex.requirements import (
VCS,
Constraint,
LocalProjectRequirement,
LogicalLine,
ParseError,
PyPIRequirement,
Source,
URLRequirement,
VCSRequirement,
VCSScheme,
parse_requirement_file,
parse_requirement_from_project_name_and_specifier,
parse_requirements,
Expand Down Expand Up @@ -162,6 +165,27 @@ def url_req(
)


def vcs_req(
vcs, # type: VCS.Value
url, # type: str
project_name, # type: str
extras=None, # type: Optional[Iterable[str]]
specifier=None, # type: Optional[str]
marker=None, # type: Optional[str]
editable=False, # type: bool
):
# type: (...) -> VCSRequirement
return VCSRequirement(
line=DUMMY_LINE,
vcs=vcs,
url=url,
requirement=parse_requirement_from_project_name_and_specifier(
project_name, extras=extras, specifier=specifier, marker=marker
),
editable=editable,
)


def local_req(
path, # type: str
extras=None, # type: Optional[Iterable[str]]
Expand Down Expand Up @@ -343,12 +367,13 @@ def test_parse_requirements_stress(chroot):
url_req(project_name="SomeProject", url="https://example.com/somewhere/over/here"),
local_req(path=os.path.realpath("somewhere/over/here")),
req(project_name="FooProject", specifier=">=1.2"),
url_req(
vcs_req(
vcs=VCS.Git,
project_name="MyProject",
url="git+https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709",
url="https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709",
),
url_req(project_name="MyProject", url="git+ssh://git.example.com/MyProject"),
url_req(project_name="MyProject", url="git+file:/home/user/projects/MyProject"),
vcs_req(vcs=VCS.Git, project_name="MyProject", url="ssh://git.example.com/MyProject"),
vcs_req(vcs=VCS.Git, project_name="MyProject", url="file:///home/user/projects/MyProject"),
Constraint(DUMMY_LINE, Requirement.parse("AnotherProject")),
local_req(
path=os.path.realpath("extra/a/local/project"),
Expand All @@ -368,9 +393,10 @@ def test_parse_requirements_stress(chroot):
extras=["foo"],
marker="python_version == '3.9'",
),
url_req(
vcs_req(
vcs=VCS.Mercurial,
project_name="AnotherProject",
url="hg+http://hg.example.com/MyProject@da39a3ee5e6b",
url="http://hg.example.com/MyProject@da39a3ee5e6b",
extras=["more", "extra"],
marker="python_version == '3.9.*'",
),
Expand All @@ -387,11 +413,16 @@ def test_parse_requirements_stress(chroot):
specifier="==1.9.2",
marker="python_version == '3.4.*' and sys_platform == 'win32'",
),
url_req(project_name="Django", url="git+https://github.com/django/django.git"),
url_req(project_name="Django", url="git+https://github.com/django/django.git@stable/2.1.x"),
url_req(
vcs_req(vcs=VCS.Git, project_name="Django", url="https://github.com/django/django.git"),
vcs_req(
vcs=VCS.Git,
project_name="Django",
url="https://github.com/django/django.git@stable/2.1.x",
),
vcs_req(
vcs=VCS.Git,
project_name="Django",
url="git+https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8",
url="https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8",
),
url_req(
project_name="Django",
Expand Down

0 comments on commit 851666c

Please sign in to comment.