Add support for url dependencies (#1260)

docs/docs/versions.md

@@ -113,6 +113,24 @@ my-package = { path = "../my-package/dist/my-package-0.1.0.tar.gz" }
     You can install path dependencies in editable/development mode.
     Just pass `--develop my-package` (repeatable as much as you want) to
     the `install` command.
+
+
+### `url` dependencies
+
+To depend on a library located on a remote archive,
+you can use the `url` property:
+
+```toml
+[tool.poetry.dependencies]
+# directory
+my-package = { url = "https://example.com/my-package-0.1.0.tar.gz" }
+```
+
+with the corresponding `add` call:
+
+```bash
+poetry add https://example.com/my-package-0.1.0.tar.gz
+```
 
 
 ### Python restricted dependencies

poetry/console/commands/add.py

@@ -76,7 +76,11 @@ def handle(self):
             for key in poetry_content[section]:
                 if key.lower() == name.lower():
                     pair = self._parse_requirements([name])[0]
-                    if "git" in pair or pair.get("version") == "latest":
+                    if (
+                        "git" in pair
+                        or "url" in pair
+                        or pair.get("version") == "latest"
+                    ):
                         continue
 
                     raise ValueError("Package {} is already present".format(name))

poetry/console/commands/init.py

@@ -14,6 +14,8 @@
 
 from poetry.utils._compat import Path
 from poetry.utils._compat import OrderedDict
+from poetry.utils._compat import urlparse
+from poetry.utils.helpers import temporary_directory
 
 from .command import Command
 from .env_command import EnvCommand
@@ -149,6 +151,7 @@ def handle(self):
             "  - A git url with a revision (<b>https://github.com/sdispater/poetry.git@develop</b>)\n"
             "  - A file path (<b>../my-package/my-package.whl</b>)\n"
             "  - A directory (<b>../my-package/</b>)\n"
+            "  - An url (<b>https://example.com/packages/my-package-0.1.0.tar.gz</b>)\n"
         )
         help_displayed = False
         if self.confirm(question, True):
@@ -211,6 +214,7 @@ def _determine_requirements(
                 constraint = self._parse_requirements([package])[0]
                 if (
                     "git" in constraint
+                    or "url" in constraint
                     or "path" in constraint
                     or "version" in constraint
                 ):
@@ -276,7 +280,7 @@ def _determine_requirements(
         requires = self._parse_requirements(requires)
         result = []
         for requirement in requires:
-            if "git" in requirement or "path" in requirement:
+            if "git" in requirement or "url" in requirement or "path" in requirement:
                 result.append(requirement)
                 continue
             elif "version" not in requirement:
@@ -343,28 +347,42 @@ def _parse_requirements(
                 extras = [e.strip() for e in extras_m.group(1).split(",")]
                 requirement, _ = requirement.split("[")
 
-            if requirement.startswith(("git+https://", "git+ssh://")):
-                url = requirement.lstrip("git+")
-                rev = None
-                if "@" in url:
-                    url, rev = url.split("@")
+            url_parsed = urlparse.urlparse(requirement)
+            if url_parsed.scheme and url_parsed.netloc:
+                # Url
+                if url_parsed.scheme in ["git+https", "git+ssh"]:
+                    url = requirement.lstrip("git+")
+                    rev = None
+                    if "@" in url:
+                        url, rev = url.split("@")
+
+                    pair = OrderedDict(
+                        [("name", url.split("/")[-1].rstrip(".git")), ("git", url)]
+                    )
+                    if rev:
+                        pair["rev"] = rev
 
-                pair = OrderedDict(
-                    [("name", url.split("/")[-1].rstrip(".git")), ("git", url)]
-                )
-                if rev:
-                    pair["rev"] = rev
+                    if extras:
+                        pair["extras"] = extras
 
-                if extras:
-                    pair["extras"] = extras
+                    package = Provider.get_package_from_vcs(
+                        "git", url, reference=pair.get("rev")
+                    )
+                    pair["name"] = package.name
+                    result.append(pair)
 
-                package = Provider.get_package_from_vcs(
-                    "git", url, reference=pair.get("rev")
-                )
-                pair["name"] = package.name
-                result.append(pair)
+                    continue
+                elif url_parsed.scheme in ["http", "https"]:
+                    package = Provider.get_package_from_url(requirement)
 
-                continue
+                    pair = OrderedDict(
+                        [("name", package.name), ("url", package.source_url)]
+                    )
+                    if extras:
+                        pair["extras"] = extras
+
+                    result.append(pair)
+                    continue
             elif (os.path.sep in requirement or "/" in requirement) and cwd.joinpath(
                 requirement
             ).exists():

poetry/json/schemas/poetry-schema.json

@@ -210,6 +210,9 @@
                         {
                             "$ref": "#/definitions/path-dependency"
                         },
+                        {
+                            "$ref": "#/definitions/url-dependency"
+                        },
                         {
                             "$ref": "#/definitions/multiple-constraints-dependency"
                         }
@@ -394,6 +397,42 @@
                 }
             }
         },
+        "url-dependency": {
+            "type": "object",
+            "required": [
+                "url"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The url to the file."
+                },
+                "python": {
+                    "type": "string",
+                    "description": "The python versions for which the dependency should be installed."
+                },
+                "platform": {
+                    "type": "string",
+                    "description": "The platform(s) for which the dependency should be installed."
+                },
+                "markers": {
+                    "type": "string",
+                    "description": "The PEP 508 compliant environment markers for which the dependency should be installed."
+                },
+                "optional": {
+                    "type": "boolean",
+                    "description": "Whether the dependency is optional or not."
+                },
+                "extras": {
+                    "type": "array",
+                    "description": "The required extras for this dependency.",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
         "multiple-constraints-dependency": {
             "type": "array",
             "minItems": 1,

poetry/packages/__init__.py

@@ -20,6 +20,7 @@
 from .utils.utils import is_url
 from .utils.utils import path_to_url
 from .utils.utils import strip_extras
+from .url_dependency import URLDependency
 from .vcs_dependency import VCSDependency
 
 

poetry/packages/dependency.py

@@ -171,6 +171,9 @@ def is_file(self):
     def is_directory(self):
         return False
 
+    def is_url(self):
+        return False
+
     def accepts(self, package):  # type: (poetry.packages.Package) -> bool
         """
         Determines if the given package matches this dependency.

poetry/packages/package.py

@@ -18,8 +18,8 @@
 from .dependency import Dependency
 from .directory_dependency import DirectoryDependency
 from .file_dependency import FileDependency
+from .url_dependency import URLDependency
 from .vcs_dependency import VCSDependency
-from .utils.utils import convert_markers
 from .utils.utils import create_nested_marker
 
 AUTHOR_REGEX = re.compile(r"(?u)^(?P<name>[- .,\w\d'’\"()]+)(?: <(?P<email>.+?)>)?$")
@@ -111,7 +111,7 @@ def pretty_string(self):
 
     @property
     def full_pretty_version(self):
-        if self.source_type in ["file", "directory"]:
+        if self.source_type in ["file", "directory", "url"]:
             return "{} {}".format(self._pretty_version, self.source_url)
 
         if self.source_type not in ["hg", "git"]:
@@ -314,6 +314,8 @@ def add_dependency(
                         base=self.root_dir,
                         develop=constraint.get("develop", True),
                     )
+            elif "url" in constraint:
+                dependency = URLDependency(name, constraint["url"], category=category)
             else:
                 version = constraint["version"]
 

poetry/packages/url_dependency.py

@@ -0,0 +1,40 @@
+from poetry.utils._compat import urlparse
+
+from .dependency import Dependency
+
+
+class URLDependency(Dependency):
+    def __init__(
+        self,
+        name,
+        url,  # type: str
+        category="main",  # type: str
+        optional=False,  # type: bool
+    ):
+        self._url = url
+
+        parsed = urlparse.urlparse(url)
+        if not parsed.scheme or not parsed.netloc:
+            raise ValueError("{} does not seem like a valid url".format(url))
+
+        super(URLDependency, self).__init__(
+            name, "*", category=category, optional=optional, allows_prereleases=True
+        )
+
+    @property
+    def url(self):
+        return self._url
+
+    @property
+    def base_pep_508_name(self):  # type: () -> str
+        requirement = self.pretty_name
+
+        if self.extras:
+            requirement += "[{}]".format(",".join(self.extras))
+
+        requirement += " @ {}".format(self._url)
+
+        return requirement
+
+    def is_url(self):  # type: () -> bool
+        return True

poetry/puzzle/provider.py

@@ -17,6 +17,7 @@
 from poetry.packages import FileDependency
 from poetry.packages import Package
 from poetry.packages import PackageCollection
+from poetry.packages import URLDependency
 from poetry.packages import VCSDependency
 from poetry.packages import dependency_from_pep_508
 
@@ -30,10 +31,13 @@
 from poetry.utils._compat import PY35
 from poetry.utils._compat import Path
 from poetry.utils._compat import OrderedDict
+from poetry.utils._compat import urlparse
 from poetry.utils.helpers import parse_requires
 from poetry.utils.helpers import safe_rmtree
+from poetry.utils.helpers import temporary_directory
 from poetry.utils.env import EnvManager
 from poetry.utils.env import EnvCommandError
+from poetry.utils.inspector import Inspector
 from poetry.utils.setup_reader import SetupReader
 from poetry.utils.toml_file import TomlFile
 
@@ -63,6 +67,7 @@ def __init__(
         self._package = package
         self._pool = pool
         self._io = io
+        self._inspector = Inspector()
         self._python_constraint = package.python_constraint
         self._search_for = {}
         self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
@@ -127,6 +132,8 @@ def search_for(self, dependency):  # type: (Dependency) -> List[Package]
             packages = self.search_for_file(dependency)
         elif dependency.is_directory():
             packages = self.search_for_directory(dependency)
+        elif dependency.is_url():
+            packages = self.search_for_url(dependency)
         else:
             constraint = dependency.constraint
 
@@ -234,18 +241,18 @@ def search_for_file(self, dependency):  # type: (FileDependency) -> List[Package
 
     @classmethod
     def get_package_from_file(cls, file_path):  # type: (Path) -> Package
-        if file_path.suffix == ".whl":
-            meta = pkginfo.Wheel(str(file_path))
-        else:
-            # Assume sdist
-            meta = pkginfo.SDist(str(file_path))
+        info = Inspector().inspect(file_path)
+        if not info["name"]:
+            raise RuntimeError(
+                "Unable to determine the package name of {}".format(file_path)
+            )
 
-        package = Package(meta.name, meta.version)
+        package = Package(info["name"], info["version"])
         package.source_type = "file"
         package.source_url = file_path.as_posix()
 
-        package.description = meta.summary
-        for req in meta.requires_dist:
+        package.description = info["summary"]
+        for req in info["requires_dist"]:
             dep = dependency_from_pep_508(req)
             for extra in dep.in_extras:
                 if extra not in package.extras:
@@ -256,8 +263,8 @@ def get_package_from_file(cls, file_path):  # type: (Path) -> Package
             if not dep.is_optional():
                 package.requires.append(dep)
 
-        if meta.requires_python:
-            package.python_versions = meta.requires_python
+        if info["requires_python"]:
+            package.python_versions = info["requires_python"]
 
         return package
 
@@ -428,6 +435,40 @@ def get_package_from_directory(
 
         return package
 
+    def search_for_url(self, dependency):  # type: (URLDependency) -> List[Package]
+        package = self.get_package_from_url(dependency.url)
+
+        if dependency.name != package.name:
+            # For now, the dependency's name must match the actual package's name
+            raise RuntimeError(
+                "The dependency name for {} does not match the actual package's name: {}".format(
+                    dependency.name, package.name
+                )
+            )
+
+        for extra in dependency.extras:
+            if extra in package.extras:
+                for dep in package.extras[extra]:
+                    dep.activate()
+
+                package.requires += package.extras[extra]
+
+        return [package]
+
+    @classmethod
+    def get_package_from_url(cls, url):  # type: (str) -> Package
+        with temporary_directory() as temp_dir:
+            temp_dir = Path(temp_dir)
+            file_name = os.path.basename(urlparse.urlparse(url).path)
+            Inspector().download(url, temp_dir / file_name)
+
+            package = cls.get_package_from_file(temp_dir / file_name)
+
+        package.source_type = "url"
+        package.source_url = url
+
+        return package
+
     def incompatibilities_for(
         self, package
     ):  # type: (DependencyPackage) -> List[Incompatibility]
@@ -495,6 +536,7 @@ def complete_package(
         if not package.is_root() and package.source_type not in {
             "directory",
             "file",
+            "url",
             "git",
         }:
             package = DependencyPackage(