add --skip-existing to publish

add `--skip-existing` option to publish command to continue if an upload
fails due to the file already existing. this is helpful in e.g. ci
settings. resolves #2254

docs/cli.md

@@ -404,6 +404,7 @@ Should match a repository name set by the [`config`](#config) command.
 * `--username (-u)`: The username to access the repository.
 * `--password (-p)`: The password to access the repository.
 * `--dry-run`: Perform all actions except upload the package.
+* `--skip-existing`: Ignore errors from files already existing in the repository.
 
 ## config
 

poetry/console/commands/publish.py

@@ -28,6 +28,11 @@ class PublishCommand(Command):
         ),
         option("build", None, "Build the package before publishing."),
         option("dry-run", None, "Perform all actions except upload the package."),
+        option(
+            "skip-existing",
+            None,
+            "Ignore errors from files already existing in the repository.",
+        ),
     ]
 
     help = """The publish command builds and uploads the package to a remote repository.
@@ -82,4 +87,5 @@ def handle(self) -> Optional[int]:
             cert,
             client_cert,
             self.option("dry-run"),
+            self.option("skip-existing"),
         )

poetry/publishing/publisher.py

@@ -46,6 +46,7 @@ def publish(
         cert: Optional[Path] = None,
         client_cert: Optional[Path] = None,
         dry_run: Optional[bool] = False,
+        skip_existing: Optional[bool] = False,
     ) -> None:
         if not repository_name:
             url = "https://upload.pypi.org/legacy/"
@@ -102,4 +103,5 @@ def publish(
             cert=cert or get_cert(self._poetry.config, repository_name),
             client_cert=resolved_client_cert,
             dry_run=dry_run,
+            skip_existing=skip_existing,
         )

poetry/publishing/uploader.py

@@ -52,6 +52,32 @@ def __init__(self, error: Union[ConnectionError, HTTPError, str]) -> None:
         super().__init__(message)
 
 
+def _ignore_existing(
+    response, skip_existing
+):  # type: (requests.Response, bool) -> bool
+    # based on https://github.com/pypa/twine/blob/62cec5b77037345b4fe1a85bbc6b104440799496/twine/commands/upload.py#L32
+    if not skip_existing:
+        return False
+
+    status = response.status_code
+    reason = getattr(response, "reason", "").lower()
+    text = getattr(response, "text", "").lower()
+
+    # NOTE(sigmavirus24): PyPI presently returns a 400 status code with the
+    # error message in the reason attribute. Other implementations return a
+    # 403 or 409 status code.
+    return (
+        # pypiserver (https://pypi.org/project/pypiserver)
+        status == 409
+        # PyPI / TestPyPI
+        or (status == 400 and "already exist" in reason)
+        # Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss)
+        or (status == 400 and "updating asset" in reason)
+        # Artifactory (https://jfrog.com/artifactory/)
+        or (status == 403 and "overwrite artifact" in text)
+    )
+
+
 class Uploader:
     def __init__(self, poetry: "Poetry", io: "NullIO") -> None:
         self._poetry = poetry
@@ -113,6 +139,7 @@ def upload(
         cert: Optional[Path] = None,
         client_cert: Optional[Path] = None,
         dry_run: bool = False,
+        skip_existing: Optional[bool] = False,
     ) -> None:
         session = self.make_session()
 
@@ -123,7 +150,7 @@ def upload(
             session.cert = str(client_cert)
 
         try:
-            self._upload(session, url, dry_run)
+            self._upload(session, url, dry_run, skip_existing)
         finally:
             session.close()
 
@@ -206,10 +233,14 @@ def post_data(self, file: Path) -> Dict[str, Any]:
         return data
 
     def _upload(
-        self, session: requests.Session, url: str, dry_run: Optional[bool] = False
+        self,
+        session: requests.Session,
+        url: str,
+        dry_run: Optional[bool] = False,
+        skip_existing: Optional[bool] = False,
     ) -> None:
         try:
-            self._do_upload(session, url, dry_run)
+            self._do_upload(session, url, dry_run, skip_existing)
         except HTTPError as e:
             if (
                 e.response.status_code == 400
@@ -223,14 +254,18 @@ def _upload(
             raise UploadError(e)
 
     def _do_upload(
-        self, session: requests.Session, url: str, dry_run: Optional[bool] = False
+        self,
+        session: requests.Session,
+        url: str,
+        dry_run: Optional[bool] = False,
+        skip_existing: Optional[bool] = False,
     ) -> None:
         for file in self.files:
             # TODO: Check existence
 
-            resp = self._upload_file(session, url, file, dry_run)
+            resp = self._upload_file(session, url, file, dry_run, skip_existing)
 
-            if not dry_run:
+            if not dry_run and not _ignore_existing(resp, skip_existing):
                 resp.raise_for_status()
 
     def _upload_file(
@@ -239,6 +274,7 @@ def _upload_file(
         url: str,
         file: Path,
         dry_run: Optional[bool] = False,
+        skip_existing: Optional[bool] = False,
     ) -> requests.Response:
         from cleo.ui.progress_bar import ProgressBar
 
@@ -296,7 +332,14 @@ def _upload_file(
                         f" - Uploading <c1>{file.name}</c1> <error>FAILED</>"
                     )
                 raise UploadError(e)
+            else:
+                if _ignore_existing(resp, skip_existing):
+                    if self._io.output.is_decorated():
+                        self._io.overwrite(
+                            f" - Uploading <c1>{file.name}</c1> <warning>File exists. Skipping</>"
+                        )
             finally:
+                bar.finish()
                 self._io.write_line("")
 
         return resp

tests/console/commands/test_publish.py

@@ -50,7 +50,7 @@ def test_publish_with_cert(app_tester, mocker):
     app_tester.execute("publish --cert path/to/ca.pem")
 
     assert [
-        (None, None, None, Path("path/to/ca.pem"), None, False)
+        (None, None, None, Path("path/to/ca.pem"), None, False, False)
     ] == publisher_publish.call_args
 
 
@@ -59,7 +59,7 @@ def test_publish_with_client_cert(app_tester, mocker):
 
     app_tester.execute("publish --client-cert path/to/client.pem")
     assert [
-        (None, None, None, None, Path("path/to/client.pem"), False)
+        (None, None, None, None, Path("path/to/client.pem"), False, False)
     ] == publisher_publish.call_args
 
 
@@ -78,3 +78,22 @@ def test_publish_dry_run(app_tester, http):
     assert "Publishing simple-project (1.2.3) to PyPI" in output
     assert "- Uploading simple-project-1.2.3.tar.gz" in error
     assert "- Uploading simple_project-1.2.3-py2.py3-none-any.whl" in error
+
+
+def test_publish_skip_existing(app_tester, http):
+    http.register_uri(
+        http.POST, "https://upload.pypi.org/legacy/", status=409, body="Conflict"
+    )
+
+    exit_code = app_tester.execute(
+        "publish --skip-existing --username foo --password bar"
+    )
+
+    assert 0 == exit_code
+
+    output = app_tester.io.fetch_output()
+    error = app_tester.io.fetch_error()
+
+    assert "Publishing simple-project (1.2.3) to PyPI" in output
+    assert "- Uploading simple-project-1.2.3.tar.gz" in error
+    assert "- Uploading simple_project-1.2.3-py2.py3-none-any.whl" in error

tests/publishing/test_publisher.py

@@ -26,7 +26,7 @@ def test_publish_publishes_to_pypi_by_default(fixture_dir, mocker, config):
     assert [("foo", "bar")] == uploader_auth.call_args
     assert [
         ("https://upload.pypi.org/legacy/",),
-        {"cert": None, "client_cert": None, "dry_run": False},
+        {"cert": None, "client_cert": None, "dry_run": False, "skip_existing": False},
     ] == uploader_upload.call_args
 
 
@@ -57,7 +57,7 @@ def test_publish_can_publish_to_given_repository(
     assert [("foo", "bar")] == uploader_auth.call_args
     assert [
         ("http://foo.bar",),
-        {"cert": None, "client_cert": None, "dry_run": False},
+        {"cert": None, "client_cert": None, "dry_run": False, "skip_existing": False},
     ] == uploader_upload.call_args
     assert "Publishing my-package (1.2.3) to foo" in io.fetch_output()
 
@@ -87,7 +87,7 @@ def test_publish_uses_token_if_it_exists(fixture_dir, mocker, config):
     assert [("__token__", "my-token")] == uploader_auth.call_args
     assert [
         ("https://upload.pypi.org/legacy/",),
-        {"cert": None, "client_cert": None, "dry_run": False},
+        {"cert": None, "client_cert": None, "dry_run": False, "skip_existing": False},
     ] == uploader_upload.call_args
 
 
@@ -111,7 +111,12 @@ def test_publish_uses_cert(fixture_dir, mocker, config):
     assert [("foo", "bar")] == uploader_auth.call_args
     assert [
         ("https://foo.bar",),
-        {"cert": Path(cert), "client_cert": None, "dry_run": False},
+        {
+            "cert": Path(cert),
+            "client_cert": None,
+            "dry_run": False,
+            "skip_existing": False,
+        },
     ] == uploader_upload.call_args
 
 
@@ -132,7 +137,12 @@ def test_publish_uses_client_cert(fixture_dir, mocker, config):
 
     assert [
         ("https://foo.bar",),
-        {"cert": None, "client_cert": Path(client_cert), "dry_run": False},
+        {
+            "cert": None,
+            "client_cert": Path(client_cert),
+            "dry_run": False,
+            "skip_existing": False,
+        },
     ] == uploader_upload.call_args
 
 
@@ -150,5 +160,5 @@ def test_publish_read_from_environment_variable(fixture_dir, environ, mocker, co
     assert [("bar", "baz")] == uploader_auth.call_args
     assert [
         ("https://foo.bar",),
-        {"cert": None, "client_cert": None, "dry_run": False},
+        {"cert": None, "client_cert": None, "dry_run": False, "skip_existing": False},
     ] == uploader_upload.call_args

tests/publishing/test_uploader.py

@@ -60,3 +60,11 @@ def test_uploader_registers_for_appropriate_400_errors(mocker, http, uploader):
         uploader.upload("https://foo.com")
 
     assert 1 == register.call_count
+
+
+def test_uploader_skips_existing(http):
+    http.register_uri(http.POST, "https://foo.com", status=409, body="Conflict")
+    uploader = Uploader(Factory().create_poetry(project("simple_project")), NullIO())
+
+    # should not raise
+    uploader.upload("https://foo.com", skip_existing=True)