Allow trailing / in base-url (#1111)

Closes #1110

esrally/track/loader.py

@@ -398,7 +398,7 @@ def download(self, base_url, target_path, size_in_bytes, detail_on_missing_root_
         if self.offline:
             raise exceptions.SystemSetupError("Cannot find %s. Please disable offline mode and retry again." % target_path)
 
-        data_url = "%s/%s" % (base_url, file_name)
+        data_url = f"{urllib.parse.urljoin(base_url + '/', file_name)}"
         try:
             io.ensure_dir(os.path.dirname(target_path))
             if size_in_bytes:

esrally/utils/net.py

@@ -185,7 +185,7 @@ def download(url, local_path, expected_size_in_bytes=None, progress_indicator=No
     """
     Downloads a single file from a URL to the provided local path.
 
-    :param url: The remote URL specifying one file that should be downloaded. May be either a HTTP, HTTPS or S3 URL.
+    :param url: The remote URL specifying one file that should be downloaded. May be either a HTTP, HTTPS, S3 or GS URL.
     :param local_path: The local file name of the file that should be downloaded.
     :param expected_size_in_bytes: The expected file size in bytes if known. It will be used to verify that all data have been downloaded.
     :param progress_indicator A callable that can be use to report progress to the user. It is expected to take two parameters

tests/track/loader_test.py

@@ -271,6 +271,41 @@ def test_download_document_archive_if_no_file_available(self, is_file, get_size,
                                     "/tmp/docs.json.bz2", 200, progress_indicator=mock.ANY)
         prepare_file_offset_table.assert_called_with("/tmp/docs.json")
 
+    @mock.patch("esrally.utils.io.prepare_file_offset_table")
+    @mock.patch("esrally.utils.io.decompress")
+    @mock.patch("esrally.utils.net.download")
+    @mock.patch("esrally.utils.io.ensure_dir")
+    @mock.patch("os.path.getsize")
+    @mock.patch("os.path.isfile")
+    def test_download_document_with_trailing_baseurl_slash(self, is_file, get_size, ensure_dir, download, decompress,
+                                                           prepare_file_offset_table):
+        # uncompressed file does not exist
+        # file check for uncompressed file before download attempt (for potential error message)
+        # after download uncompressed file exists
+        # after download uncompressed file exists (main loop)
+        is_file.side_effect = [False, False, True, True]
+        # uncompressed file size is 2000
+        get_size.return_value = 2000
+
+        prepare_file_offset_table.return_value = 5
+
+        p = loader.DocumentSetPreparator(track_name="unit-test", offline=False, test_mode=False)
+
+        p.prepare_document_set(document_set=track.Documents(source_format=track.Documents.SOURCE_FORMAT_BULK,
+                                                            base_url="http://benchmarks.elasticsearch.org/corpora/unit-test/",
+                                                            document_file="docs.json",
+                                                            # --> We don't provide a document archive here <--
+                                                            document_archive=None,
+                                                            number_of_documents=5,
+                                                            compressed_size_in_bytes=200,
+                                                            uncompressed_size_in_bytes=2000),
+                               data_root="/tmp")
+
+        ensure_dir.assert_called_with("/tmp")
+        download.assert_called_with("http://benchmarks.elasticsearch.org/corpora/unit-test/docs.json",
+                                    "/tmp/docs.json", 2000, progress_indicator=mock.ANY)
+        prepare_file_offset_table.assert_called_with("/tmp/docs.json")
+
     @mock.patch("esrally.utils.io.prepare_file_offset_table")
     @mock.patch("esrally.utils.net.download")
     @mock.patch("esrally.utils.io.ensure_dir")