From cd70f77ca6bb7ce6bb3d203459125ea5cbe85914 Mon Sep 17 00:00:00 2001 From: quyykk Date: Fri, 28 Apr 2023 14:59:36 +0200 Subject: [PATCH 01/16] Upload the package zip in chunks for GHA. --- include/vcpkg/archives.h | 3 +++ include/vcpkg/base/downloads.h | 9 +++++-- src/vcpkg/archives.cpp | 17 ++++++++++++ src/vcpkg/base/downloads.cpp | 48 +++++++++++++++++++++++++++++++--- src/vcpkg/binarycaching.cpp | 8 ++---- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index be10f3c354..8fca0fb354 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -37,4 +37,7 @@ namespace vcpkg const Path& archive_path); std::vector> decompress_in_parallel(View jobs); + + // Split an archive into chunks. + ExpectedL split_archive(Filesystem& fs, const Path& path, int64_t file_size, int64_t chunk_size); } diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 8e4738edb5..181c5d2b62 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -38,8 +38,13 @@ namespace vcpkg StringView url, const std::vector& secrets, View headers, - const Path& file, - StringView request = "PUT"); + const Path& file); + ExpectedL patch_file_in_pieces(Filesystem& fs, + StringView url, + View headers, + const Path& file, + int64_t file_size, + unsigned int chunk_size = 450 * 1024 * 1024); std::vector url_heads(View urls, View headers, View secrets); struct DownloadManagerConfig diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index b6f4d18594..811ccf9e70 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -356,4 +356,21 @@ namespace vcpkg return filtered_results; } + + ExpectedL split_archive(Filesystem& fs, const Path& path, int64_t file_size, int64_t chunk_size) + { + auto file = fs.open_for_read(path, VCPKG_LINE_INFO); + + std::vector buffer(chunk_size); + for (int64_t begin = 0, i = 0; begin < file_size; begin += chunk_size, ++i) + { + auto bytes_read = file.read(buffer.data(), sizeof(unsigned char), chunk_size); + if (!bytes_read) break; + + auto chunk_file = fs.open_for_write(path + std::to_string(i), VCPKG_LINE_INFO); + chunk_file.write(buffer.data(), sizeof(unsigned char), bytes_read); + } + + return {Unit{}}; + } } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 81f75de535..0473154d35 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace vcpkg { static std::string replace_secrets(std::string input, View secrets) @@ -520,8 +522,7 @@ namespace vcpkg StringView url, const std::vector& secrets, View headers, - const Path& file, - StringView request) + const Path& file) { static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; @@ -550,7 +551,7 @@ namespace vcpkg } Command cmd; - cmd.string_arg("curl").string_arg("-X").string_arg(request); + cmd.string_arg("curl").string_arg("-X").string_arg("PUT"); for (auto&& header : headers) { cmd.string_arg("-H").string_arg(header); @@ -579,6 +580,47 @@ namespace vcpkg return res; } + ExpectedL patch_file_in_pieces(Filesystem& fs, + StringView url, + View headers, + const Path& file, + int64_t file_size, + unsigned int chunk_size) + { + Command base_cmd; + base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH"); + for (auto&& header : headers) + { + base_cmd.string_arg("-H").string_arg(header); + } + base_cmd.string_arg(url); + + split_archive(fs, file, file_size, chunk_size); + + int counter = 0; + for (int64_t i = 0; i < file_size; i += chunk_size, ++counter) + { + const int64_t end = std::min(i + chunk_size, file_size) - 1; + const std::string range = std::to_string(i) + "-" + std::to_string(end); + + auto cmd = base_cmd; + cmd.string_arg("-H").string_arg("Content-Range: bytes " + range + "/" + std::to_string(file_size)); + cmd.string_arg("-T").string_arg(file + std::to_string(counter)); + + auto res = cmd_execute_and_capture_output(cmd); + if (auto pres = res.get()) + { + if (pres->exit_code == 0) continue; + + Debug::print(pres->output, '\n'); + return msg::format_error( + msgCurlFailedToPut, msg::exit_code = pres->exit_code, msg::url = url.to_string()); + } + } + + return 0; + } + #if defined(_WIN32) enum class WinHttpTrialResult { diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 9123926750..56684174af 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -1130,13 +1130,9 @@ namespace if (auto cacheId = reserve_cache_entry(abi, cache_size)) { std::vector headers{ - m_token_header, - m_accept_header.to_string(), - "Content-Type: application/octet-stream", - "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", - }; + m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; auto url = m_write_url + "/" + std::to_string(*cacheId.get()); - if (put_file(fs, url, {}, headers, tmp_archive_path, "PATCH")) + if (patch_file_in_pieces(fs, url, headers, tmp_archive_path, cache_size)) { Json::Object commit; commit.insert("size", std::to_string(cache_size)); From db160c1b0b6709df3921ec016a3f5e40ba6764a7 Mon Sep 17 00:00:00 2001 From: quyykk Date: Sun, 14 May 2023 11:19:53 +0200 Subject: [PATCH 02/16] Fix merge conflicts. --- src/vcpkg/binarycaching.cpp | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index a8cee6ceaf..c5315f5dec 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -876,7 +876,7 @@ namespace struct GHABinaryPushProvider : IWriteBinaryProvider { - GHABinaryPushProvider(const Filesystem& fs, const std::string& url, const std::string& token) + GHABinaryPushProvider(Filesystem& fs, const std::string& url, const std::string& token) : m_fs(fs), m_url(url + "_apis/artifactcache/caches"), m_token_header("Authorization: Bearer " + token) { } @@ -928,23 +928,13 @@ namespace if (auto cacheId = reserve_cache_entry(abi, cache_size)) { std::vector headers{ - m_token_header, - m_accept_header.to_string(), - "Content-Type: application/octet-stream", - "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", - }; + m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; auto url = m_url + "/" + std::to_string(*cacheId.get()); - if (put_file(m_fs, url, {}, headers, zip_path, "PATCH")) + if (patch_file_in_pieces(m_fs, url, headers, zip_path, cache_size)) { - std::vector headers{ - m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; - auto url = m_write_url + "/" + std::to_string(*cacheId.get()); - if (patch_file_in_pieces(fs, url, headers, tmp_archive_path, cache_size)) - { - Json::Object commit; - commit.insert("size", std::to_string(cache_size)); - auto cmd = command().string_arg(url).string_arg("-d").string_arg(stringify(commit)); - + Json::Object commit; + commit.insert("size", std::to_string(cache_size)); + auto cmd = command().string_arg(url).string_arg("-d").string_arg(stringify(commit)); auto res = cmd_execute_and_capture_output(cmd); if (res.has_value() && !res.get()->exit_code) { @@ -960,7 +950,7 @@ namespace static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1"; - const Filesystem& m_fs; + Filesystem& m_fs; std::string m_url; std::string m_token_header; }; From 5f343b166e9a74882b854760d3e1dce46dd9217c Mon Sep 17 00:00:00 2001 From: quyykk Date: Sun, 14 May 2023 11:47:31 +0200 Subject: [PATCH 03/16] Address review comments --- src/vcpkg/archives.cpp | 7 ++++--- src/vcpkg/base/downloads.cpp | 17 +++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index d081d26d11..c4ecdf9a91 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -365,12 +365,13 @@ namespace vcpkg auto file = fs.open_for_read(path, VCPKG_LINE_INFO); std::vector buffer(chunk_size); - for (int64_t begin = 0, i = 0; begin < file_size; begin += chunk_size, ++i) + std::size_t bytes_read = 0; + for (int64_t begin = 0, i = 0; begin < file_size; begin += bytes_read, ++i) { - auto bytes_read = file.read(buffer.data(), sizeof(unsigned char), chunk_size); + bytes_read = file.read(buffer.data(), sizeof(unsigned char), chunk_size); if (!bytes_read) break; - auto chunk_file = fs.open_for_write(path + std::to_string(i), VCPKG_LINE_INFO); + auto chunk_file = fs.open_for_write(fmt::format("{}{}", path, i), VCPKG_LINE_INFO); chunk_file.write(buffer.data(), sizeof(unsigned char), bytes_read); } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index b9809a0429..62af05f3f8 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -602,23 +602,24 @@ namespace vcpkg for (int64_t i = 0; i < file_size; i += chunk_size, ++counter) { const int64_t end = std::min(i + chunk_size, file_size) - 1; - const std::string range = std::to_string(i) + "-" + std::to_string(end); + const std::string range = fmt::format("{}-{}", i, end); auto cmd = base_cmd; - cmd.string_arg("-H").string_arg("Content-Range: bytes " + range + "/" + std::to_string(file_size)); - cmd.string_arg("-T").string_arg(file + std::to_string(counter)); + cmd.string_arg("-H").string_arg(fmt::format("Content-Range: bytes {}/{}", range, file_size)); + cmd.string_arg("-T").string_arg(fmt::format("{}{}", file, counter)); auto res = cmd_execute_and_capture_output(cmd); - if (auto pres = res.get()) + if (!res.get() || res.get()->exit_code) { - if (pres->exit_code == 0) continue; - - Debug::print(pres->output, '\n'); return msg::format_error( - msgCurlFailedToPut, msg::exit_code = pres->exit_code, msg::url = url.to_string()); + msgCurlFailedToPut, msg::exit_code = res.get()->exit_code, msg::url = url.to_string()); } } + for(int64_t j = 0; j < counter; ++j) + { + fs.remove(fmt::format("{}{}", file, j), VCPKG_LINE_INFO); + } return 0; } From 905209b0f39932f4bd1d40a2f020b7392a12c7d6 Mon Sep 17 00:00:00 2001 From: quyykk Date: Sun, 14 May 2023 11:54:32 +0200 Subject: [PATCH 04/16] Fix formatting --- src/vcpkg/base/downloads.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 62af05f3f8..5ae3a1d892 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -616,7 +616,7 @@ namespace vcpkg } } - for(int64_t j = 0; j < counter; ++j) + for (int64_t j = 0; j < counter; ++j) { fs.remove(fmt::format("{}{}", file, j), VCPKG_LINE_INFO); } From 41d09b3fd11b5c4d95d2453a547cc26eef9c9a4d Mon Sep 17 00:00:00 2001 From: quyykk Date: Wed, 30 Aug 2023 18:48:49 +0200 Subject: [PATCH 05/16] Pass the data through stdin instead of spliting the archive on disk. --- include/vcpkg/archives.h | 3 --- include/vcpkg/base/downloads.h | 4 ++-- src/vcpkg/archives.cpp | 18 ----------------- src/vcpkg/base/downloads.cpp | 37 +++++++++++++++++++--------------- src/vcpkg/binarycaching.cpp | 1 + 5 files changed, 24 insertions(+), 39 deletions(-) diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index 822ca7cde3..debcfb4665 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -78,7 +78,4 @@ namespace vcpkg }; std::vector> decompress_in_parallel(View jobs); - - // Split an archive into chunks. - ExpectedL split_archive(Filesystem& fs, const Path& path, int64_t file_size, int64_t chunk_size); } diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 7ebfcd1f4a..813c543449 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -48,8 +48,8 @@ namespace vcpkg StringView url, View headers, const Path& file, - int64_t file_size, - unsigned int chunk_size = 450 * 1024 * 1024); + std::size_t file_size, + std::size_t chunk_size = 450 * 1024 * 1024); ExpectedL invoke_http_request(StringView method, View headers, diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index 9dfe1723f3..2e0d890948 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -405,22 +405,4 @@ namespace vcpkg return filtered_results; } - - ExpectedL split_archive(Filesystem& fs, const Path& path, int64_t file_size, int64_t chunk_size) - { - auto file = fs.open_for_read(path, VCPKG_LINE_INFO); - - std::vector buffer(chunk_size); - std::size_t bytes_read = 0; - for (int64_t begin = 0, i = 0; begin < file_size; begin += bytes_read, ++i) - { - bytes_read = file.read(buffer.data(), sizeof(unsigned char), chunk_size); - if (!bytes_read) break; - - auto chunk_file = fs.open_for_write(fmt::format("{}{}", path, i), VCPKG_LINE_INFO); - chunk_file.write(buffer.data(), sizeof(unsigned char), bytes_read); - } - - return {Unit{}}; - } } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 6daaf6a83c..e076b5f3b1 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -599,6 +599,7 @@ namespace vcpkg Command cmd; cmd.string_arg("curl").string_arg("-X").string_arg("PUT"); + for (auto&& header : headers) { cmd.string_arg("-H").string_arg(header); @@ -628,11 +629,11 @@ namespace vcpkg } ExpectedL patch_file(const Filesystem& fs, - StringView url, - View headers, - const Path& file, - int64_t file_size, - unsigned int chunk_size) + StringView url, + View headers, + const Path& file, + std::size_t file_size, + std::size_t chunk_size) { Command base_cmd; base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH"); @@ -642,19 +643,27 @@ namespace vcpkg } base_cmd.string_arg(url); - //split_archive(fs, file, file_size, chunk_size); - - int counter = 0; - for (int64_t i = 0; i < file_size; i += chunk_size, ++counter) + auto file_ptr = fs.open_for_read(file, VCPKG_LINE_INFO); + std::vector buffer(chunk_size); + std::size_t bytes_read = 0; + for (std::size_t i = 0; i < file_size; i += bytes_read) { const int64_t end = std::min(i + chunk_size, file_size) - 1; const std::string range = fmt::format("{}-{}", i, end); + bytes_read = file_ptr.read(buffer.data(), sizeof(decltype(buffer)::value_type), chunk_size); + if (!bytes_read) break; + auto cmd = base_cmd; cmd.string_arg("-H").string_arg(fmt::format("Content-Range: bytes {}/{}", range, file_size)); - cmd.string_arg("-T").string_arg(fmt::format("{}{}", file, counter)); - - auto res = cmd_execute_and_capture_output(cmd); + cmd.string_arg("--data-binary").string_arg("@-"); + + auto res = cmd_execute_and_capture_output(cmd, + default_working_directory, + default_environment, + Encoding::Utf8, + EchoInDebug::Hide, + StringView(buffer.data(), bytes_read)); if (!res.get() || res.get()->exit_code) { return msg::format_error( @@ -662,10 +671,6 @@ namespace vcpkg } } - for (int64_t j = 0; j < counter; ++j) - { - //fs.remove(fmt::format("{}{}", file, j), VCPKG_LINE_INFO); - } return 0; } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index ecf0dee218..c3d714bbd2 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -917,6 +917,7 @@ namespace std::vector headers{ m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; auto url = m_url + "/" + std::to_string(*cacheId.get()); + if (patch_file(m_fs, url, headers, zip_path, cache_size)) { Json::Object commit; From d025512c6b9fc3bc565e89b251558a3e5afb5ed3 Mon Sep 17 00:00:00 2001 From: quyykk Date: Wed, 30 Aug 2023 19:00:47 +0200 Subject: [PATCH 06/16] Fix warning. --- src/vcpkg/binarycaching.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index c3d714bbd2..8597383bba 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -914,11 +914,11 @@ namespace if (auto cacheId = reserve_cache_entry(request.spec.name(), abi, cache_size)) { - std::vector headers{ + std::vector custom_headers{ m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; auto url = m_url + "/" + std::to_string(*cacheId.get()); - if (patch_file(m_fs, url, headers, zip_path, cache_size)) + if (patch_file(m_fs, url, custom_headers, zip_path, cache_size)) { Json::Object commit; commit.insert("size", std::to_string(cache_size)); From 126232c6f19461b566974579ebb8b1e00c3464c4 Mon Sep 17 00:00:00 2001 From: quyykk Date: Wed, 30 Aug 2023 19:12:54 +0200 Subject: [PATCH 07/16] Fix formatting. --- include/vcpkg/base/downloads.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 813c543449..7c6ab18182 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -45,11 +45,11 @@ namespace vcpkg View headers, const Path& file); ExpectedL patch_file(const Filesystem& fs, - StringView url, - View headers, - const Path& file, - std::size_t file_size, - std::size_t chunk_size = 450 * 1024 * 1024); + StringView url, + View headers, + const Path& file, + std::size_t file_size, + std::size_t chunk_size = 450 * 1024 * 1024); ExpectedL invoke_http_request(StringView method, View headers, From 18868f13c4187af6b9d8e670f7d4a72e250ff6f3 Mon Sep 17 00:00:00 2001 From: quyykk Date: Wed, 30 Aug 2023 22:29:15 +0200 Subject: [PATCH 08/16] Send the correct range in the HTTP request. --- src/vcpkg/base/downloads.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index e076b5f3b1..a853429689 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -648,12 +648,12 @@ namespace vcpkg std::size_t bytes_read = 0; for (std::size_t i = 0; i < file_size; i += bytes_read) { - const int64_t end = std::min(i + chunk_size, file_size) - 1; - const std::string range = fmt::format("{}-{}", i, end); - bytes_read = file_ptr.read(buffer.data(), sizeof(decltype(buffer)::value_type), chunk_size); if (!bytes_read) break; + const std::size_t end = std::min(i + bytes_read, file_size) - 1; + const std::string range = fmt::format("{}-{}", i, end); + auto cmd = base_cmd; cmd.string_arg("-H").string_arg(fmt::format("Content-Range: bytes {}/{}", range, file_size)); cmd.string_arg("--data-binary").string_arg("@-"); From e6bf3045fa1c39c945832855c43fa3e76b605f46 Mon Sep 17 00:00:00 2001 From: quyykk Date: Thu, 7 Sep 2023 13:47:23 +0200 Subject: [PATCH 09/16] Some small cleanups. --- src/vcpkg/base/downloads.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index a853429689..06685db9a0 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -651,12 +651,11 @@ namespace vcpkg bytes_read = file_ptr.read(buffer.data(), sizeof(decltype(buffer)::value_type), chunk_size); if (!bytes_read) break; - const std::size_t end = std::min(i + bytes_read, file_size) - 1; - const std::string range = fmt::format("{}-{}", i, end); - auto cmd = base_cmd; - cmd.string_arg("-H").string_arg(fmt::format("Content-Range: bytes {}/{}", range, file_size)); - cmd.string_arg("--data-binary").string_arg("@-"); + cmd.string_arg("-H") + .string_arg(fmt::format("Content-Range: bytes {}-{}/{}", i, i + bytes_read - 1, file_size)) + .string_arg("--data-binary") + .string_arg("@-"); auto res = cmd_execute_and_capture_output(cmd, default_working_directory, From 8b15eddd9a256937952408e8db3172424c7827ab Mon Sep 17 00:00:00 2001 From: quyykk Date: Sat, 9 Sep 2023 20:48:37 +0200 Subject: [PATCH 10/16] Address review comments. --- include/vcpkg/base/downloads.h | 2 +- src/vcpkg/base/downloads.cpp | 54 +++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 7c6ab18182..d60caa0bd8 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -44,7 +44,7 @@ namespace vcpkg const std::vector& secrets, View headers, const Path& file); - ExpectedL patch_file(const Filesystem& fs, + ExpectedL patch_file(const Filesystem& fs, StringView url, View headers, const Path& file, diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 06685db9a0..d30ff0d196 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -18,6 +18,8 @@ namespace vcpkg { + static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; + static std::string replace_secrets(std::string input, View secrets) { const auto replacement = msg::format(msgSecretBanner); @@ -571,8 +573,6 @@ namespace vcpkg View headers, const Path& file) { - static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - if (Strings::starts_with(url, "ftp://")) { // HTTP headers are ignored for FTP clients @@ -628,15 +628,16 @@ namespace vcpkg return res; } - ExpectedL patch_file(const Filesystem& fs, - StringView url, - View headers, - const Path& file, - std::size_t file_size, - std::size_t chunk_size) + ExpectedL patch_file(const Filesystem& fs, + StringView url, + View headers, + const Path& file, + std::size_t file_size, + std::size_t chunk_size) { Command base_cmd; - base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH"); + base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH").string_arg("-w").string_arg( + "\\n" + guid_marker.to_string() + "%{http_code}"); for (auto&& header : headers) { base_cmd.string_arg("-H").string_arg(header); @@ -649,7 +650,11 @@ namespace vcpkg for (std::size_t i = 0; i < file_size; i += bytes_read) { bytes_read = file_ptr.read(buffer.data(), sizeof(decltype(buffer)::value_type), chunk_size); - if (!bytes_read) break; + if (!bytes_read) + { + return msg::format_error( + msgFileReadFailed, msg::path = file, msg::byte_offset = i, msg::count = chunk_size); + } auto cmd = base_cmd; cmd.string_arg("-H") @@ -657,20 +662,29 @@ namespace vcpkg .string_arg("--data-binary") .string_arg("@-"); - auto res = cmd_execute_and_capture_output(cmd, - default_working_directory, - default_environment, - Encoding::Utf8, - EchoInDebug::Hide, - StringView(buffer.data(), bytes_read)); - if (!res.get() || res.get()->exit_code) + int code = 0; + auto res = cmd_execute_and_stream_lines( + cmd, + [&code](StringView line) { + if (Strings::starts_with(line, guid_marker)) + { + code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); + } + }, + default_working_directory, + default_environment, + Encoding::Utf8, + StringView(buffer.data(), bytes_read)); + if (!res.get() || *res.get() != 0 || (code >= 100 && code < 200) || code >= 300) { - return msg::format_error( - msgCurlFailedToPut, msg::exit_code = res.get()->exit_code, msg::url = url.to_string()); + return msg::format_error(msgCurlFailedToPutHttp, + msg::exit_code = res.has_value() ? *res.get() : -1, + msg::url = url, + msg::value = code); } } - return 0; + return Unit{}; } std::string format_url_query(StringView base_url, View query_params) From 1150fedca7e0ada0515e4cccb9a19d16dab4d8f0 Mon Sep 17 00:00:00 2001 From: quyykk Date: Sat, 9 Sep 2023 20:54:08 +0200 Subject: [PATCH 11/16] Format :/ --- include/vcpkg/base/downloads.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index d60caa0bd8..5de257575d 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -45,11 +45,11 @@ namespace vcpkg View headers, const Path& file); ExpectedL patch_file(const Filesystem& fs, - StringView url, - View headers, - const Path& file, - std::size_t file_size, - std::size_t chunk_size = 450 * 1024 * 1024); + StringView url, + View headers, + const Path& file, + std::size_t file_size, + std::size_t chunk_size = 450 * 1024 * 1024); ExpectedL invoke_http_request(StringView method, View headers, From dbed226766ef650459787d7d03642898d02278f5 Mon Sep 17 00:00:00 2001 From: Nick Date: Sat, 9 Sep 2023 21:02:57 +0200 Subject: [PATCH 12/16] Update src/vcpkg/base/downloads.cpp Co-authored-by: autoantwort <41973254+autoantwort@users.noreply.github.com> --- src/vcpkg/base/downloads.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index d30ff0d196..4ad52de71c 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -637,7 +637,7 @@ namespace vcpkg { Command base_cmd; base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH").string_arg("-w").string_arg( - "\\n" + guid_marker.to_string() + "%{http_code}"); + "\\n" + guid_marker.to_string() + "%{http_code}\n"); for (auto&& header : headers) { base_cmd.string_arg("-H").string_arg(header); From ef45cb8744f6397f59dcb91f8f93b7cad3429fc1 Mon Sep 17 00:00:00 2001 From: quyykk Date: Sat, 9 Sep 2023 21:04:12 +0200 Subject: [PATCH 13/16] More fixes. --- src/vcpkg/base/downloads.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 4ad52de71c..9383b4fe61 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -18,8 +18,6 @@ namespace vcpkg { - static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - static std::string replace_secrets(std::string input, View secrets) { const auto replacement = msg::format(msgSecretBanner); @@ -573,6 +571,8 @@ namespace vcpkg View headers, const Path& file) { + static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; + if (Strings::starts_with(url, "ftp://")) { // HTTP headers are ignored for FTP clients @@ -635,6 +635,8 @@ namespace vcpkg std::size_t file_size, std::size_t chunk_size) { + static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; + Command base_cmd; base_cmd.string_arg("curl").string_arg("-X").string_arg("PATCH").string_arg("-w").string_arg( "\\n" + guid_marker.to_string() + "%{http_code}\n"); From 2568af09ab423da0770e73a37600c1d8392f678f Mon Sep 17 00:00:00 2001 From: quyykk Date: Mon, 11 Mar 2024 21:22:10 +0100 Subject: [PATCH 14/16] Format fixes. --- src/vcpkg/base/downloads.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 8b1e2c7eaa..087f879e7d 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -656,15 +656,12 @@ namespace vcpkg int code = 0; RedirectedProcessLaunchSettings launch_settings; launch_settings.stdin_content = {buffer.data(), bytes_read}; - auto res = cmd_execute_and_stream_lines( - cmd, - launch_settings, - [&code](StringView line) { - if (Strings::starts_with(line, guid_marker)) - { - code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); - } - }); + auto res = cmd_execute_and_stream_lines(cmd, launch_settings, [&code](StringView line) { + if (Strings::starts_with(line, guid_marker)) + { + code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); + } + }); if (!res.get() || *res.get() != 0 || (code >= 100 && code < 200) || code >= 300) { return msg::format_error(msgCurlFailedToPutHttp, From d397db393dd63655d3c1c8db5dce0408167906e2 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 12 Mar 2024 10:28:39 +0100 Subject: [PATCH 15/16] Apply suggestions from code review. Thanks! Co-authored-by: Thomas1664 <46387399+Thomas1664@users.noreply.github.com> --- src/vcpkg/base/downloads.cpp | 2 +- src/vcpkg/binarycaching.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 087f879e7d..ea55f1d700 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -665,7 +665,7 @@ namespace vcpkg if (!res.get() || *res.get() != 0 || (code >= 100 && code < 200) || code >= 300) { return msg::format_error(msgCurlFailedToPutHttp, - msg::exit_code = res.has_value() ? *res.get() : -1, + msg::exit_code = res.value_or(-1), msg::url = url, msg::value = code); } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index d133f971c3..122eaad244 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -910,8 +910,11 @@ namespace if (auto cacheId = reserve_cache_entry(request.spec.name(), abi, cache_size)) { - std::vector custom_headers{ - m_token_header, m_accept_header.to_string(), "Content-Type: application/octet-stream"}; + const std::string custom_headers[] = { + m_token_header, + m_accept_header.to_string(), + "Content-Type: application/octet-stream", + }; auto url = m_url + "/" + std::to_string(*cacheId.get()); if (patch_file(m_fs, url, custom_headers, zip_path, cache_size)) From 608a72bdcba7ce4828cd309a6fa75810a7d3e536 Mon Sep 17 00:00:00 2001 From: quyykk Date: Tue, 12 Mar 2024 11:06:49 +0100 Subject: [PATCH 16/16] Format fixes. --- src/vcpkg/base/downloads.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index ea55f1d700..feb72d7135 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -664,10 +664,8 @@ namespace vcpkg }); if (!res.get() || *res.get() != 0 || (code >= 100 && code < 200) || code >= 300) { - return msg::format_error(msgCurlFailedToPutHttp, - msg::exit_code = res.value_or(-1), - msg::url = url, - msg::value = code); + return msg::format_error( + msgCurlFailedToPutHttp, msg::exit_code = res.value_or(-1), msg::url = url, msg::value = code); } }