Introduce 'x-script' asset caching protocol (#237)

* Introduce 'x-script' asset caching protocol

* Remove use of std::function

* Extract abi_stable_format.h

include/vcpkg/base/api_stable_format.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <vcpkg/base/expected.h>
+#include <vcpkg/base/stringview.h>
+
+#include <string>
+
+namespace vcpkg
+{
+    namespace details
+    {
+        template<class F>
+        void api_stable_format_cb(void* f, std::string& s, StringView sv)
+        {
+            (*(F*)(f))(s, sv);
+        }
+
+        ExpectedS<std::string> api_stable_format_impl(StringView fmtstr,
+                                                      void (*cb)(void*, std::string&, StringView),
+                                                      void* data);
+
+    }
+
+    // This function exists in order to provide an API-stable formatting function similar to `std::format()` that does
+    // not depend on the feature set of fmt or the C++ standard library and thus can be contractual for user interfaces.
+    template<class F>
+    ExpectedS<std::string> api_stable_format(StringView fmtstr, F&& handler)
+    {
+        return details::api_stable_format_impl(fmtstr, &details::api_stable_format_cb<F>, &handler);
+    }
+}

include/vcpkg/base/downloads.h

@@ -43,6 +43,7 @@ namespace vcpkg::Downloads
         std::vector<std::string> m_write_headers;
         std::vector<std::string> m_secrets;
         bool m_block_origin = false;
+        Optional<std::string> m_script;
     };
 
     // Handles downloading and uploading to a content addressable mirror

include/vcpkg/base/expected.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <vcpkg/base/fwd/expected.h>
 #include <vcpkg/base/fwd/messages.h>
 
 #include <vcpkg/base/checks.h>
@@ -236,10 +237,4 @@ namespace vcpkg
         ErrorHolder<S> m_s;
         ExpectedHolder<T> m_t;
     };
-
-    template<class T>
-    using Expected = ExpectedT<T, std::error_code>;
-
-    template<class T>
-    using ExpectedS = ExpectedT<T, std::string>;
 }

include/vcpkg/base/files.h

@@ -4,6 +4,7 @@
 
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/pragmas.h>
+#include <vcpkg/base/stringliteral.h>
 #include <vcpkg/base/stringview.h>
 
 #include <stdio.h>

include/vcpkg/base/fwd/expected.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string>
+#include <system_error>
+
+namespace vcpkg
+{
+    template<class T, class S>
+    struct ExpectedT;
+
+    template<class T>
+    using Expected = ExpectedT<T, std::error_code>;
+
+    template<class T>
+    using ExpectedS = ExpectedT<T, std::string>;
+}

include/vcpkg/base/strings.h

@@ -3,15 +3,13 @@
 #include <vcpkg/base/cstringview.h>
 #include <vcpkg/base/lineinfo.h>
 #include <vcpkg/base/optional.h>
-#include <vcpkg/base/pragmas.h>
-#include <vcpkg/base/stringliteral.h>
 #include <vcpkg/base/stringview.h>
-#include <vcpkg/base/view.h>
 
 #include <errno.h>
 #include <inttypes.h>
 #include <limits.h>
 
+#include <algorithm>
 #include <vector>
 
 namespace vcpkg::Strings::details
@@ -72,11 +70,6 @@ namespace vcpkg::Strings::details
     {
         to_string(into, t);
     }
-
-    constexpr struct
-    {
-        char operator()(char c) const noexcept { return (c < 'A' || c > 'Z') ? c : c - 'A' + 'a'; }
-    } tolower_char;
 }
 
 namespace vcpkg::Strings
@@ -142,11 +135,7 @@ namespace vcpkg::Strings
 
     bool case_insensitive_ascii_equals(StringView left, StringView right);
 
-    template<class It>
-    void ascii_to_lowercase(It first, It last)
-    {
-        std::transform(first, last, first, details::tolower_char);
-    }
+    void ascii_to_lowercase(char* first, char* last);
     std::string ascii_to_lowercase(std::string&& s);
 
     std::string ascii_to_uppercase(std::string&& s);

include/vcpkg/base/system.process.h

@@ -3,6 +3,7 @@
 #include <vcpkg/base/fwd/system.process.h>
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/view.h>
 #include <vcpkg/base/zstringview.h>
 
 #include <functional>
@@ -34,7 +35,10 @@ namespace vcpkg
         Command& string_arg(StringView s) &;
         Command& raw_arg(StringView s) &
         {
-            buf.push_back(' ');
+            if (!buf.empty())
+            {
+                buf.push_back(' ');
+            }
             buf.append(s.data(), s.size());
             return *this;
         }
@@ -109,10 +113,13 @@ namespace vcpkg
 
     ExitCodeAndOutput cmd_execute_and_capture_output(const Command& cmd_line,
                                                      InWorkingDirectory wd,
-                                                     const Environment& env = {});
-    inline ExitCodeAndOutput cmd_execute_and_capture_output(const Command& cmd_line, const Environment& env = {})
+                                                     const Environment& env = {},
+                                                     bool tee_in_debug = false);
+    inline ExitCodeAndOutput cmd_execute_and_capture_output(const Command& cmd_line,
+                                                            const Environment& env = {},
+                                                            bool tee_in_debug = false)
     {
-        return cmd_execute_and_capture_output(cmd_line, InWorkingDirectory{Path()}, env);
+        return cmd_execute_and_capture_output(cmd_line, InWorkingDirectory{Path()}, env, tee_in_debug);
     }
 
     std::vector<ExitCodeAndOutput> cmd_execute_and_capture_output_parallel(View<Command> cmd_lines,

include/vcpkg/cmakevars.h

@@ -5,6 +5,7 @@
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/optional.h>
+#include <vcpkg/base/span.h>
 
 #include <vcpkg/packagespec.h>
 

include/vcpkg/commands.h

@@ -1,7 +1,12 @@
 #pragma once
 
+#include <vcpkg/base/span.h>
+#include <vcpkg/base/stringview.h>
+
 #include <vcpkg/commands.interface.h>
 
+#include <string>
+
 namespace vcpkg::Commands
 {
     template<class T>

src/vcpkg-test/strings.cpp

@@ -1,5 +1,7 @@
 #include <catch2/catch.hpp>
 
+#include <vcpkg/base/api_stable_format.h>
+#include <vcpkg/base/expected.h>
 #include <vcpkg/base/strings.h>
 
 #include <stdint.h>
@@ -112,3 +114,42 @@ TEST_CASE ("inplace_replace_all(char)", "[strings]")
     inplace_replace_all(target, 'x', '?');
     REQUIRE(target == "hewwo");
 }
+
+TEST_CASE ("api_stable_format(sv,append_f)", "[strings]")
+{
+    namespace Strings = vcpkg::Strings;
+    using vcpkg::api_stable_format;
+    using vcpkg::nullopt;
+    using vcpkg::StringView;
+
+    std::string target;
+    auto res = api_stable_format("{", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(!res.has_value());
+    res = api_stable_format("}", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(!res.has_value());
+    res = api_stable_format("{ {", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(!res.has_value());
+    res = api_stable_format("{ {}", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(!res.has_value());
+
+    res = api_stable_format("}}", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(*res.get() == "}");
+    res = api_stable_format("{{", [](std::string&, StringView) { CHECK(false); });
+    REQUIRE(*res.get() == "{");
+
+    res = api_stable_format("{x}{y}{z}", [](std::string& out, StringView t) {
+        CHECK((t == "x" || t == "y" || t == "z"));
+        Strings::append(out, t, t);
+    });
+    REQUIRE(*res.get() == "xxyyzz");
+    res = api_stable_format("{x}}}", [](std::string& out, StringView t) {
+        CHECK(t == "x");
+        Strings::append(out, "hello");
+    });
+    REQUIRE(*res.get() == "hello}");
+    res = api_stable_format("123{x}456", [](std::string& out, StringView t) {
+        CHECK(t == "x");
+        Strings::append(out, "hello");
+    });
+    REQUIRE(*res.get() == "123hello456");
+}

src/vcpkg/base/downloads.cpp

@@ -1,6 +1,8 @@
+#include <vcpkg/base/api_stable_format.h>
 #include <vcpkg/base/cache.h>
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/hash.h>
+#include <vcpkg/base/json.h>
 #include <vcpkg/base/lockguarded.h>
 #include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.h>
@@ -611,6 +613,17 @@ namespace vcpkg::Downloads
                                                const Optional<std::string>& sha512) const
     {
         std::string errors;
+        if (urls.size() == 0)
+        {
+            if (auto hash = sha512.get())
+            {
+                Strings::append(errors, "Error: No urls specified to download SHA: ", *hash);
+            }
+            else
+            {
+                Strings::append(errors, "Error: No urls specified and no hash specified.");
+            }
+        }
         if (auto hash = sha512.get())
         {
             if (auto read_template = m_config.m_read_url_template.get())
@@ -620,22 +633,58 @@ namespace vcpkg::Downloads
                         fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
                     return read_url;
             }
-        }
-
-        if (!m_config.m_block_origin)
-        {
-            if (urls.size() == 0)
+            else if (auto script = m_config.m_script.get())
             {
-                if (auto hash = sha512.get())
-                {
-                    Strings::append(errors, "Error: No urls specified to download SHA: ", *hash, '\n');
-                }
-                else
+                if (urls.size() != 0)
                 {
-                    Strings::append(errors, "Error: No urls specified\n");
+                    const auto download_path_part_path =
+                        download_path + Strings::concat(".", get_process_id(), ".part");
+
+                    const auto escaped_url = Command(urls[0]).extract();
+                    const auto escaped_sha512 = Command(*hash).extract();
+                    const auto escaped_dpath = Command(download_path_part_path).extract();
+
+                    auto cmd = api_stable_format(*script, [&](std::string& out, StringView key) {
+                                   if (key == "url")
+                                   {
+                                       Strings::append(out, escaped_url);
+                                   }
+                                   else if (key == "sha512")
+                                   {
+                                       Strings::append(out, escaped_sha512);
+                                   }
+                                   else if (key == "dst")
+                                   {
+                                       Strings::append(out, escaped_dpath);
+                                   }
+                               }).value_or_exit(VCPKG_LINE_INFO);
+
+                    auto res = cmd_execute_and_capture_output(Command{}.raw_arg(cmd), get_clean_environment(), true);
+                    if (res.exit_code == 0)
+                    {
+                        auto maybe_error =
+                            try_verify_downloaded_file_hash(fs, "<mirror-script>", download_path_part_path, *hash);
+                        if (auto err = maybe_error.get())
+                        {
+                            Strings::append(errors, *err);
+                        }
+                        else
+                        {
+                            fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
+                            return urls[0];
+                        }
+                    }
+                    else
+                    {
+                        Strings::append(errors, res.output);
+                    }
                 }
             }
-            else
+        }
+
+        if (!m_config.m_block_origin)
+        {
+            if (urls.size() != 0)
             {
                 auto maybe_url =
                     try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors);