[vcpkg] Add experimental x-azblob binary provider (microsoft#13626)

* [vcpkg] Add experimental x-azblob binary provider * [vcpkg] Test azblob storage provider in CI * [vcpkg] Address some CR comments from microsoft#13639 * [vcpkg] Fixup azure-pipelines * [vcpkg] Fix regression where the downloaded package is purged before decompressing * [vcpkg] Further refactor vcpkg::Downloads * [vcpkg] Enable OSX for x-azblob testing * [vcpkg] Reduce diff against master * [vcpkg] Extract Downloads::details::split_uri_view * [vcpkg] Address PR comments * [vcpkg] Add testing and metrics for x-azblob * [vcpkg] Add docs for x-azblob This includes a note that it is currently experimental * [vcpkg] Address CR comments * [vcpkg] Revert pipeline changes except OSX to minimize disruption Co-authored-by: Robert Schumacher <[email protected]> Co-authored-by: Billy Robert O'Neal III <[email protected]>
strega-nil · Nov 18, 2020 · f65bc01 · f65bc01
1 parent 9897e27
commit f65bc01
Show file tree

Hide file tree

Showing 16 changed files with 933 additions and 281 deletions.
diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h
@@ -1,16 +1,41 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/optional.h>
+#include <vcpkg/base/view.h>
 
 namespace vcpkg::Downloads
 {
+    namespace details
+    {
+        struct SplitURIView
+        {
+            StringView scheme;
+            Optional<StringView> authority;
+            StringView path_query_fragment;
+        };
+
+        // e.g. {"https","//example.org", "/index.html"}
+        Optional<SplitURIView> split_uri_view(StringView uri);
+    }
+
     void verify_downloaded_file_hash(const Files::Filesystem& fs,
                                      const std::string& url,
                                      const fs::path& path,
                                      const std::string& sha512);
 
+    // Returns url that was successfully downloaded from
+    std::string download_file(Files::Filesystem& fs,
+                              View<std::string> urls,
+                              const fs::path& download_path,
+                              const std::string& sha512);
+
     void download_file(Files::Filesystem& fs,
                        const std::string& url,
                        const fs::path& download_path,
                        const std::string& sha512);
+
+    std::vector<int> download_files(Files::Filesystem& fs, View<std::pair<std::string, fs::path>> url_pairs);
+    int put_file(const Files::Filesystem&, StringView url, const fs::path& file);
+    std::vector<int> url_heads(View<std::string> urls);
 }
diff --git a/include/vcpkg/base/fwd/lockguarded.h b/include/vcpkg/base/fwd/lockguarded.h
@@ -0,0 +1,10 @@
+#pragma once
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuardPtr;
+
+    template<class T>
+    struct LockGuarded;
+}
diff --git a/include/vcpkg/base/lockguarded.h b/include/vcpkg/base/lockguarded.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vcpkg/base/fwd/lockguarded.h>
+
+#include <mutex>
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuarded
+    {
+        friend struct LockGuardPtr<T>;
+
+        LockGuardPtr<T> lock() { return *this; }
+
+    private:
+        std::mutex m_mutex;
+        T m_t;
+    };
+
+    template<class T>
+    struct LockGuardPtr
+    {
+        T& operator*() { return m_ptr; }
+        T* operator->() { return &m_ptr; }
+
+        T* get() { return &m_ptr; }
+
+        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
+
+    private:
+        std::lock_guard<std::mutex> m_lock;
+        T& m_ptr;
+    };
+}
diff --git a/include/vcpkg/base/util.h b/include/vcpkg/base/util.h
@@ -5,7 +5,6 @@
 #include <algorithm>
 #include <functional>
 #include <map>
-#include <mutex>
 #include <type_traits>
 #include <unordered_map>
 #include <utility>
@@ -222,36 +221,6 @@ namespace vcpkg::Util
         ~ResourceBase() = default;
     };
 
-    template<class T>
-    struct LockGuardPtr;
-
-    template<class T>
-    struct LockGuarded
-    {
-        friend struct LockGuardPtr<T>;
-
-        LockGuardPtr<T> lock() { return *this; }
-
-    private:
-        std::mutex m_mutex;
-        T m_t;
-    };
-
-    template<class T>
-    struct LockGuardPtr
-    {
-        T& operator*() { return m_ptr; }
-        T* operator->() { return &m_ptr; }
-
-        T* get() { return &m_ptr; }
-
-        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
-
-    private:
-        std::unique_lock<std::mutex> m_lock;
-        T& m_ptr;
-    };
-
     namespace Enum
     {
         template<class E>

diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h
@@ -8,8 +8,12 @@
 
 #include <vcpkg/packagespec.h>
 
+#include <unordered_map>
+
 namespace vcpkg
 {
+    struct MergeBinaryProviders;
+
     enum class RestoreResult
     {
         missing,
@@ -20,18 +24,32 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
-        /// `plan`.
-        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+
         /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
         /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
-        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
-        /// missing packages.
-        virtual RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
+        /// <summary>Gives the BinaryProvider an opportunity to batch any downloading or server communication for
+        /// executing `plan`.</summary>
+        /// <remarks>Must only be called once for a given binary provider instance</remarks>
+        /// <param name="actions">InOut vector of actions to be prefetched</param>
+        virtual void prefetch(const VcpkgPaths& paths,
+                              std::vector<const Dependencies::InstallPlanAction*>& actions) = 0;
+
+        /// <summary>Requests the result of <c>try_restore()</c> without actually downloading the package. Used by CI to
+        /// determine missing packages.</summary>
+        /// <param name="results_map">InOut map to track the restored packages. Should be initialized to
+        /// <c>{&amp;action, RestoreResult::missing}</c> for all install actions</param>
+        virtual void precheck(
+            const VcpkgPaths& paths,
+            std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) = 0;
     };
 
+    std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> binary_provider_precheck(
+        const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider);
+
     IBinaryProvider& null_binary_provider();
 
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(View<std::string> args);

diff --git a/include/vcpkg/dependencies.h b/include/vcpkg/dependencies.h
@@ -61,6 +61,7 @@ namespace vcpkg::Dependencies
         std::string displayname() const;
         const std::string& public_abi() const;
         bool has_package_abi() const;
+        Optional<const std::string&> package_abi() const;
         const Build::PreBuildInfo& pre_build_info(LineInfo linfo) const;
 
         PackageSpec spec;

diff --git a/include/vcpkg/globalstate.h b/include/vcpkg/globalstate.h
@@ -1,7 +1,8 @@
 #pragma once
 
+#include <vcpkg/base/fwd/lockguarded.h>
+
 #include <vcpkg/base/chrono.h>
-#include <vcpkg/base/util.h>
 
 #include <atomic>
 #include <string>

diff --git a/include/vcpkg/metrics.h b/include/vcpkg/metrics.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/lockguarded.h>
 #include <vcpkg/base/util.h>
 
 #include <string>

diff --git a/src/vcpkg-test/binaryconfigparser.cpp b/src/vcpkg-test/binaryconfigparser.cpp
@@ -298,3 +298,39 @@ TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]")
         REQUIRE(parsed.has_value());
     }
 }
+
+TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,?sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,,sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,invalid", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,read", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,write", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,readwrite", {});
+        REQUIRE(parsed.has_value());
+    }
+}
diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp
@@ -0,0 +1,59 @@
+#include <catch2/catch.hpp>
+
+#include <vcpkg/base/downloads.h>
+
+using namespace vcpkg;
+
+TEST_CASE ("Downloads::details::split_uri_view", "[downloads]")
+{
+    {
+        auto x = Downloads::details::split_uri_view("https://github.com/Microsoft/vcpkg");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "https");
+        REQUIRE(x.get()->authority.value_or("") == "//github.com");
+        REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("hello");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:/path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "/path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file://user:pw@host");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host");
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("ftp://host:port/");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "ftp");
+        REQUIRE(x.get()->authority.value_or({}) == "//host:port");
+        REQUIRE(x.get()->path_query_fragment == "/");
+    }
+}