[vcpkg] Add experimental x-azblob binary provider (#13626)

* [vcpkg] Add experimental x-azblob binary provider

* [vcpkg] Test azblob storage provider in CI

* [vcpkg] Address some CR comments from #13639

* [vcpkg] Fixup azure-pipelines

* [vcpkg] Fix regression where the downloaded package is purged before decompressing

* [vcpkg] Further refactor vcpkg::Downloads

* [vcpkg] Enable OSX for x-azblob testing

* [vcpkg] Reduce diff against master

* [vcpkg] Extract Downloads::details::split_uri_view

* [vcpkg] Address PR comments

* [vcpkg] Add testing and metrics for x-azblob

* [vcpkg] Add docs for x-azblob

This includes a note that it is currently experimental

* [vcpkg] Address CR comments

* [vcpkg] Revert pipeline changes except OSX to minimize disruption

Co-authored-by: Robert Schumacher <[email protected]>
Co-authored-by: Billy Robert O'Neal III <[email protected]>

docs/users/binarycaching.md

@@ -108,7 +108,8 @@ By default, zip-based archives will be cached at the first valid location of:
 | `default[,<rw>]`            | Adds the default file-based location
 | `files,<path>[,<rw>]`       | Adds a custom file-based location
 | `nuget,<uri>[,<rw>]`        | Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI
-| `nugetconfig,<path>[,<rw>]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter <br>of the NuGet CLI. This config should specify `defaultPushSource` for uploads.
+| `nugetconfig,<path>[,<rw>]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This config should specify `defaultPushSource` for uploads.
+| `x-azblob,<baseuri>,<sas>[,<rw>]`    | **Experimental: will change or be removed without warning**<br> Adds an Azure Blob Storage source. Uses Shared Access Signature validation. URL should include the container path.
 | `interactive`               | Enables interactive credential management for NuGet (for debugging; requires `--debug` on the command line)
 
 The `<rw>` optional parameter for certain sources controls whether they will be consulted for

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -14,6 +14,9 @@ jobs:
     value: /Users/vagrant/Data
   - name: VCPKG_DOWNLOADS
     value: /Users/vagrant/Data/downloads
+  - group: azblob-test-sas-group
+  - name: BINARY_SOURCE_STUB
+    value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"
 
   steps:
   - bash: df -h
@@ -40,7 +43,7 @@ jobs:
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -BinarySourceStub "$(BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/test-modified-ports.ps1

@@ -16,16 +16,19 @@ The location used as scratch space for 'installed', 'packages', and 'buildtrees'
 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
 
 .PARAMETER ArchivesRoot
-The location where the binary caching archives are stored. Shared across runs of this script. If
-this parameter is not set, binary caching will not be used.
+Equivalent to '-BinarySourceStub "files,$ArchivesRoot"'
+
+.PARAMETER BinarySourceStub
+The type and parameters of the binary source. Shared across runs of this script. If
+this parameter is not set, binary caching will not be used. Example: "files,W:\"
 
 .PARAMETER BuildReason
 The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).
 If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set and this is not,
 binary caching will default to read-write mode.
 #>
 
-[CmdletBinding()]
+[CmdletBinding(DefaultParameterSetName="ArchivesRoot")]
 Param(
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
@@ -35,7 +38,10 @@ Param(
     $WorkingRoot,
     [ValidateNotNullOrEmpty()]
     $ArtifactStagingDirectory = '.',
+    [Parameter(ParameterSetName='ArchivesRoot')]
     $ArchivesRoot = $null,
+    [Parameter(ParameterSetName='BinarySourceStub')]
+    $BinarySourceStub = $null,
     $BuildReason = $null
 )
 
@@ -49,7 +55,7 @@ $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'packages'
 
-$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot))
+$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or -Not ([string]::IsNullOrWhiteSpace($BinarySourceStub))
 $commonArgs = @()
 if ($usingBinaryCaching) {
     $commonArgs += @('--binarycaching')
@@ -78,8 +84,12 @@ if ($usingBinaryCaching) {
         Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
         $binaryCachingMode = 'write'
     }
-
-    $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+    if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) {
+        $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
+    }
+    else {
+        $commonArgs += @("--binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+    }
 }
 
 if ($Triplet -eq 'x64-linux') {

toolsrc/include/vcpkg/base/downloads.h

@@ -1,16 +1,41 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/optional.h>
+#include <vcpkg/base/view.h>
 
 namespace vcpkg::Downloads
 {
+    namespace details
+    {
+        struct SplitURIView
+        {
+            StringView scheme;
+            Optional<StringView> authority;
+            StringView path_query_fragment;
+        };
+
+        // e.g. {"https","//example.org", "/index.html"}
+        Optional<SplitURIView> split_uri_view(StringView uri);
+    }
+
     void verify_downloaded_file_hash(const Files::Filesystem& fs,
                                      const std::string& url,
                                      const fs::path& path,
                                      const std::string& sha512);
 
+    // Returns url that was successfully downloaded from
+    std::string download_file(Files::Filesystem& fs,
+                              View<std::string> urls,
+                              const fs::path& download_path,
+                              const std::string& sha512);
+
     void download_file(Files::Filesystem& fs,
                        const std::string& url,
                        const fs::path& download_path,
                        const std::string& sha512);
+
+    std::vector<int> download_files(Files::Filesystem& fs, View<std::pair<std::string, fs::path>> url_pairs);
+    int put_file(const Files::Filesystem&, StringView url, const fs::path& file);
+    std::vector<int> url_heads(View<std::string> urls);
 }

toolsrc/include/vcpkg/base/fwd/lockguarded.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuardPtr;
+
+    template<class T>
+    struct LockGuarded;
+}

toolsrc/include/vcpkg/base/lockguarded.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vcpkg/base/fwd/lockguarded.h>
+
+#include <mutex>
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuarded
+    {
+        friend struct LockGuardPtr<T>;
+
+        LockGuardPtr<T> lock() { return *this; }
+
+    private:
+        std::mutex m_mutex;
+        T m_t;
+    };
+
+    template<class T>
+    struct LockGuardPtr
+    {
+        T& operator*() { return m_ptr; }
+        T* operator->() { return &m_ptr; }
+
+        T* get() { return &m_ptr; }
+
+        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
+
+    private:
+        std::lock_guard<std::mutex> m_lock;
+        T& m_ptr;
+    };
+}

toolsrc/include/vcpkg/base/util.h

@@ -5,7 +5,6 @@
 #include <algorithm>
 #include <functional>
 #include <map>
-#include <mutex>
 #include <type_traits>
 #include <unordered_map>
 #include <utility>
@@ -222,36 +221,6 @@ namespace vcpkg::Util
         ~ResourceBase() = default;
     };
 
-    template<class T>
-    struct LockGuardPtr;
-
-    template<class T>
-    struct LockGuarded
-    {
-        friend struct LockGuardPtr<T>;
-
-        LockGuardPtr<T> lock() { return *this; }
-
-    private:
-        std::mutex m_mutex;
-        T m_t;
-    };
-
-    template<class T>
-    struct LockGuardPtr
-    {
-        T& operator*() { return m_ptr; }
-        T* operator->() { return &m_ptr; }
-
-        T* get() { return &m_ptr; }
-
-        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
-
-    private:
-        std::unique_lock<std::mutex> m_lock;
-        T& m_ptr;
-    };
-
     namespace Enum
     {
         template<class E>

toolsrc/include/vcpkg/binarycaching.h

@@ -8,8 +8,12 @@
 
 #include <vcpkg/packagespec.h>
 
+#include <unordered_map>
+
 namespace vcpkg
 {
+    struct MergeBinaryProviders;
+
     enum class RestoreResult
     {
         missing,
@@ -20,18 +24,32 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
-        /// `plan`.
-        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+
         /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
         /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
-        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
-        /// missing packages.
-        virtual RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
+        /// <summary>Gives the BinaryProvider an opportunity to batch any downloading or server communication for
+        /// executing `plan`.</summary>
+        /// <remarks>Must only be called once for a given binary provider instance</remarks>
+        /// <param name="actions">InOut vector of actions to be prefetched</param>
+        virtual void prefetch(const VcpkgPaths& paths,
+                              std::vector<const Dependencies::InstallPlanAction*>& actions) = 0;
+
+        /// <summary>Requests the result of <c>try_restore()</c> without actually downloading the package. Used by CI to
+        /// determine missing packages.</summary>
+        /// <param name="results_map">InOut map to track the restored packages. Should be initialized to
+        /// <c>{&amp;action, RestoreResult::missing}</c> for all install actions</param>
+        virtual void precheck(
+            const VcpkgPaths& paths,
+            std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) = 0;
     };
 
+    std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> binary_provider_precheck(
+        const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider);
+
     IBinaryProvider& null_binary_provider();
 
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(View<std::string> args);

toolsrc/include/vcpkg/dependencies.h

@@ -61,6 +61,7 @@ namespace vcpkg::Dependencies
         std::string displayname() const;
         const std::string& public_abi() const;
         bool has_package_abi() const;
+        Optional<const std::string&> package_abi() const;
         const Build::PreBuildInfo& pre_build_info(LineInfo linfo) const;
 
         PackageSpec spec;

toolsrc/include/vcpkg/globalstate.h

@@ -1,7 +1,8 @@
 #pragma once
 
+#include <vcpkg/base/fwd/lockguarded.h>
+
 #include <vcpkg/base/chrono.h>
-#include <vcpkg/base/util.h>
 
 #include <atomic>
 #include <string>

toolsrc/include/vcpkg/metrics.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/lockguarded.h>
 #include <vcpkg/base/util.h>
 
 #include <string>

toolsrc/src/vcpkg-test/binaryconfigparser.cpp

@@ -298,3 +298,39 @@ TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]")
         REQUIRE(parsed.has_value());
     }
 }
+
+TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,?sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,,sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,invalid", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,read", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,write", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,readwrite", {});
+        REQUIRE(parsed.has_value());
+    }
+}

toolsrc/src/vcpkg-test/downloads.cpp

@@ -0,0 +1,59 @@
+#include <catch2/catch.hpp>
+
+#include <vcpkg/base/downloads.h>
+
+using namespace vcpkg;
+
+TEST_CASE ("Downloads::details::split_uri_view", "[downloads]")
+{
+    {
+        auto x = Downloads::details::split_uri_view("https://github.com/Microsoft/vcpkg");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "https");
+        REQUIRE(x.get()->authority.value_or("") == "//github.com");
+        REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("hello");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:/path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "/path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file://user:pw@host");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host");
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("ftp://host:port/");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "ftp");
+        REQUIRE(x.get()->authority.value_or({}) == "//host:port");
+        REQUIRE(x.get()->path_query_fragment == "/");
+    }
+}