Configuring overlay ports and overlay triplets through the manifest (#…

…743)

* overlays through manifest implemented

* turned artifacts on

* artifacts settings once more

* fixed formatting

* brought overlays to vcpkgpaths

* artifacts settings

* format

* sending merged overlay triplets

* moved constructor

* passing parameters by reference

* e2e tests

* bad paths test

* fix

* resolves overlay paths relative to vcpkg-configuration.json

* path fix

* Update test config

* modified make_relative_to_manifest

* Fix build

* changed lambda function

* made manifest directory const

* Revert

* Revert lambda function

* Add get_manifest_directory()

* making make_relative_to_manifest cleaner

* Comments

* Comments

Co-authored-by: Valeria Conde <[email protected]>

.gitignore

@@ -22,4 +22,4 @@ node_modules/
 /ce/test/vcpkg-ce.test.build.log
 /ce/common/temp
 /vcpkg-root
-/CMakePresets.json
+/CMakePresets.json

azure-pipelines/e2e_projects/overlays-bad-paths/env-overlays/vcpkg-empty-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_projects/overlays-bad-paths/env-overlays/vcpkg-empty-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "vcpkg-empty-port",
+  "version": "0"
+}

azure-pipelines/e2e_projects/overlays-bad-paths/my-triplets/x64-windows-static.cmake

@@ -0,0 +1,4 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE static)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+

azure-pipelines/e2e_projects/overlays-bad-paths/vcpkg-configuration.json

@@ -0,0 +1,4 @@
+{
+  "overlay-ports": [ "./azure-pipelines/e2e_projects/overlays-project-with-config/config-overlays" ],
+  "overlay-triplets": [ 123 ]
+}

azure-pipelines/e2e_projects/overlays-bad-paths/vcpkg.json

@@ -0,0 +1,5 @@
+{
+	"dependencies": [
+		"vcpkg-empty-port"
+	]
+}

azure-pipelines/e2e_projects/overlays-project-config-embedded/cli-overlays/another-vcpkg-empty-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_projects/overlays-project-config-embedded/cli-overlays/another-vcpkg-empty-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "another-vcpkg-empty-port",
+  "version": "0"
+}

azure-pipelines/e2e_projects/overlays-project-config-embedded/config-overlays/beicode/portfile.cmake

@@ -0,0 +1,21 @@
+vcpkg_from_github(
+  OUT_SOURCE_PATH SOURCE_PATH
+  REPO northwindtraders/beicode
+  REF 19a1f95c2f56a27ced90227b5e2754a602a08e69
+  SHA512 7b2bb7acb2a8ff07bff59cfa27247a7b2cced03828919cd65cc0c8cf1f724f5f1e947ed6992dcdbc913fb470694a52613d1861eaaadbf8903e94eb9cdfe4d000
+  HEAD_REF main
+)
+
+vcpkg_configure_cmake(
+  SOURCE_PATH "${SOURCE_PATH}"
+  PREFER_NINJA
+)
+vcpkg_install_cmake()
+vcpkg_fixup_cmake_targets()
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+
+file(
+  INSTALL "${SOURCE_PATH}/LICENSE"
+  DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}"
+  RENAME copyright)

azure-pipelines/e2e_projects/overlays-project-config-embedded/config-overlays/beicode/vcpkg.json

@@ -0,0 +1,6 @@
+{
+  "name": "beicode",
+  "version": "1.0.0",
+  "description": "A simple utf-8 based unicode decoding and encoding library",
+  "homepage": "https://github.com/northwindtraders/beicode"
+}

azure-pipelines/e2e_projects/overlays-project-config-embedded/env-overlays/vcpkg-empty-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_projects/overlays-project-config-embedded/env-overlays/vcpkg-empty-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "vcpkg-empty-port",
+  "version": "0"
+}

azure-pipelines/e2e_projects/overlays-project-config-embedded/my-triplets/x64-windows-static.cmake

@@ -0,0 +1,4 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE static)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+

azure-pipelines/e2e_projects/overlays-project-config-embedded/vcpkg.json

@@ -0,0 +1,12 @@
+{
+	"dependencies": [
+		"fmt",
+		"vcpkg-empty-port",
+		"another-vcpkg-empty-port",
+		"beicode"
+	],
+	"vcpkg-configuration": {
+		"overlay-ports": [ "./config-overlays" ],
+		"overlay-triplets": [ "./my-triplets" ]
+	}
+}

azure-pipelines/e2e_projects/overlays-project-with-config/cli-overlays/another-vcpkg-empty-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_projects/overlays-project-with-config/cli-overlays/another-vcpkg-empty-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "another-vcpkg-empty-port",
+  "version": "0"
+}

azure-pipelines/e2e_projects/overlays-project-with-config/config-overlays/beicode/portfile.cmake

@@ -0,0 +1,21 @@
+vcpkg_from_github(
+  OUT_SOURCE_PATH SOURCE_PATH
+  REPO northwindtraders/beicode
+  REF 19a1f95c2f56a27ced90227b5e2754a602a08e69
+  SHA512 7b2bb7acb2a8ff07bff59cfa27247a7b2cced03828919cd65cc0c8cf1f724f5f1e947ed6992dcdbc913fb470694a52613d1861eaaadbf8903e94eb9cdfe4d000
+  HEAD_REF main
+)
+
+vcpkg_configure_cmake(
+  SOURCE_PATH "${SOURCE_PATH}"
+  PREFER_NINJA
+)
+vcpkg_install_cmake()
+vcpkg_fixup_cmake_targets()
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+
+file(
+  INSTALL "${SOURCE_PATH}/LICENSE"
+  DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}"
+  RENAME copyright)

azure-pipelines/e2e_projects/overlays-project-with-config/config-overlays/beicode/vcpkg.json

@@ -0,0 +1,6 @@
+{
+  "name": "beicode",
+  "version": "1.0.0",
+  "description": "A simple utf-8 based unicode decoding and encoding library",
+  "homepage": "https://github.com/northwindtraders/beicode"
+}

azure-pipelines/e2e_projects/overlays-project-with-config/env-overlays/vcpkg-empty-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

azure-pipelines/e2e_projects/overlays-project-with-config/env-overlays/vcpkg-empty-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "vcpkg-empty-port",
+  "version": "0"
+}

azure-pipelines/e2e_projects/overlays-project-with-config/my-triplets/x64-windows-static.cmake

@@ -0,0 +1,4 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE static)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+

azure-pipelines/e2e_projects/overlays-project-with-config/vcpkg-configuration.json

@@ -0,0 +1,4 @@
+{
+  "overlay-ports": [ "./config-overlays" ],
+  "overlay-triplets": [ "./my-triplets" ]
+}

azure-pipelines/e2e_projects/overlays-project-with-config/vcpkg.json

@@ -0,0 +1,8 @@
+{
+	"dependencies": [
+		"fmt",
+		"vcpkg-empty-port",
+		"another-vcpkg-empty-port",
+		"beicode"
+	]
+}

azure-pipelines/end-to-end-tests-dir/overlays.ps1

@@ -0,0 +1,30 @@
+. $PSScriptRoot/../end-to-end-tests-prelude.ps1
+
+# Tests a simple project with overlay ports and triplets configured on a vcpkg-configuration.json file
+$e2eProjects = "$PSScriptRoot/../e2e_projects"
+
+$manifestRoot = "$e2eProjects/overlays-project-with-config"
+$env:VCPKG_OVERLAY_PORTS = "$manifestRoot/env-overlays"
+
+Run-Vcpkg install --x-manifest-root=$manifestRoot `
+    --overlay-ports=$manifestRoot/cli-overlays `
+    --overlay-triplets=$manifestRoot/my-triplets `
+    --x-install-root=$installRoot
+Throw-IfFailed
+
+# Tests overlays configured in env and cli on a project with configuration embedded on the manifest file
+$manifestRoot = "$e2eProjects/overlays-project-config-embedded"
+$env:VCPKG_OVERLAY_PORTS = "$manifestRoot/env-overlays"
+Run-Vcpkg install --x-manifest-root=$manifestRoot `
+    --overlay-ports=$manifestRoot/cli-overlays `
+    --overlay-triplets=$manifestRoot/my-triplets `
+    --x-install-root=$installRoot
+Throw-IfFailed
+
+# Config with bad paths
+$manifestRoot = "$e2eProjects/overlays-bad-paths"
+$env:VCPKG_OVERLAY_PORTS = "$manifestRoot/env_overlays"
+Run-Vcpkg install --x-manifest-root=$manifestRoot `
+    --overlay-triplets=$manifestRoot/my-triplets `
+    --x-install-root=$installRoot
+Throw-IfNotFailed

include/vcpkg/configuration.h

@@ -34,6 +34,8 @@ namespace vcpkg
         std::vector<RegistryConfig> registries;
         Json::Object ce_metadata;
         Json::Object extra_info;
+        std::vector<std::string> overlay_ports;
+        std::vector<std::string> overlay_triplets;
 
         Json::Object serialize() const;
         void validate_as_active();

include/vcpkg/vcpkgcmdarguments.h

@@ -158,10 +158,12 @@ namespace vcpkg
         Optional<std::string> host_triplet;
         constexpr static StringLiteral OVERLAY_PORTS_ENV = "VCPKG_OVERLAY_PORTS";
         constexpr static StringLiteral OVERLAY_PORTS_ARG = "overlay-ports";
-        std::vector<std::string> overlay_ports;
+        std::vector<std::string> cli_overlay_ports;
+        std::vector<std::string> env_overlay_ports;
         constexpr static StringLiteral OVERLAY_TRIPLETS_ENV = "VCPKG_OVERLAY_TRIPLETS";
         constexpr static StringLiteral OVERLAY_TRIPLETS_ARG = "overlay-triplets";
-        std::vector<std::string> overlay_triplets;
+        std::vector<std::string> cli_overlay_triplets;
+        std::vector<std::string> env_overlay_triplets;
 
         constexpr static StringLiteral BINARY_SOURCES_ARG = "binarysource";
         std::vector<std::string> binary_sources;

include/vcpkg/vcpkgpaths.h

@@ -122,6 +122,9 @@ namespace vcpkg
         const Path triplets;
         const Path community_triplets;
 
+        std::vector<std::string> overlay_ports;
+        std::vector<std::string> overlay_triplets;
+
         std::string get_toolver_diagnostics() const;
 
         const ToolCache& get_tool_cache() const;

src/vcpkg-test/arguments.cpp

@@ -38,13 +38,13 @@ TEST_CASE ("VcpkgCmdArguments from lowercase argument sequence", "[arguments]")
     REQUIRE(v.print_metrics);
     REQUIRE(*v.print_metrics.get());
 
-    REQUIRE(v.overlay_ports.size() == 2);
-    REQUIRE(v.overlay_ports.at(0) == "C:\\ports1");
-    REQUIRE(v.overlay_ports.at(1) == "C:\\ports2");
+    REQUIRE(v.cli_overlay_ports.size() == 2);
+    REQUIRE(v.cli_overlay_ports.at(0) == "C:\\ports1");
+    REQUIRE(v.cli_overlay_ports.at(1) == "C:\\ports2");
 
-    REQUIRE(v.overlay_triplets.size() == 2);
-    REQUIRE(v.overlay_triplets.at(0) == "C:\\tripletsA");
-    REQUIRE(v.overlay_triplets.at(1) == "C:\\tripletsB");
+    REQUIRE(v.cli_overlay_triplets.size() == 2);
+    REQUIRE(v.cli_overlay_triplets.at(0) == "C:\\tripletsA");
+    REQUIRE(v.cli_overlay_triplets.at(1) == "C:\\tripletsB");
 }
 
 TEST_CASE ("VcpkgCmdArguments from uppercase argument sequence", "[arguments]")
@@ -75,13 +75,13 @@ TEST_CASE ("VcpkgCmdArguments from uppercase argument sequence", "[arguments]")
     REQUIRE(v.print_metrics);
     REQUIRE(*v.print_metrics.get());
 
-    REQUIRE(v.overlay_ports.size() == 2);
-    REQUIRE(v.overlay_ports.at(0) == "C:\\ports1");
-    REQUIRE(v.overlay_ports.at(1) == "C:\\ports2");
+    REQUIRE(v.cli_overlay_ports.size() == 2);
+    REQUIRE(v.cli_overlay_ports.at(0) == "C:\\ports1");
+    REQUIRE(v.cli_overlay_ports.at(1) == "C:\\ports2");
 
-    REQUIRE(v.overlay_triplets.size() == 2);
-    REQUIRE(v.overlay_triplets.at(0) == "C:\\tripletsA");
-    REQUIRE(v.overlay_triplets.at(1) == "C:\\tripletsB");
+    REQUIRE(v.cli_overlay_triplets.size() == 2);
+    REQUIRE(v.cli_overlay_triplets.at(0) == "C:\\tripletsA");
+    REQUIRE(v.cli_overlay_triplets.at(1) == "C:\\tripletsB");
 }
 
 TEST_CASE ("VcpkgCmdArguments from argument sequence with valued options", "[arguments]")

src/vcpkg-test/configmetadata.cpp

@@ -466,6 +466,105 @@ TEST_CASE ("metadata demands", "[ce-metadata]")
 
 TEST_CASE ("serialize configuration", "[ce-metadata]")
 {
+    SECTION ("only overlay ports")
+    {
+        std::string raw = R"json({
+    "overlay-ports": [
+		"./my-ports/fmt",
+		"/custom-ports",
+		"../share/team-ports",
+        "my-ports/fmt"
+	]
+})json";
+        // parsing of configuration is tested elsewhere
+        auto config = parse_test_configuration(raw);
+        Test::check_json_eq(Json::parse_object(raw).value_or_exit(VCPKG_LINE_INFO), config.serialize());
+    }
+
+    SECTION ("invalid overlay ports")
+    {
+        std::string raw = R"json({
+    "overlay-ports": [
+		"./my-ports/fmt" ,
+		"/custom-ports",
+		123
+	]
+})json";
+        check_errors(raw, R"(
+$.overlay-ports[2]: mismatched type: expected an overlay path
+)");
+    }
+
+    SECTION ("only overlay triplets")
+    {
+        std::string raw = R"json({
+    "overlay-triplets": [
+		"./team-triplets"
+	]
+})json";
+        // parsing of configuration is tested elsewhere
+        auto config = parse_test_configuration(raw);
+        Test::check_json_eq(Json::parse_object(raw).value_or_exit(VCPKG_LINE_INFO), config.serialize());
+    }
+
+    SECTION ("invalid overlay triplets")
+    {
+        std::string raw = R"json({
+    "overlay-triplets": [
+		123
+	]
+})json";
+        check_errors(raw, R"(
+$.overlay-triplets[0]: mismatched type: expected a triplet path
+)");
+    }
+
+    SECTION ("both overlay ports and overlay triplets")
+    {
+        std::string raw = R"json({
+    "overlay-ports": [
+		"./my-ports/fmt" ,
+		"/custom-ports",
+		"../share/team-ports"
+	],
+    "overlay-triplets": [
+		"./team-triplets"
+	]
+})json";
+        // parsing of configuration is tested elsewhere
+        auto config = parse_test_configuration(raw);
+        Test::check_json_eq(Json::parse_object(raw).value_or_exit(VCPKG_LINE_INFO), config.serialize());
+    }
+
+    SECTION ("overriden default registry, registries and overlays")
+    {
+        std::string raw = R"json({
+    "default-registry": {
+        "kind": "builtin",
+        "baseline": "843e0ba0d8f9c9c572e45564263eedfc7745e74f"
+    },
+    "registries": [
+        {
+            "kind": "git",
+            "repository": "https://github.com/microsoft/vcpkg",
+            "baseline": "843e0ba0d8f9c9c572e45564263eedfc7745e74f",
+            "packages": [ "zlib" ]
+        }
+    ],
+    "overlay-ports": [
+		"./my-ports/fmt" ,
+		"/custom-ports",
+		"../share/team-ports"
+	],
+    "overlay-triplets": [
+		"./team-triplets"
+	]
+})json";
+        // parsing of configuration is tested elsewhere
+        auto config = parse_test_configuration(raw);
+        Test::check_json_eq(Json::parse_object(raw).value_or_exit(VCPKG_LINE_INFO), config.serialize());
+    }
+
     SECTION ("null default registry")
     {
         std::string raw = R"json({

src/vcpkg.cpp

@@ -118,8 +118,6 @@ static void inner(vcpkg::Filesystem& fs, const VcpkgCmdArguments& args)
         }
     }
 
-    LockGuardPtr<Metrics>(g_metrics)->track_bool_property(BoolMetric::OptionOverlayPorts, !args.overlay_ports.empty());
-
     if (const auto command_function = find_command(Commands::get_available_basic_commands()))
     {
         LockGuardPtr<Metrics>(g_metrics)->track_string_property(StringMetric::CommandName, command_function->name);
@@ -129,6 +127,8 @@ static void inner(vcpkg::Filesystem& fs, const VcpkgCmdArguments& args)
     const VcpkgPaths paths(fs, args);
     paths.track_feature_flag_metrics();
 
+    LockGuardPtr<Metrics>(g_metrics)->track_bool_property(BoolMetric::OptionOverlayPorts, !paths.overlay_ports.empty());
+
     fs.current_path(paths.root, VCPKG_LINE_INFO);
 
     if (const auto command_function = find_command(Commands::get_available_paths_commands()))

src/vcpkg/build.cpp

@@ -189,7 +189,7 @@ namespace vcpkg::Build
         const FullPackageSpec spec = check_and_get_full_package_spec(
             std::move(first_arg), default_triplet, COMMAND_STRUCTURE.example_text, paths);
 
-        PathsPortFileProvider provider(paths, make_overlay_provider(paths, args.overlay_ports));
+        PathsPortFileProvider provider(paths, make_overlay_provider(paths, paths.overlay_ports));
         return perform_ex(args, spec, host_triplet, provider, binary_cache, null_build_logs_recorder(), paths);
     }
 } // namespace vcpkg::Build