diff --git a/docs/users/assetcaching.md b/docs/users/assetcaching.md new file mode 100644 index 00000000000000..425e1158fd7c42 --- /dev/null +++ b/docs/users/assetcaching.md @@ -0,0 +1,43 @@ +# Asset Caching + +**The latest version of this documentation is available on [GitHub](https://github.com/Microsoft/vcpkg/tree/master/docs/users/assetcaching.md).** + +**Experimental feature: this may change or be removed at any time** + +Vcpkg can utilize mirrors to cache downloaded assets, ensuring continued operation even if the original source changes +or disappears. + +In-tool help is available via `vcpkg help assetcaching`. + +## Configuration + +Asset caching can be configured by setting the environment variable `X_VCPKG_ASSET_SOURCES` to a semicolon-delimited +list of source strings. Characters can be escaped using backtick (\`). + +### Valid source strings + +The `` optional parameter for certain strings controls how they will be accessed. It can be specified as `read`, +`write`, or `readwrite` and defaults to `read`. + +#### `clear` + +Syntax: `clear` + +Removes all previous sources + +#### `x-azurl` + +Syntax: `x-azurl,[,[,]]` + +Adds an Azure Blob Storage source, optionally using Shared Access Signature validation. URL should include the container +path and be terminated with a trailing `/`. SAS, if defined, should be prefixed with a `?`. Non-Azure servers will also +work if they respond to GET and PUT requests of the form: ``. + +See also the [binary caching documentation for Azure Blob Storage](binarycaching.md#azure-blob-storage-experimental) for +more information on how to set up an `x-azurl` source. + +#### `x-block-origin` + +Syntax: `x-block-origin` + +Disables use of the original URLs in case the mirror does not have the file available. diff --git a/docs/users/config-environment.md b/docs/users/config-environment.md index b8353e4e80c477..af733e9a31f7fe 100644 --- a/docs/users/config-environment.md +++ b/docs/users/config-environment.md @@ -89,3 +89,9 @@ This environment variable changes the metadata of produced NuGet packages. See [ #### VCPKG_USE_NUGET_CACHE This environment variable allows using NuGet's cache for every nuget-based binary source. See [Binary Caching](binarycaching.md#NuGets-cache) for more details. + +#### X_VCPKG_ASSET_SOURCES + +> Note: This is an experimental feature and may change or be removed at any time + +This environment variable allows using a private mirror for all SHA512-tagged assets. See [Asset Caching](assetcaching.md) for more details. diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index df5cceda6a404e..83ef7e27c30408 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -14,6 +14,9 @@ jobs: value: /mnt/vcpkg-ci - name: VCPKG_DOWNLOADS value: /mnt/vcpkg-ci/downloads + - group: azblob-test-sas-group + - name: X_VCPKG_ASSET_SOURCES + value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite" steps: - bash: df -h diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 6f327c1e9a370d..19612c3cc88ecc 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -17,6 +17,8 @@ jobs: - group: osx-2021-04-16 - name: BINARY_SOURCE_STUB value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)" + - name: X_VCPKG_ASSET_SOURCES + value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite" steps: - bash: df -h diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index c2d74b8bcc4cfa..b4a7f25033f5e2 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -14,6 +14,9 @@ jobs: value: D:\ - name: VCPKG_DOWNLOADS value: D:\downloads + - group: azblob-test-sas-group + - name: X_VCPKG_ASSET_SOURCES + value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite" steps: - task: PowerShell@2 diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index efd858e7fb47cc..9c5509994cd2c5 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -106,10 +106,16 @@ charls:x64-windows=skip charls:x64-windows-static=skip charls:x64-windows-static-md=skip charls:x86-windows=skip +# chartdir does not offer stable download URLs chartdir:arm64-windows=fail chartdir:arm-uwp=fail chartdir:x64-uwp=fail chartdir:x64-windows-static=fail +chartdir:x86-windows=fail +chartdir:x64-windows=fail +chartdir:x64-windows-static-md=fail +chartdir:x64-linux=fail +chartdir:x64-osx=fail chmlib:arm-uwp=fail chmlib:x64-uwp=fail # Chromium Base requires a recent version of Clang to be installed. @@ -1636,6 +1642,8 @@ v8:arm64-windows=fail v8:arm-uwp=fail v8:x64-osx=fail v8:x64-uwp=fail +# The domain hosting vamp-sdk uses a newer root cert than is available in our linux CI +vamp-sdk:x64-linux=fail vectorclass:arm64-windows=fail vectorclass:arm-uwp=fail vlpp:x64-osx=fail diff --git a/scripts/cmake/vcpkg_download_distfile.cmake b/scripts/cmake/vcpkg_download_distfile.cmake index 2e72726b6baf02..5af4345ba465c0 100644 --- a/scripts/cmake/vcpkg_download_distfile.cmake +++ b/scripts/cmake/vcpkg_download_distfile.cmake @@ -67,7 +67,12 @@ function(vcpkg_download_distfile VAR) if(NOT DEFINED vcpkg_download_distfile_FILENAME) message(FATAL_ERROR "vcpkg_download_distfile requires a FILENAME argument.") endif() - if(NOT _VCPKG_INTERNAL_NO_HASH_CHECK) + if(vcpkg_download_distfile_SILENT_EXIT) + message(WARNING "SILENT_EXIT has been deprecated as an argument to vcpkg_download_distfile -- remove the argument to resolve this warning") + endif() + if(_VCPKG_INTERNAL_NO_HASH_CHECK) + set(vcpkg_download_distfile_SKIP_SHA512 1) + else() if(NOT vcpkg_download_distfile_SKIP_SHA512 AND NOT DEFINED vcpkg_download_distfile_SHA512) message(FATAL_ERROR "vcpkg_download_distfile requires a SHA512 argument. If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.") endif() @@ -113,7 +118,7 @@ function(vcpkg_download_distfile VAR) if(EXISTS "${downloaded_file_path}") if(NOT vcpkg_download_distfile_QUIET) - message(STATUS "Using cached ${downloaded_file_path}") + message(STATUS "Using ${downloaded_file_path}") endif() test_hash("${downloaded_file_path}" "cached file" "Please delete the file and retry if this file should be downloaded again.") else() @@ -127,7 +132,7 @@ function(vcpkg_download_distfile VAR) vcpkg_find_acquire_program("ARIA2") message(STATUS "Downloading ${vcpkg_download_distfile_FILENAME}...") if(vcpkg_download_distfile_HEADERS) - foreach(header ${vcpkg_download_distfile_HEADERS}) + foreach(header IN LISTS vcpkg_download_distfile_HEADERS) list(APPEND request_headers "--header=${header}") endforeach() endif() @@ -159,29 +164,60 @@ function(vcpkg_download_distfile VAR) ) set(download_success 1) endif() - else() + elseif(vcpkg_download_distfile_SKIP_SHA512) + set(download_success 0) + set(request_headers) + if(vcpkg_download_distfile_HEADERS) + foreach(header IN LISTS vcpkg_download_distfile_HEADERS) + list(APPEND request_headers HTTPHEADER ${header}) + endforeach() + endif() foreach(url IN LISTS vcpkg_download_distfile_URLS) message(STATUS "Downloading ${url} -> ${vcpkg_download_distfile_FILENAME}...") - if(vcpkg_download_distfile_HEADERS) - foreach(header ${vcpkg_download_distfile_HEADERS}) - list(APPEND request_headers HTTPHEADER ${header}) - endforeach() - endif() file(DOWNLOAD "${url}" "${download_file_path_part}" STATUS download_status ${request_headers}) list(GET download_status 0 status_code) if (NOT "${status_code}" STREQUAL "0") message(STATUS "Downloading ${url}... Failed. Status: ${download_status}") - set(download_success 0) else() + test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n") + get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) + file(MAKE_DIRECTORY "${downloaded_file_dir}") + file(RENAME ${download_file_path_part} ${downloaded_file_path}) set(download_success 1) break() endif() endforeach(url) + else() + set(urls) + foreach(url IN LISTS vcpkg_download_distfile_URLS) + list(APPEND urls "--url=${url}") + endforeach() + if(NOT vcpkg_download_distfile_QUIET) + message(STATUS "Downloading ${vcpkg_download_distfile_URLS} -> ${vcpkg_download_distfile_FILENAME}...") + endif() + set(request_headers) + if(vcpkg_download_distfile_HEADERS) + foreach(header IN LISTS vcpkg_download_distfile_HEADERS) + list(APPEND request_headers "--header=${header}") + endforeach() + endif() + vcpkg_execute_in_download_mode( + COMMAND "$ENV{VCPKG_COMMAND}" x-download "${downloaded_file_path}" "${vcpkg_download_distfile_SHA512}" ${urls} ${request_headers} --debug + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE failure + WORKING_DIRECTORY "${DOWNLOADS}" + ) + if(failure) + message("${output}") + set(download_success 0) + else() + set(download_success 1) + endif() endif() - if (NOT vcpkg_download_distfile_SILENT_EXIT) - if (NOT download_success) - message(FATAL_ERROR + if(NOT download_success) + message(FATAL_ERROR " \n" " Failed to download file.\n" " If you use a proxy, please check your proxy setting. Possible causes are:\n" @@ -205,23 +241,6 @@ function(vcpkg_download_distfile VAR) " and (https://github.com/microsoft/vcpkg-tool/pull/77)\n" " \n" " Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n") - else() - test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n") - get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) - file(MAKE_DIRECTORY "${downloaded_file_dir}") - file(RENAME ${download_file_path_part} ${downloaded_file_path}) - endif() - else() - if (NOT download_success) - message(WARNING - " \n" - " Failed to download file.\n") - else() - test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n") - get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) - file(MAKE_DIRECTORY "${downloaded_file_dir}") - file(RENAME ${download_file_path_part} ${downloaded_file_path}) - endif() endif() endif() set(${VAR} ${downloaded_file_path} PARENT_SCOPE) diff --git a/scripts/cmake/vcpkg_from_sourceforge.cmake b/scripts/cmake/vcpkg_from_sourceforge.cmake index 82286b346d02f8..de0e6c509ed9b0 100644 --- a/scripts/cmake/vcpkg_from_sourceforge.cmake +++ b/scripts/cmake/vcpkg_from_sourceforge.cmake @@ -67,34 +67,6 @@ Specifies that the default removal of the top level folder should not occur. #]===] function(vcpkg_from_sourceforge) - macro(check_file_content) - if (EXISTS ${ARCHIVE}) - file(SIZE ${ARCHIVE} DOWNLOAD_FILE_SIZE) - if (DOWNLOAD_FILE_SIZE LESS_EQUAL 1024) - file(READ ${ARCHIVE} _FILE_CONTENT_) - string(FIND "${_FILE_CONTENT_}" "the Sourceforge site is currently in Disaster Recovery mode." OUT_CONTENT) - message("OUT_CONTENT: ${OUT_CONTENT}") - if (OUT_CONTENT EQUAL -1) - set(download_success 1) - else() - file(REMOVE ${ARCHIVE}) - endif() - endif() - endif() - endmacro() - - macro(check_file_sha512) - file(SHA512 ${ARCHIVE} FILE_HASH) - if(NOT FILE_HASH STREQUAL _vdus_SHA512) - message(FATAL_ERROR - "\nFile does not have expected hash:\n" - " File path: [ ${ARCHIVE} ]\n" - " Expected hash: [ ${_vdus_SHA512} ]\n" - " Actual hash: [ ${FILE_HASH} ]\n" - "${CUSTOM_ERROR_ADVICE}\n") - endif() - endmacro() - set(booleanValueArgs DISABLE_SSL NO_REMOVE_ONE_LEVEL) set(oneValueArgs OUT_SOURCE_PATH REPO REF SHA512 FILENAME WORKING_DIRECTORY) set(multipleValuesArgs PATCHES) @@ -154,7 +126,8 @@ function(vcpkg_from_sourceforge) string(SUBSTRING "${_vdus_SHA512}" 0 10 SANITIZED_REF) - list(APPEND SOURCEFORGE_MIRRORS + set(Z_VCPKG_SOURCEFORGE_MIRRORS ${SOURCEFORGE_MIRRORS}) + list(APPEND Z_VCPKG_SOURCEFORGE_MIRRORS cfhcable # United States pilotfiber # New York, NY gigenet # Chicago, IL @@ -176,56 +149,18 @@ function(vcpkg_from_sourceforge) ufpr # Curitiba, Brazil tenet # Wynberg, South Africa ) - - # Try to use auto-select first - set(DOWNLOAD_URL ${URL}/download) - message(STATUS "Trying auto-select mirror...") + + set(URLS "${URL}/download") + foreach(SOURCEFORGE_MIRROR IN LISTS Z_VCPKG_SOURCEFORGE_MIRRORS) + list(APPEND URLS "${URL}/download?use_mirror=${SOURCEFORGE_MIRROR}") + endforeach() + vcpkg_download_distfile(ARCHIVE - URLS "${DOWNLOAD_URL}" - SKIP_SHA512 + URLS ${URLS} + SHA512 "${_vdus_SHA512}" FILENAME "${_vdus_FILENAME}" - SILENT_EXIT ) - check_file_content() - if (download_success) - check_file_sha512() - else() - message(STATUS "The default mirror is in Disaster Recovery mode, trying other mirrors...") - endif() - - if (NOT download_success EQUAL 1) - foreach(SOURCEFORGE_MIRROR ${SOURCEFORGE_MIRRORS}) - set(DOWNLOAD_URL ${URL}/download?use_mirror=${SOURCEFORGE_MIRROR}) - message(STATUS "Trying mirror ${SOURCEFORGE_MIRROR}...") - vcpkg_download_distfile(ARCHIVE - URLS "${DOWNLOAD_URL}" - SKIP_SHA512 - FILENAME "${_vdus_FILENAME}" - SILENT_EXIT - ) - - if (EXISTS ${ARCHIVE}) - set(download_success 1) - check_file_content() - if (download_success) - check_file_sha512() - else() - message(STATUS "Mirror ${SOURCEFORGE_MIRROR} is in Disaster Recovery mode, trying other mirrors...") - endif() - break() - endif() - endforeach() - endif() - if (NOT download_success) - message(FATAL_ERROR [[ - Couldn't download source from any of the sourceforge mirrors, please check your network. - If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment - variables to "http[s]://user:password@your-proxy-ip-address:port/". - Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues - ]]) - endif() - vcpkg_extract_source_archive_ex( OUT_SOURCE_PATH SOURCE_PATH ARCHIVE "${ARCHIVE}"