[vcpkg] Add experimental $X_VCPKG_ASSET_SOURCES for source caching (#13639)

* [vcpkg] Add experimental $VCPKG_X_READWRITE_MIRROR_URL_TEMPLATE for source caching

* [vcpkg] Use prerelease version for testing

* [docs] Add assetcaching.md

* Revert bootstrap changes

* [vcpkg_download_distfile] Deprecate SILENT_EXIT

* cmake_parse_arguments() always defines option arguments

* Adjust CI baseline

* [vcpkg_download_distfile] Address CR comments, restore ARIA2

* [CI] Add source caching to Linux and OSX CI passes

* [ci.baseline.txt] Skip chartdir on more platforms

8 files changed, 124 insertions, 105 deletions

diff --git a/docs/users/assetcaching.md b/docs/users/assetcaching.md
new file mode 100644
index 000000000..425e1158f
--- /dev/null
+++ b/docs/users/assetcaching.md
@@ -0,0 +1,43 @@
+# Asset Caching
+
+**The latest version of this documentation is available on [GitHub](https://github.com/Microsoft/vcpkg/tree/master/docs/users/assetcaching.md).**
+
+**Experimental feature: this may change or be removed at any time**
+
+Vcpkg can utilize mirrors to cache downloaded assets, ensuring continued operation even if the original source changes
+or disappears.
+
+In-tool help is available via `vcpkg help assetcaching`.
+
+## Configuration
+
+Asset caching can be configured by setting the environment variable `X_VCPKG_ASSET_SOURCES` to a semicolon-delimited
+list of source strings. Characters can be escaped using backtick (\`).
+
+### Valid source strings
+
+The `<rw>` optional parameter for certain strings controls how they will be accessed. It can be specified as `read`,
+`write`, or `readwrite` and defaults to `read`.
+
+#### `clear`
+
+Syntax: `clear`
+
+Removes all previous sources
+
+#### `x-azurl`
+
+Syntax: `x-azurl,<url>[,<sas>[,<rw>]]`
+
+Adds an Azure Blob Storage source, optionally using Shared Access Signature validation. URL should include the container
+path and be terminated with a trailing `/`. SAS, if defined, should be prefixed with a `?`. Non-Azure servers will also
+work if they respond to GET and PUT requests of the form: `<url><sha512><sas>`.
+
+See also the [binary caching documentation for Azure Blob Storage](binarycaching.md#azure-blob-storage-experimental) for
+more information on how to set up an `x-azurl` source.
+
+#### `x-block-origin`
+
+Syntax: `x-block-origin`
+
+Disables use of the original URLs in case the mirror does not have the file available.
diff --git a/docs/users/config-environment.md b/docs/users/config-environment.md
index 132f84fb6..73dd64a21 100644
--- a/docs/users/config-environment.md
+++ b/docs/users/config-environment.md
@@ -87,3 +87,9 @@ This environment variable changes the metadata of produced NuGet packages. See [
 #### VCPKG_USE_NUGET_CACHE

 

 This environment variable allows using NuGet's cache for every nuget-based binary source. See [Binary Caching](binarycaching.md#NuGets-cache) for more details.

+

+#### X_VCPKG_ASSET_SOURCES

+

+> Note: This is an experimental feature and may change or be removed at any time

+

+This environment variable allows using a private mirror for all SHA512-tagged assets. See [Asset Caching](assetcaching.md) for more details.

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index df5cceda6..83ef7e27c 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -14,6 +14,9 @@ jobs:
     value: /mnt/vcpkg-ci

   - name: VCPKG_DOWNLOADS

     value: /mnt/vcpkg-ci/downloads

+  - group: azblob-test-sas-group

+  - name: X_VCPKG_ASSET_SOURCES

+    value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"

 

   steps:

   - bash: df -h

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index 6f327c1e9..19612c3cc 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -17,6 +17,8 @@ jobs:
   - group: osx-2021-04-16

   - name: BINARY_SOURCE_STUB

     value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"

+  - name: X_VCPKG_ASSET_SOURCES

+    value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"

 

   steps:

   - bash: df -h

diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index f7d52656a..c6aefcd2b 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -14,6 +14,9 @@ jobs:
     value: D:\

   - name: VCPKG_DOWNLOADS

     value: D:\downloads

+  - group: azblob-test-sas-group

+  - name: X_VCPKG_ASSET_SOURCES

+    value: "x-azurl,$(azblob-root-url),$(azblob-test-sas),readwrite"

 

   steps:

   - task: PowerShell@2

diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
index bc7e0336f..c7839e882 100644
--- a/scripts/ci.baseline.txt
+++ b/scripts/ci.baseline.txt
@@ -106,10 +106,16 @@ charls:x64-windows=skip
 charls:x64-windows-static=skip
 charls:x64-windows-static-md=skip
 charls:x86-windows=skip
+# chartdir does not offer stable download URLs
 chartdir:arm64-windows=fail
 chartdir:arm-uwp=fail
 chartdir:x64-uwp=fail
 chartdir:x64-windows-static=fail
+chartdir:x86-windows=fail
+chartdir:x64-windows=fail
+chartdir:x64-windows-static-md=fail
+chartdir:x64-linux=fail
+chartdir:x64-osx=fail
 chmlib:arm-uwp=fail
 chmlib:x64-uwp=fail
 # Chromium Base requires a recent version of Clang to be installed.
@@ -1625,6 +1631,8 @@ v8:arm64-windows=fail
 v8:arm-uwp=fail
 v8:x64-osx=fail
 v8:x64-uwp=fail
+# The domain hosting vamp-sdk uses a newer root cert than is available in our linux CI
+vamp-sdk:x64-linux=fail
 vectorclass:arm64-windows=fail
 vectorclass:arm-uwp=fail
 vlpp:x64-osx=fail
diff --git a/scripts/cmake/vcpkg_download_distfile.cmake b/scripts/cmake/vcpkg_download_distfile.cmake
index 2e72726b6..5af4345ba 100644
--- a/scripts/cmake/vcpkg_download_distfile.cmake
+++ b/scripts/cmake/vcpkg_download_distfile.cmake
@@ -67,7 +67,12 @@ function(vcpkg_download_distfile VAR)
     if(NOT DEFINED vcpkg_download_distfile_FILENAME)
         message(FATAL_ERROR "vcpkg_download_distfile requires a FILENAME argument.")
     endif()
-    if(NOT _VCPKG_INTERNAL_NO_HASH_CHECK)
+    if(vcpkg_download_distfile_SILENT_EXIT)
+        message(WARNING "SILENT_EXIT has been deprecated as an argument to vcpkg_download_distfile -- remove the argument to resolve this warning")
+    endif()
+    if(_VCPKG_INTERNAL_NO_HASH_CHECK)
+        set(vcpkg_download_distfile_SKIP_SHA512 1)
+    else()
         if(NOT vcpkg_download_distfile_SKIP_SHA512 AND NOT DEFINED vcpkg_download_distfile_SHA512)
             message(FATAL_ERROR "vcpkg_download_distfile requires a SHA512 argument. If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.")
         endif()
@@ -113,7 +118,7 @@ function(vcpkg_download_distfile VAR)
 
     if(EXISTS "${downloaded_file_path}")
         if(NOT vcpkg_download_distfile_QUIET)
-            message(STATUS "Using cached ${downloaded_file_path}")
+            message(STATUS "Using ${downloaded_file_path}")
         endif()
         test_hash("${downloaded_file_path}" "cached file" "Please delete the file and retry if this file should be downloaded again.")
     else()
@@ -127,7 +132,7 @@ function(vcpkg_download_distfile VAR)
             vcpkg_find_acquire_program("ARIA2")
             message(STATUS "Downloading ${vcpkg_download_distfile_FILENAME}...")
             if(vcpkg_download_distfile_HEADERS)
-                foreach(header ${vcpkg_download_distfile_HEADERS})
+                foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
                     list(APPEND request_headers "--header=${header}")
                 endforeach()
             endif()
@@ -159,29 +164,60 @@ function(vcpkg_download_distfile VAR)
                 )
                 set(download_success 1)
             endif()
-        else()
+        elseif(vcpkg_download_distfile_SKIP_SHA512)
+            set(download_success 0)
+            set(request_headers)
+            if(vcpkg_download_distfile_HEADERS)
+                foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
+                    list(APPEND request_headers HTTPHEADER ${header})
+                endforeach()
+            endif()
             foreach(url IN LISTS vcpkg_download_distfile_URLS)
                 message(STATUS "Downloading ${url} -> ${vcpkg_download_distfile_FILENAME}...")
-                if(vcpkg_download_distfile_HEADERS)
-                    foreach(header ${vcpkg_download_distfile_HEADERS})
-                        list(APPEND request_headers HTTPHEADER ${header})
-                    endforeach()
-                endif()
                 file(DOWNLOAD "${url}" "${download_file_path_part}" STATUS download_status ${request_headers})
                 list(GET download_status 0 status_code)
                 if (NOT "${status_code}" STREQUAL "0")
                     message(STATUS "Downloading ${url}... Failed. Status: ${download_status}")
-                    set(download_success 0)
                 else()
+                    test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
+                    get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
+                    file(MAKE_DIRECTORY "${downloaded_file_dir}")
+                    file(RENAME ${download_file_path_part} ${downloaded_file_path})
                     set(download_success 1)
                     break()
                 endif()
             endforeach(url)
+        else()
+            set(urls)
+            foreach(url IN LISTS vcpkg_download_distfile_URLS)
+                list(APPEND urls "--url=${url}")
+            endforeach()
+            if(NOT vcpkg_download_distfile_QUIET)
+                message(STATUS "Downloading ${vcpkg_download_distfile_URLS} -> ${vcpkg_download_distfile_FILENAME}...")
+            endif()
+            set(request_headers)
+            if(vcpkg_download_distfile_HEADERS)
+                foreach(header IN LISTS vcpkg_download_distfile_HEADERS)
+                    list(APPEND request_headers "--header=${header}")
+                endforeach()
+            endif()
+            vcpkg_execute_in_download_mode(
+                COMMAND "$ENV{VCPKG_COMMAND}" x-download "${downloaded_file_path}" "${vcpkg_download_distfile_SHA512}" ${urls} ${request_headers} --debug
+                OUTPUT_VARIABLE output
+                ERROR_VARIABLE output
+                RESULT_VARIABLE failure
+                WORKING_DIRECTORY "${DOWNLOADS}"
+            )
+            if(failure)
+                message("${output}")
+                set(download_success 0)
+            else()
+                set(download_success 1)
+            endif()
         endif()
 
-        if (NOT vcpkg_download_distfile_SILENT_EXIT)
-            if (NOT download_success)
-                message(FATAL_ERROR
+        if(NOT download_success)
+            message(FATAL_ERROR
                 "    \n"
                 "    Failed to download file.\n"  
                 "    If you use a proxy, please check your proxy setting. Possible causes are:\n"
@@ -205,23 +241,6 @@ function(vcpkg_download_distfile VAR)
                 "    and (https://github.com/microsoft/vcpkg-tool/pull/77)\n"
                 "    \n"
                 "    Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n")
-            else()
-                test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
-                get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
-                file(MAKE_DIRECTORY "${downloaded_file_dir}")
-                file(RENAME ${download_file_path_part} ${downloaded_file_path})
-            endif()
-        else()
-            if (NOT download_success)
-                message(WARNING
-                "    \n"
-                "    Failed to download file.\n")
-            else()
-                test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n")
-                get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY)
-                file(MAKE_DIRECTORY "${downloaded_file_dir}")
-                file(RENAME ${download_file_path_part} ${downloaded_file_path})
-            endif()
         endif()
     endif()
     set(${VAR} ${downloaded_file_path} PARENT_SCOPE)
diff --git a/scripts/cmake/vcpkg_from_sourceforge.cmake b/scripts/cmake/vcpkg_from_sourceforge.cmake
index 82286b346..de0e6c509 100644
--- a/scripts/cmake/vcpkg_from_sourceforge.cmake
+++ b/scripts/cmake/vcpkg_from_sourceforge.cmake
@@ -67,34 +67,6 @@ Specifies that the default removal of the top level folder should not occur.
 #]===]
 
 function(vcpkg_from_sourceforge)
-    macro(check_file_content)
-        if (EXISTS ${ARCHIVE})
-            file(SIZE ${ARCHIVE} DOWNLOAD_FILE_SIZE)
-            if (DOWNLOAD_FILE_SIZE LESS_EQUAL 1024)
-                file(READ ${ARCHIVE} _FILE_CONTENT_)
-                string(FIND "${_FILE_CONTENT_}" "the Sourceforge site is currently in Disaster Recovery mode." OUT_CONTENT)
-                message("OUT_CONTENT: ${OUT_CONTENT}")
-                if (OUT_CONTENT EQUAL -1)
-                    set(download_success 1)
-                else()
-                    file(REMOVE ${ARCHIVE})
-                endif()
-            endif()
-        endif()
-    endmacro()
-    
-    macro(check_file_sha512)
-        file(SHA512 ${ARCHIVE} FILE_HASH)
-        if(NOT FILE_HASH STREQUAL _vdus_SHA512)
-            message(FATAL_ERROR
-                "\nFile does not have expected hash:\n"
-                "        File path: [ ${ARCHIVE} ]\n"
-                "    Expected hash: [ ${_vdus_SHA512} ]\n"
-                "      Actual hash: [ ${FILE_HASH} ]\n"
-                "${CUSTOM_ERROR_ADVICE}\n")
-        endif()
-    endmacro()
-    
     set(booleanValueArgs DISABLE_SSL NO_REMOVE_ONE_LEVEL)
     set(oneValueArgs OUT_SOURCE_PATH REPO REF SHA512 FILENAME WORKING_DIRECTORY)
     set(multipleValuesArgs PATCHES)
@@ -154,7 +126,8 @@ function(vcpkg_from_sourceforge)
 
     string(SUBSTRING "${_vdus_SHA512}" 0 10 SANITIZED_REF)
 
-    list(APPEND SOURCEFORGE_MIRRORS
+    set(Z_VCPKG_SOURCEFORGE_MIRRORS ${SOURCEFORGE_MIRRORS})
+    list(APPEND Z_VCPKG_SOURCEFORGE_MIRRORS
         cfhcable        # United States
         pilotfiber      # New York, NY
         gigenet         # Chicago, IL
@@ -176,56 +149,18 @@ function(vcpkg_from_sourceforge)
         ufpr            # Curitiba, Brazil
         tenet           # Wynberg, South Africa
     )
-    
-    # Try to use auto-select first
-    set(DOWNLOAD_URL ${URL}/download)
-    message(STATUS "Trying auto-select mirror...")
+
+    set(URLS "${URL}/download")
+    foreach(SOURCEFORGE_MIRROR IN LISTS Z_VCPKG_SOURCEFORGE_MIRRORS)
+        list(APPEND URLS "${URL}/download?use_mirror=${SOURCEFORGE_MIRROR}")
+    endforeach()
+
     vcpkg_download_distfile(ARCHIVE
-        URLS "${DOWNLOAD_URL}"
-        SKIP_SHA512
+        URLS ${URLS}
+        SHA512 "${_vdus_SHA512}"
         FILENAME "${_vdus_FILENAME}"
-        SILENT_EXIT
     )
-    check_file_content()
-    if (download_success)
-        check_file_sha512()
-    else()
-        message(STATUS "The default mirror is in Disaster Recovery mode, trying other mirrors...")
-    endif()
-    
-    if (NOT download_success EQUAL 1)
-        foreach(SOURCEFORGE_MIRROR ${SOURCEFORGE_MIRRORS})
-            set(DOWNLOAD_URL ${URL}/download?use_mirror=${SOURCEFORGE_MIRROR})
-            message(STATUS "Trying mirror ${SOURCEFORGE_MIRROR}...")
-            vcpkg_download_distfile(ARCHIVE
-                URLS "${DOWNLOAD_URL}"
-                SKIP_SHA512
-                FILENAME "${_vdus_FILENAME}"
-                SILENT_EXIT
-            )
-            
-            if (EXISTS ${ARCHIVE})
-                set(download_success 1)
-                check_file_content()
-                if (download_success)
-                    check_file_sha512()
-                else()
-                    message(STATUS "Mirror ${SOURCEFORGE_MIRROR} is in Disaster Recovery mode, trying other mirrors...")
-                endif()
-                break()
-            endif()
-        endforeach()
-    endif()
 
-    if (NOT download_success)
-        message(FATAL_ERROR [[
-            Couldn't download source from any of the sourceforge mirrors, please check your network.
-            If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment
-            variables to "http[s]://user:password@your-proxy-ip-address:port/".
-            Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues
-        ]])
-    endif()
-    
     vcpkg_extract_source_archive_ex(
         OUT_SOURCE_PATH SOURCE_PATH
         ARCHIVE "${ARCHIVE}"