[vcpkg] Add experimental x-azblob binary provider (#13626)

* [vcpkg] Add experimental x-azblob binary provider

* [vcpkg] Test azblob storage provider in CI

* [vcpkg] Address some CR comments from #13639

* [vcpkg] Fixup azure-pipelines

* [vcpkg] Fix regression where the downloaded package is purged before decompressing

* [vcpkg] Further refactor vcpkg::Downloads

* [vcpkg] Enable OSX for x-azblob testing

* [vcpkg] Reduce diff against master

* [vcpkg] Extract Downloads::details::split_uri_view

* [vcpkg] Address PR comments

* [vcpkg] Add testing and metrics for x-azblob

* [vcpkg] Add docs for x-azblob

This includes a note that it is currently experimental

* [vcpkg] Address CR comments

* [vcpkg] Revert pipeline changes except OSX to minimize disruption

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

2 files changed, 20 insertions, 7 deletions

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index 6f2e90fec..c0e1c3a26 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -14,6 +14,9 @@ jobs:
     value: /Users/vagrant/Data

   - name: VCPKG_DOWNLOADS

     value: /Users/vagrant/Data/downloads

+  - group: azblob-test-sas-group

+  - name: BINARY_SOURCE_STUB

+    value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"

 

   steps:

   - bash: df -h

@@ -40,7 +43,7 @@ jobs:
     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'

+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -BinarySourceStub "$(BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1
index bac16c595..98ffbf53a 100755
--- a/scripts/azure-pipelines/test-modified-ports.ps1
+++ b/scripts/azure-pipelines/test-modified-ports.ps1
@@ -16,8 +16,11 @@ The location used as scratch space for 'installed', 'packages', and 'buildtrees'
 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.

 

 .PARAMETER ArchivesRoot

-The location where the binary caching archives are stored. Shared across runs of this script. If

-this parameter is not set, binary caching will not be used.

+Equivalent to '-BinarySourceStub "files,$ArchivesRoot"'

+

+.PARAMETER BinarySourceStub

+The type and parameters of the binary source. Shared across runs of this script. If

+this parameter is not set, binary caching will not be used. Example: "files,W:\"

 

 .PARAMETER BuildReason

 The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).

@@ -25,7 +28,7 @@ If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set
 binary caching will default to read-write mode.

 #>

 

-[CmdletBinding()]

+[CmdletBinding(DefaultParameterSetName="ArchivesRoot")]

 Param(

     [Parameter(Mandatory = $true)]

     [ValidateNotNullOrEmpty()]

@@ -35,7 +38,10 @@ Param(
     $WorkingRoot,

     [ValidateNotNullOrEmpty()]

     $ArtifactStagingDirectory = '.',

+    [Parameter(ParameterSetName='ArchivesRoot')]

     $ArchivesRoot = $null,

+    [Parameter(ParameterSetName='BinarySourceStub')]

+    $BinarySourceStub = $null,

     $BuildReason = $null

 )

 

@@ -49,7 +55,7 @@ $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'

 $packagesRoot = Join-Path $WorkingRoot 'packages'

 

-$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot))

+$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or -Not ([string]::IsNullOrWhiteSpace($BinarySourceStub))

 $commonArgs = @()

 if ($usingBinaryCaching) {

     $commonArgs += @('--binarycaching')

@@ -78,8 +84,12 @@ if ($usingBinaryCaching) {
         Write-Host "Build reason was $BuildReason, using binary caching in write only mode."

         $binaryCachingMode = 'write'

     }

-

-    $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")

+    if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) {

+        $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")

+    }

+    else {

+        $commonArgs += @("--binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")

+    }

 }

 

 if ($Triplet -eq 'x64-linux') {