[vcpkg registries] Add git registries (#15054)

* [vcpkg registries] Add git registries support

* Add git registries support to the registries module of vcpkg.
* add e2e tests for git registries
* fix vcpkg.cmake for registries

* fix CRs, remove a thing

* better error messages

* Billy CRs

* fix Robert's CR comment

* I learned about `-c` today

* format

* fix baseline.json

* failing to find baseline is technically not a bug

6 files changed, 207 insertions, 18 deletions

diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1
index b985a61f5..777c0901f 100644
--- a/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1
+++ b/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1
@@ -1,13 +1,187 @@
 . "$PSScriptRoot/../end-to-end-tests-prelude.ps1"
 
 
-$commonArgs += @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions")
+$builtinRegistryArgs = $commonArgs + @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions")
 
-Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port'
+Run-Vcpkg install @builtinRegistryArgs 'vcpkg-internal-e2e-test-port'
 Throw-IfNotFailed
 
-Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
-Throw-IfFailed
+# We should not look into the port_versions directory unless we have a baseline,
+# even if we pass the registries feature flag
+Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
+Throw-IfNotFailed
 
-Run-Vcpkg install @commonArgs --feature-flags=registries 'zlib'
+Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'zlib'
 Throw-IfFailed
+
+# Test git and filesystem registries
+Refresh-TestRoot
+$filesystemRegistry = "$TestingRoot/filesystem-registry"
+$gitRegistryUpstream = "$TestingRoot/git-registry-upstream"
+
+# build a filesystem registry
+New-Item -Path $filesystemRegistry -ItemType Directory
+$filesystemRegistry = (Get-Item $filesystemRegistry).FullName
+
+Copy-Item -Recurse `
+    -LiteralPath "$PSScriptRoot/../../e2e_ports/vcpkg-internal-e2e-test-port" `
+    -Destination "$filesystemRegistry"
+New-Item `
+    -Path "$filesystemRegistry/port_versions" `
+    -ItemType Directory
+Copy-Item `
+    -LiteralPath "$PSScriptRoot/../../e2e_ports/port_versions/baseline.json" `
+    -Destination "$filesystemRegistry/port_versions/baseline.json"
+New-Item `
+    -Path "$filesystemRegistry/port_versions/v-" `
+    -ItemType Directory
+
+$vcpkgInternalE2eTestPortJson = @{
+    "versions" = @(
+        @{
+            "version-string" = "1.0.0";
+            "path" = "$/vcpkg-internal-e2e-test-port"
+        }
+    )
+}
+New-Item `
+    -Path "$filesystemRegistry/port_versions/v-/vcpkg-internal-e2e-test-port.json" `
+    -ItemType File `
+    -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgInternalE2eTestPortJson)
+
+
+# build a git registry
+New-Item -Path $gitRegistryUpstream -ItemType Directory
+$gitRegistryUpstream = (Get-Item $gitRegistryUpstream).FullName
+
+Push-Location $gitRegistryUpstream
+try
+{
+    $gitConfigOptions = @(
+        '-c', 'user.name=Nobody',
+        '-c', 'user.email=nobody@example.com',
+        '-c', 'core.autocrlf=false'
+    )
+
+    $CurrentTest = 'git init .'
+    git @gitConfigOptions init .
+    Throw-IfFailed
+    Copy-Item -Recurse -LiteralPath "$PSScriptRoot/../../e2e_ports/port_versions" -Destination .
+    Copy-Item -Recurse -LiteralPath "$PSScriptRoot/../../e2e_ports/vcpkg-internal-e2e-test-port" -Destination .
+
+    $CurrentTest = 'git add -A'
+    git @gitConfigOptions add -A
+    Throw-IfFailed
+    $CurrentTest = 'git commit'
+    git @gitConfigOptions commit -m 'initial commit'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+# actually test the registries
+$vcpkgJson = @{
+    "name" = "manifest-test";
+    "version-string" = "1.0.0";
+    "dependencies" = @(
+        "vcpkg-internal-e2e-test-port"
+    )
+}
+
+$manifestDir = "$TestingRoot/builtin-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+
+try
+{
+    $vcpkgJsonWithBaseline = $vcpkgJson.Clone()
+    $vcpkgJsonWithBaseline['$x-default-baseline'] = 'default'
+
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfNotFailed
+
+    New-Item -Path 'vcpkg.json' -ItemType File -Force `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJsonWithBaseline)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+
+# test the filesystem registry
+$manifestDir = "$TestingRoot/filesystem-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+try
+{
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    $vcpkgConfigurationJson = @{
+        "default-registry" = $null;
+        "registries" = @(
+            @{
+                "kind" = "filesystem";
+                "path" = $filesystemRegistry;
+                "packages" = @( "vcpkg-internal-e2e-test-port" )
+            }
+        )
+    }
+    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+# test the git registry
+$manifestDir = "$TestingRoot/git-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+try
+{
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    $vcpkgConfigurationJson = @{
+        "default-registry" = $null;
+        "registries" = @(
+            @{
+                "kind" = "git";
+                "repository" = $gitRegistryUpstream;
+                "packages" = @( "vcpkg-internal-e2e-test-port" )
+            }
+        )
+    }
+    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1
index 5d61b2101..f5f9de20b 100644
--- a/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1
+++ b/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1
@@ -69,7 +69,7 @@ Throw-IfNotFailed
 

 git fetch https://github.com/vicroms/test-registries

 $CurrentTest = "default baseline"

-./vcpkg $commonArgs "--feature-flags=versions" install `

+./vcpkg $commonArgs --debug "--feature-flags=versions" install `

     "--x-manifest-root=scripts/testing/version-files/default-baseline-2" `

     "--x-builtin-port-versions-dir=scripts/testing/version-files/default-baseline-2/port_versions"

 Throw-IfFailed

diff --git a/scripts/azure-pipelines/end-to-end-tests-prelude.ps1 b/scripts/azure-pipelines/end-to-end-tests-prelude.ps1
index 1e340eff7..deadb210c 100644
--- a/scripts/azure-pipelines/end-to-end-tests-prelude.ps1
+++ b/scripts/azure-pipelines/end-to-end-tests-prelude.ps1
@@ -12,10 +12,19 @@ $commonArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",

     "--x-install-root=$installRoot",

     "--x-packages-root=$packagesRoot",

-    "--overlay-ports=scripts/e2e_ports/overlays"

+    "--overlay-ports=$PSScriptRoot/../e2e_ports/overlays"

 )

 $Script:CurrentTest = 'unassigned'

 

+if ($IsWindows)

+{

+    $VcpkgExe = Get-Item './vcpkg.exe'

+}

+else

+{

+    $VcpkgExe = Get-Item './vcpkg'

+}

+

 function Refresh-TestRoot {

     Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue

     mkdir $TestingRoot | Out-Null

@@ -59,17 +68,9 @@ function Run-Vcpkg {
         [Parameter(ValueFromRemainingArguments)]

         [string[]]$TestArgs

     )

-

-    if ($IsWindows) {

-        $vcpkgName = 'vcpkg.exe'

-    } else {

-        $vcpkgName = 'vcpkg'

-    }

-

-    $Script:CurrentTest = "./$vcpkgName $($testArgs -join ' ')"

+    $Script:CurrentTest = "vcpkg $($testArgs -join ' ')"

     Write-Host $Script:CurrentTest

-    & "./$vcpkgName" @testArgs | Tee-Object -Variable 'ConsoleOutput'

-    return $ConsoleOutput

+    & $VcpkgExe @testArgs

 }

 

 Refresh-TestRoot

diff --git a/scripts/azure-pipelines/end-to-end-tests.ps1 b/scripts/azure-pipelines/end-to-end-tests.ps1
index 029120d54..3c9dd067f 100644
--- a/scripts/azure-pipelines/end-to-end-tests.ps1
+++ b/scripts/azure-pipelines/end-to-end-tests.ps1
@@ -32,6 +32,8 @@ Param(
 

 $ErrorActionPreference = "Stop"

 

+$WorkingRoot = (Get-Item $WorkingRoot).FullName

+

 $AllTests = Get-ChildItem $PSScriptRoot/end-to-end-tests-dir/*.ps1

 if ($Filter -ne $Null) {

     $AllTests = $AllTests | ? { $_.Name -match $Filter }

diff --git a/scripts/buildsystems/vcpkg.cmake b/scripts/buildsystems/vcpkg.cmake
index bc54afb70..e13c3333b 100644
--- a/scripts/buildsystems/vcpkg.cmake
+++ b/scripts/buildsystems/vcpkg.cmake
@@ -384,6 +384,11 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
             endforeach()
         endif()
 
+        if(DEFINED VCPKG_FEATURE_FLAGS OR DEFINED CACHE{VCPKG_FEATURE_FLAGS})
+            list(JOIN VCPKG_FEATURE_FLAGS "," _VCPKG_FEATURE_FLAGS)
+            set(_VCPKG_FEATURE_FLAGS "--feature-flags=${_VCPKG_FEATURE_FLAGS}")
+        endif()
+
         foreach(feature IN LISTS VCPKG_MANIFEST_FEATURES)
             list(APPEND _VCPKG_ADDITIONAL_MANIFEST_PARAMS "--x-feature=${feature}")
         endforeach()
@@ -405,6 +410,7 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
                 "--x-wait-for-lock"
                 "--x-manifest-root=${_VCPKG_MANIFEST_DIR}"
                 "--x-install-root=${_VCPKG_INSTALLED_DIR}"
+                "${_VCPKG_FEATURE_FLAGS}"
                 ${_VCPKG_ADDITIONAL_MANIFEST_PARAMS}
                 ${VCPKG_INSTALL_OPTIONS}
             OUTPUT_VARIABLE _VCPKG_MANIFEST_INSTALL_LOGTEXT
@@ -422,6 +428,10 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
             set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
                 "${_VCPKG_MANIFEST_DIR}/vcpkg.json"
                 "${_VCPKG_INSTALLED_DIR}/vcpkg/status")
+            if(EXISTS "${_VCPKG_MANIFEST_DIR}/vcpkg-configuration.json")
+                set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
+                    "${_VCPKG_MANIFEST_DIR}/vcpkg-configuration.json")
+            endif()
         else()
             message(STATUS "Running vcpkg install - failed")
             _vcpkg_add_fatal_error("vcpkg install failed. See logs for more information: ${_VCPKG_MANIFEST_INSTALL_LOGFILE}")
diff --git a/scripts/e2e_ports/port_versions/baseline.json b/scripts/e2e_ports/port_versions/baseline.json
index 98e63d8b5..2413f8afc 100644
--- a/scripts/e2e_ports/port_versions/baseline.json
+++ b/scripts/e2e_ports/port_versions/baseline.json
@@ -1,3 +1,5 @@
 {
-	"vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" }
+    "default": {
+        "vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" }
+    }
 }