[vcpkg] Add `vcpkg export` to E2E tests. Enable E2E tests on all platforms. (#12198)

* [vcpkg] Add `vcpkg export` to E2E tests. Enable E2E tests on MacOS.

* [vcpkg] Fix export --raw --output-dir=/path/ by changing directory to new export root

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

6 files changed, 63 insertions, 32 deletions

diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml
index 80c547633..cda29f7e6 100644
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@@ -23,6 +23,9 @@ stages:
   displayName: Check the formatting of port manifests

   pool: $(windows-pool)

   dependsOn: []

+  variables:

+  - name: VCPKG_DOWNLOADS

+    value: D:\downloads

   jobs:

   - job:

     workspace:

@@ -32,7 +35,7 @@ stages:
       displayName: 'Check port manifest Formatting'

       inputs:

         filePath: 'scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1'

-        arguments: '-Root . -Downloads D:\Downloads'

+        arguments: '-Root .'

 - stage: run_port_ci

   displayName: 'Run the Port CI'

   dependsOn:

diff --git a/scripts/azure-pipelines/end-to-end-tests.ps1 b/scripts/azure-pipelines/end-to-end-tests.ps1
index b0e447657..43ec7aa73 100644
--- a/scripts/azure-pipelines/end-to-end-tests.ps1
+++ b/scripts/azure-pipelines/end-to-end-tests.ps1
@@ -158,8 +158,13 @@ Require-FileNotExists "$installRoot/$Triplet/include/rapidjson/rapidjson.h"
 Require-FileNotExists "$buildtreesRoot/rapidjson/src"

 Require-FileExists "$TestingRoot/packages.config"

 

-& $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"

-Throw-IfFailed

+if ($IsLinux -or $IsMacOS) {

+    mono $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"

+    Throw-IfFailed

+} else {

+    & $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"

+    Throw-IfFailed

+}

 

 Remove-Item -Recurse -Force $NuGetRoot -ErrorAction SilentlyContinue

 mkdir $NuGetRoot

@@ -177,3 +182,15 @@ Require-FileExists "$buildtreesRoot/tinyxml/src"
 if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) {

     throw "In '$CurrentTest': did not create exactly 1 NuGet package"

 }

+

+# Test export

+$args = $commonArgs + @("export","rapidjson","tinyxml","--nuget","--nuget-id=vcpkg-export","--nuget-version=1.0.0","--output=vcpkg-export-output","--raw","--zip","--output-dir=$TestingRoot")

+$CurrentTest = "./vcpkg $($args -join ' ')"

+Write-Host $CurrentTest

+Require-FileNotExists "$TestingRoot/vcpkg-export-output"

+Require-FileNotExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"

+Require-FileNotExists "$TestingRoot/vcpkg-export-output.zip"

+./vcpkg @args

+Require-FileExists "$TestingRoot/vcpkg-export-output"

+Require-FileExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"

+Require-FileExists "$TestingRoot/vcpkg-export-output.zip"

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index 4d4eada62..969348980 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -9,6 +9,12 @@ jobs:
   workspace:

     clean: resources

   timeoutInMinutes: 1440 # 1 day

+  variables:

+  - name: WORKING_ROOT

+    value: /mnt/vcpkg-ci

+  - name: VCPKG_DOWNLOADS

+    value: /mnt/vcpkg-ci/downloads

+

   steps:

   - bash: df -h

     displayName: 'Report on Disk Space'

@@ -19,17 +25,15 @@ jobs:
     displayName: 'Create /home/agent'

     # Note: /mnt is the Azure machines' temporary disk.

   - bash: |

-      sudo mkdir /mnt/vcpkg-ci -m=777

-      sudo mkdir /mnt/vcpkg-ci/downloads -m=777

+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777

+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777

       exit 0

-    displayName: 'Create /mnt/vcpkg-ci/downloads'

+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'

   - task: Bash@3

     displayName: 'Build vcpkg'

     inputs:

       filePath: bootstrap-vcpkg.sh

       arguments: "-buildTests"

-    env:

-      VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - task: PowerShell@2

@@ -37,7 +41,7 @@ jobs:
     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactsDirectory $(System.ArtifactsDirectory)'

+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index 3c9510bac..6e48238a7 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -10,6 +10,12 @@ jobs:
   workspace:

     clean: resources

   timeoutInMinutes: 1440 # 1 day

+  variables:

+  - name: WORKING_ROOT

+    value: /Users/vagrant/Data

+  - name: VCPKG_DOWNLOADS

+    value: /Users/vagrant/Data/downloads

+

   steps:

   - bash: |

       df -h

@@ -22,28 +28,32 @@ jobs:
       brew list libtool || brew install libtool

       brew list bison || brew install bison

       brew list gfortran || brew cask install gfortran

+      brew list mono || brew install mono

       brew list yasm || brew install yasm

     displayName: 'Install brew dependencies'

   - bash: |

-      sudo mkdir /Users/vagrant/Data/downloads || 0

-      sudo chmod 777 /Users/vagrant/Data/downloads || 0

+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0

+      sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0

       exit 0

-    displayName: 'Create /Users/vagrant/Data/downloads'

+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'

   - task: Bash@3

     displayName: 'Build vcpkg'

     inputs:

       filePath: bootstrap-vcpkg.sh

       arguments: '-buildTests'

-    env:

-      VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - task: PowerShell@2

+    displayName: 'Run vcpkg end-to-end tests'

+    inputs:

+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'

+      arguments: '-Triplet x64-osx -WorkingRoot ${{ variables.WORKING_ROOT }}'

+  - task: PowerShell@2

     displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactsDirectory $(System.ArtifactsDirectory)'

+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1 b/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1
index f4385c2b4..6e97ba35c 100644
--- a/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1
+++ b/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1
@@ -3,8 +3,6 @@ Param(
     [Parameter(Mandatory=$True)]
     [string]$Root,
     [Parameter()]
-    [string]$DownloadsDirectory,
-    [Parameter()]
     [switch]$IgnoreErrors # allows one to just format
 )
 
@@ -16,11 +14,6 @@ if (-not (Test-Path "$Root/.vcpkg-root"))
     throw
 }
 
-if (-not [string]::IsNullOrEmpty($DownloadsDirectory))
-{
-    $env:VCPKG_DOWNLOADS = $DownloadsDirectory
-}
-
 if (-not (Test-Path "$Root/vcpkg.exe"))
 {
     & "$Root/bootstrap-vcpkg.bat"
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index 5ec0b300c..340138e12 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -9,6 +9,11 @@ jobs:
   workspace:

     clean: resources

   timeoutInMinutes: 1440 # 1 day

+  variables:

+  - name: WORKING_ROOT

+    value: D:\

+  - name: VCPKG_DOWNLOADS

+    value: D:\downloads

 

   steps:

   - task: PowerShell@2

@@ -21,35 +26,35 @@ jobs:
     inputs:

       filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'

   # Note: D: is the Azure machines' temporary disk.

-  - task: CmdLine@2

+  - script: .\bootstrap-vcpkg.bat

     displayName: 'Build vcpkg'

-    inputs:

-      script: |

-        set VCPKG_DOWNLOADS=D:\downloads

-        .\bootstrap-vcpkg.bat

   - task: CmdLine@2

     displayName: "Build vcpkg with CMake and Run Tests"

     condition: eq('${{ parameters.triplet }}', 'x86-windows')

     inputs:

       script: |

         :: TRANSITION, get these tools on the VMs next time we roll them

-        set VCPKG_DOWNLOADS=D:\downloads

         .\vcpkg.exe fetch cmake

         .\vcpkg.exe fetch ninja

-        set PATH=D:\downloads\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;D:\downloads\tools\ninja-1.10.0-windows;%PATH%

+        set PATH=${{ variables.VCPKG_DOWNLOADS }}\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;${{ variables.VCPKG_DOWNLOADS }}\tools\ninja-1.10.0-windows;%PATH%

         call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86

         rmdir /s /q build.x86.debug > nul 2> nul

         cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=ON -B build.x86.debug -S toolsrc

         ninja.exe -C build.x86.debug

         build.x86.debug\vcpkg-test.exe

-        powershell.exe -NoProfile -ExecutionPolicy Bypass "scripts\azure-pipelines\end-to-end-tests.ps1 -WorkingRoot \"%cd%\testing\" -triplet x86-windows"

       failOnStderr: true

   - task: PowerShell@2

+    displayName: 'Run vcpkg end-to-end tests'

+    condition: eq('${{ parameters.triplet }}', 'x86-windows')

+    inputs:

+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'

+      arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}'

+  - task: PowerShell@2

     displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

-      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactsDirectory $(System.ArtifactsDirectory)'

+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'

   - task: PowerShell@2

     displayName: 'Report on Disk Space After Build'

     condition: always()

@@ -68,7 +73,6 @@ jobs:
     inputs:

       targetType: inline

       script: |

-        $env:VCPKG_DOWNLOADS = "D:\downloads"

         ./vcpkg.exe fetch python3

         & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\

   - task: PublishBuildArtifacts@1