[vcpkg] Allow CI to pass in all relevant directories and remove use of symbolic links (#11483)

6 files changed, 28 insertions, 96 deletions

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index 022fc70e4..fc0977697 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -10,28 +10,26 @@ jobs:
     clean: resources

   timeoutInMinutes: 1440 # 1 day

   steps:

-  - bash: |

-      df -h

-    displayName: 'Report on Disk Space Before Environment'

-    condition: always()

-  - task: Bash@3

-    displayName: 'Initialize Environment'

-    inputs:

-      filePath: scripts/azure-pipelines/linux/initialize-environment.sh

+  - bash: df -h

+    displayName: 'Report on Disk Space'

   - bash: |

       sudo mkdir /home/agent -m=777

       sudo chown `id -u` /home/agent

       exit 0

     displayName: 'Create /home/agent'

+    # Note: /mnt is the Azure machines' temporary disk.

   - bash: |

-      df -h

-    displayName: 'Report on Disk Space After Environment'

-    condition: always()

+      sudo mkdir /mnt/vcpkg-ci -m=777

+      sudo mkdir /mnt/vcpkg-ci/downloads -m=777

+      exit 0

+    displayName: 'Create /mnt/vcpkg-ci/downloads'

   - task: Bash@3

     displayName: 'Build vcpkg'

     inputs:

       filePath: bootstrap-vcpkg.sh

       arguments: "-buildTests"

+    env:

+      VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - powershell: |

@@ -39,14 +37,14 @@ jobs:
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

       mkdir $(System.ArtifactsDirectory)/xml-results

       $env:HOME = '/home/agent'

-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching

+      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "archives/fail" -triplets "x64-linux" -errorOnRegression'

+      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "/archives/fail" -triplets "x64-linux" -errorOnRegression'

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh
deleted file mode 100755
index 1cbdd3326..000000000
--- a/scripts/azure-pipelines/linux/initialize-environment.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: MIT
-
-# Cleans up the environment to prevent contamination across builds.
-if [ ! -d "archives" ]; then
-    ln -s /archives archives
-fi
diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index e51698c45..be410e6a2 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -33,20 +33,22 @@ jobs:
     inputs:

       filePath: bootstrap-vcpkg.sh

       arguments: '-buildTests'

+    env:

+      VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - powershell: |

-      ./vcpkg x-ci-clean

-      $skip_list = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

+      ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads

+      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

       mkdir $(System.ArtifactsDirectory)/xml-results

-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skip_list --binarycaching

+      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir ~/Data/archives/fail -triplets "x64-osx" -errorOnRegression'

+      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir /Users/vagrant/Data/archives/fail -triplets "x64-osx" -errorOnRegression'

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/osx/initialize-environment.sh b/scripts/azure-pipelines/osx/initialize-environment.sh
index 6f42a809a..e94790668 100755
--- a/scripts/azure-pipelines/osx/initialize-environment.sh
+++ b/scripts/azure-pipelines/osx/initialize-environment.sh
@@ -4,20 +4,5 @@
 
 # Sets up the environment for MacOS runs of vcpkg CI
 
-mkdir -p ~/Data/installed || true
-ln -s ~/Data/installed
-
-mkdir -p ~/Data/buildtrees || true
-ln -s ~/Data/buildtrees
-
-mkdir -p ~/Data/packages || true
-ln -s ~/Data/packages
-
-rm archives || rm -rf archives || true
-ln -s ~/Data/archives
-
-mkdir -p ~/Data/downloads || true
-ln -s ~/Data/downloads
-
 #delete downloaded files that have not been used in 7 days
-find downloads/  -maxdepth 1 -type f ! -atime 7  -exec rm -f {} \;
+find ~/Data/downloads/  -maxdepth 1 -type f ! -atime 7  -exec rm -f {} \;
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index 48f3908d2..f2b14657b 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -15,33 +15,32 @@ jobs:
     displayName: 'Initialize Environment'

     inputs:

       filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'

-  - task: BatchScript@1

+    # Note: D: is the Azure machines' temporary disk.

+  - task: CmdLine@2

     displayName: 'Build vcpkg'

     inputs:

-      filename: 'bootstrap-vcpkg.bat'

+      script: |

+        set VCPKG_DOWNLOADS=D:\downloads

+        .\bootstrap-vcpkg.bat

   - powershell: |

-      .\vcpkg.exe x-ci-clean

-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt"

+      .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

+      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt

       mkdir $(System.ArtifactsDirectory)\xml-results

       # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.

       # Install them so the CI succeeds:

       if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {

-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows

+          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

       }

 

-      $command = ".\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching"

+      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

       Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII

-    displayName: 'Write Test Modified Ports Batch File'

-  - task: BatchScript@1

     displayName: '** Test Modified Ports **'

-    inputs:

-      filename: 'run_ci.cmd'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "archives\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"'

+      arguments: '-baselineFile .\scripts\ci.baseline.txt -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "W:\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"'

   - task: PublishBuildArtifacts@1

     displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs'

     inputs:

diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1
index 4211a228a..4a252df49 100644
--- a/scripts/azure-pipelines/windows/initialize-environment.ps1
+++ b/scripts/azure-pipelines/windows/initialize-environment.ps1
@@ -22,13 +22,6 @@ Param(
 $StorageAccountName = $env:StorageAccountName

 $StorageAccountKey = $env:StorageAccountKey

 

-function Remove-DirectorySymlink {

-    Param([string]$Path)

-    if (Test-Path $Path) {

-        [System.IO.Directory]::Delete($Path, $true)

-    }

-}

-

 Write-Host 'Setting up archives mount'

 if (-Not (Test-Path W:)) {

     net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey

@@ -52,40 +45,3 @@ if( Test-Path D:\downloads\tools\msys2 )
     Write-Host "removing previously installed msys2"

     Remove-Item D:\downloads\tools\msys2 -Recurse -Force

 }

-

-Write-Host 'Setting up archives path...'

-if ([string]::IsNullOrWhiteSpace($ForceAllPortsToRebuildKey))

-{

-    $archivesPath = 'W:\'

-}

-else

-{

-    $archivesPath = "W:\force\$ForceAllPortsToRebuildKey"

-    if (-Not (Test-Path $fullPath)) {

-        Write-Host 'Creating $archivesPath'

-        mkdir $archivesPath

-    }

-}

-

-Write-Host "Linking archives => $archivesPath"

-if (-Not (Test-Path archives)) {

-    cmd /c "mklink /D archives $archivesPath"

-}

-

-Write-Host 'Linking installed => E:\installed'

-if (-Not (Test-Path E:\installed)) {

-    mkdir E:\installed

-}

-

-if (-Not (Test-Path installed)) {

-    cmd /c "mklink /D installed E:\installed"

-}

-

-Write-Host 'Linking downloads => D:\downloads'

-if (-Not (Test-Path D:\downloads)) {

-    mkdir D:\downloads

-}

-

-if (-Not (Test-Path downloads)) {

-    cmd /c "mklink /D downloads D:\downloads"

-}