[vcpkg] Commonize pipelines build powershell and don't use binary caching in CI (rather than PR) builds (#12082)

4 files changed, 123 insertions, 61 deletions

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index 1e2a653e0..77842437f 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -32,28 +32,16 @@ jobs:
       VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

-  - powershell: |

-      $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads'

-      $commonArgs = @(

-        '--binarycaching',

-        '--x-binarysource=clear;files,/archives,upload',

-        '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees',

-        '--x-install-root=/mnt/vcpkg-ci/install',

-        '--x-packages-root=/mnt/vcpkg-ci/packages'

-      )

-

-      ./vcpkg x-ci-clean @commonArgs

-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

-      mkdir $(System.ArtifactsDirectory)/xml-results

-      $env:HOME = '/home/agent'

-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs

-    displayName: '** Test Modified Ports **'

   - task: PowerShell@2

-    displayName: 'Analyze results and prepare test logs'

+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "/archives/fail" -triplets "x64-linux" -errorOnRegression'

+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'

+  - task: PublishTestResults@2

+    inputs:

+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-linux.xml'

+    condition: always()

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index 85fe543c8..8008df45f 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -36,27 +36,16 @@ jobs:
       VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'

   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

-  - powershell: |

-      $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads'

-      $commonArgs = @(

-        '--binarycaching',

-        '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload',

-        '--x-buildtrees-root=/Users/vagrant/Data/buildtrees',

-        '--x-install-root=/Users/vagrant/Data/installed',

-        '--x-packages-root=/Users/vagrant/Data/packages'

-      )

-

-      ./vcpkg x-ci-clean @commonArgs

-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

-      mkdir $(System.ArtifactsDirectory)/xml-results

-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs

-    displayName: '** Test Modified Ports **'

   - task: PowerShell@2

-    displayName: 'Analyze results and prepare test logs'

+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir /Users/vagrant/Data/archives/fail -triplets "x64-osx" -errorOnRegression'

+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'

+  - task: PublishTestResults@2

+    inputs:

+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-osx.xml'

+    condition: always()

   - bash: |

       df -h

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1
new file mode 100755
index 000000000..e38115b32
--- /dev/null
+++ b/scripts/azure-pipelines/test-modified-ports.ps1
@@ -0,0 +1,102 @@
+# Copyright (c) Microsoft Corporation.

+# SPDX-License-Identifier: MIT

+#

+

+<#

+.SYNOPSIS

+Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms.

+

+.PARAMETER Triplet

+The triplet to test.

+

+.PARAMETER ArchivesRoot

+The location where the binary caching archives are stored. Shared across runs of this script.

+

+.PARAMETER WorkingRoot

+The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.

+

+.PARAMETER ArtifactStagingDirectory

+The Azure Pipelines artifact staging directory. If not supplied, defaults to the current directory.

+

+.PARAMETER ArtifactsDirectory

+The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.

+

+.PARAMETER BuildReason

+The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not

+supplied, binary caching will be used.

+#>

+

+[CmdletBinding()]

+Param(

+    [Parameter(Mandatory=$true)]

+    [ValidateNotNullOrEmpty()]

+    [string]$Triplet,

+    [Parameter(Mandatory=$true)]

+    [ValidateNotNullOrEmpty()]

+    $ArchivesRoot,

+    [Parameter(Mandatory=$true)]

+    [ValidateNotNullOrEmpty()]

+    $WorkingRoot,

+    [ValidateNotNullOrEmpty()]

+    $ArtifactStagingDirectory = '.',

+    [ValidateNotNullOrEmpty()]

+    $ArtifactsDirectory = '.',

+    $BuildReason = $null

+)

+

+if (-Not (Test-Path "triplets/$Triplet.cmake")) {

+    Write-Error "Incorrect triplet '$Triplet', please supply a valid triplet."

+}

+

+$env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'

+$buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'

+$installRoot = Join-Path $WorkingRoot 'installed'

+$packagesRoot = Join-Path $WorkingRoot 'packages'

+$commonArgs = @(

+    "--x-buildtrees-root=$buildtreesRoot",

+    "--x-install-root=$installRoot",

+    "--x-packages-root=$packagesRoot"

+)

+

+$binaryCaching = $false

+if ([string]::IsNullOrWhiteSpace($BuildReason)) {

+    Write-Host 'Build reason not specified, defaulting to using binary caching.'

+    $binaryCaching = $true

+} elseif ($BuildReason -eq 'PullRequest') {

+    Write-Host 'Build reason was Pull Request, using binary caching.'

+    $binaryCaching = $true

+}

+

+if ($binaryCaching) {

+    $commonArgs += @(

+        '--binarycaching',

+        "--x-binarysource=clear;files,$ArchivesRoot,upload"

+    )

+}

+

+if ($Triplet -eq 'x64-linux') {

+    $env:HOME = '/home/agent'

+    $executableExtension = [string]::Empty

+} elseif ($Triplet -eq 'x64-osx') {

+    $executableExtension = [string]::Empty

+} else {

+    $executableExtension = '.exe'

+}

+

+$xmlResults = Join-Path $ArtifactsDirectory 'xml-results'

+mkdir $xmlResults

+$xmlFile = Join-Path $xmlResults "$Triplet.xml"

+

+& "./vcpkg$executableExtension" x-ci-clean @commonArgs

+$skipList = . "$PSScriptRoot/generate-skip-list.ps1" -Triplet $Triplet -BaselineFile "$PSScriptRoot/../ci.baseline.txt"

+

+# WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.

+# Install them so the CI succeeds:

+if ($Triplet -in @('x64-uwp', 'arm64-windows', 'arm-uwp')) {

+    .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs

+}

+

+& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList @commonArgs

+& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults -outputDir $ArtifactStagingDirectory `

+    -failureLogDir (Join-Path $ArchivesRoot 'fail') -triplets $Triplet -errorOnRegression `

+    -baselineFile .\scripts\ci.baseline.txt

diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index 274df268f..0f2120746 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -38,33 +38,16 @@ jobs:
         ninja.exe -C build.x86.debug

         build.x86.debug\vcpkg-test.exe

       failOnStderr: true

-  - powershell: |

-      $env:VCPKG_DOWNLOADS='D:\downloads'

-      $commonArgs = @(

-        '--binarycaching',

-        '--x-binarysource=clear;files,W:\,upload',

-        '--x-buildtrees-root=D:\buildtrees',

-        '--x-install-root=D:\install',

-        '--x-packages-root=D:\packages'

-      )

-

-      .\vcpkg.exe x-ci-clean @commonArgs

-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt

-      mkdir $(System.ArtifactsDirectory)\xml-results

-      # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.

-      # Install them so the CI succeeds:

-      if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {

-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs

-      }

-

-      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs

-    displayName: '** Test Modified Ports **'

   - task: PowerShell@2

-    displayName: 'Analyze results and prepare test logs'

+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'

-      arguments: '-baselineFile .\scripts\ci.baseline.txt -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "W:\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"'

+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'

+  - task: PublishTestResults@2

+    inputs:

+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/${{ parameters.triplet }}.xml'

+    condition: always()

   - task: PublishBuildArtifacts@1

     displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs'

     inputs: