[vcpkg] Remove the tombstones and 'ignore' baseline concepts. (#12197)

This changes our PR builds to treat 'fail' in the ci.baseline.txt as 'skip' instead of using tombstones.

We currently have large numbers of spurious failures that get enshrined in PRs through no fault of a PR author, removing the tombstones concept will fix those by allowing the user to retry. This does mean we accept some risk of not detecting when a port is 'fixed', but that failure is reasonable for us to handle after we see it in CI, but that seems worth it given that it lets us get rid of the tombstone concept.

This also helps out the binary caching feature, because we don't have to figure out how to productize tombstones.

1 files changed, 12 insertions, 10 deletions

diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1
index d6b59364e..9a30321b4 100755
--- a/scripts/azure-pipelines/test-modified-ports.ps1
+++ b/scripts/azure-pipelines/test-modified-ports.ps1
@@ -15,9 +15,6 @@ The location where the binary caching archives are stored. Shared across runs of
 .PARAMETER WorkingRoot

 The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.

 

-.PARAMETER ArtifactStagingDirectory

-The Azure Pipelines artifact staging directory. If not supplied, defaults to the current directory.

-

 .PARAMETER ArtifactsDirectory

 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.

 

@@ -38,8 +35,6 @@ Param(
     [ValidateNotNullOrEmpty()]

     $WorkingRoot,

     [ValidateNotNullOrEmpty()]

-    $ArtifactStagingDirectory = '.',

-    [ValidateNotNullOrEmpty()]

     $ArtifactsDirectory = '.',

     $BuildReason = $null

 )

@@ -61,11 +56,13 @@ $commonArgs = @(
 )

 

 $binaryCachingMode = 'readwrite'

+$skipFailures = $false

 if ([string]::IsNullOrWhiteSpace($BuildReason)) {

     Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'

 }

 elseif ($BuildReason -eq 'PullRequest') {

-    Write-Host 'Build reason was Pull Request, using binary caching in read write mode.'

+    Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'

+    $skipFailures = $true

 }

 else {

     Write-Host "Build reason was $BuildReason, using binary caching in write only mode."

@@ -89,8 +86,13 @@ $xmlResults = Join-Path $ArtifactsDirectory 'xml-results'
 mkdir $xmlResults

 $xmlFile = Join-Path $xmlResults "$Triplet.xml"

 

+$failureLogs = Join-Path $ArtifactsDirectory 'failure-logs'

+

 & "./vcpkg$executableExtension" x-ci-clean @commonArgs

-$skipList = . "$PSScriptRoot/generate-skip-list.ps1" -Triplet $Triplet -BaselineFile "$PSScriptRoot/../ci.baseline.txt"

+$skipList = . "$PSScriptRoot/generate-skip-list.ps1" `

+    -Triplet $Triplet `

+    -BaselineFile "$PSScriptRoot/../ci.baseline.txt" `

+    -SkipFailures:$skipFailures

 

 # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.

 # Install them so the CI succeeds:

@@ -98,7 +100,7 @@ if ($Triplet -in @('x64-uwp', 'arm64-windows', 'arm-uwp')) {
     .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs

 }

 

-& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList @commonArgs

-& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults -outputDir $ArtifactStagingDirectory `

-    -failureLogDir (Join-Path $ArchivesRoot 'fail') -triplets $Triplet -errorOnRegression `

+& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --failure-logs=$failureLogs @commonArgs

+& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults `

+    -triplet $Triplet `

     -baselineFile .\scripts\ci.baseline.txt