From 45315bc3aaa0e718337eed13e6f6ae4e41bae04b Mon Sep 17 00:00:00 2001 From: nicole mazzuca <83086508+strega-nil-ms@users.noreply.github.com> Date: Wed, 29 Sep 2021 13:15:25 -0700 Subject: [macos ci] move out tensorflow (#20388) * [macos ci] move out tensorflow * rename ExtraPorts and CorePorts Co-authored-by: nicole mazzuca Co-authored-by: Billy Robert O'Neal III --- scripts/azure-pipelines/azure-pipelines.yml | 10 +++++ scripts/azure-pipelines/generate-skip-list.ps1 | 4 +- scripts/azure-pipelines/osx/azure-pipelines.yml | 41 +++++++++++++++--- scripts/azure-pipelines/test-modified-ports.ps1 | 55 ++++++++++++++++--------- 4 files changed, 84 insertions(+), 26 deletions(-) (limited to 'scripts') diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index f57de8835..49d30c2bc 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -52,6 +52,16 @@ jobs: - template: osx/azure-pipelines.yml parameters: poolName: $(osx-pool) + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + mode: AllPorts + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + mode: NonTensorflow +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: osx/azure-pipelines.yml + parameters: + poolName: $(osx-pool) + mode: OnlyTensorflow + jobName: x64_osx_tensorflow - template: linux/azure-pipelines.yml parameters: diff --git a/scripts/azure-pipelines/generate-skip-list.ps1 b/scripts/azure-pipelines/generate-skip-list.ps1 index 84b78b338..6c533595f 100755 --- a/scripts/azure-pipelines/generate-skip-list.ps1 +++ b/scripts/azure-pipelines/generate-skip-list.ps1 @@ -20,7 +20,8 @@ The path to the ci.baseline.txt file. Param( [string]$Triplet, [string]$BaselineFile, - [switch]$SkipFailures = $false + [switch]$SkipFailures = $false, + [String[]]$AdditionalSkips = @() ) $ErrorActionPreference = 'Stop' @@ -80,4 +81,5 @@ if ($SkipFailures) { $skip_list = $baselineForTriplet ` | Where-Object { $_ -match $targetRegex } ` | ForEach-Object { $_ -replace ":.*$" } +$skip_list += $AdditionalSkips [string]::Join(",", $skip_list) diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index aa5884f0b..f90a7c82c 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -2,8 +2,21 @@ # SPDX-License-Identifier: MIT # +parameters: +- name: poolName + type: string +- name: mode + type: string + values: + - 'AllPorts' + - 'NonTensorflow' + - 'OnlyTensorflow' +- name: jobName + type: string + default: x64_osx + jobs: -- job: x64_osx +- job: "${{ parameters.jobName }}" pool: name: ${{ parameters.poolName }} workspace: @@ -20,6 +33,18 @@ jobs: - group: vcpkg-asset-caching-credentials - name: X_VCPKG_ASSET_SOURCES value: "x-azurl,$(root-url),$(sas),readwrite" + - name: PowershellExtraArguments + ${{ if eq(parameters.mode, 'AllPorts') }}: + value: '' + ${{ if eq(parameters.mode, 'NonTensorflow') }}: + value: "-AdditionalSkips @('tensorflow', 'tensorflow-cc')" + ${{ if eq(parameters.mode, 'OnlyTensorflow') }}: + value: "-OnlyTest @('tensorflow', 'tensorflow-cc')" + - name: Postfix + ${{ if eq(parameters.mode, 'OnlyTensorflow') }}: + value: '-tensorflow' + ${{ if ne(parameters.mode, 'OnlyTensorflow') }}: + value: '' steps: - bash: df -h @@ -39,7 +64,13 @@ jobs: inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' - arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -BinarySourceStub "$(BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)' + arguments: > + -Triplet "x64-osx" + -BuildReason "$(Build.Reason)" + -BinarySourceStub "${{ variables.BINARY_SOURCE_STUB }}" + -WorkingRoot "${{ variables.WORKING_ROOT }}" + -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" + ${{ variables.PowershellExtraArguments }} pwsh: true - bash: | df -h @@ -49,15 +80,15 @@ jobs: displayName: 'Publish Artifact: failure logs for x64-osx' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs' - ArtifactName: 'failure logs for x64-osx' + ArtifactName: 'failure logs for x64-osx${{ variables.Postfix }}' condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') - bash: | python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/ displayName: 'Build a file list for all packages' condition: always() - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: file lists for x64-osx' + displayName: 'Publish Artifact: file lists for x64-osx${{ variables.Postfix }}' condition: always() inputs: PathtoPublish: scripts/list_files - ArtifactName: 'file lists for x64-osx' + ArtifactName: 'file lists for x64-osx${{ variables.Postfix }}' diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 index d55c6f629..46ea637dc 100755 --- a/scripts/azure-pipelines/test-modified-ports.ps1 +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -46,9 +46,10 @@ Param( $ArchivesRoot = $null, [Parameter(ParameterSetName='BinarySourceStub')] $BinarySourceStub = $null, - $BuildReason = $null, - [switch] - $PassingIsPassing = $false + [String]$BuildReason = $null, + [String[]]$AdditionalSkips = @(), + [String[]]$OnlyTest = $null, + [switch]$PassingIsPassing = $false ) if (-Not ((Test-Path "triplets/$Triplet.cmake") -or (Test-Path "triplets/community/$Triplet.cmake"))) { @@ -124,28 +125,42 @@ if ($LASTEXITCODE -ne 0) $skipList = . "$PSScriptRoot/generate-skip-list.ps1" ` -Triplet $Triplet ` -BaselineFile "$PSScriptRoot/../ci.baseline.txt" ` - -SkipFailures:$skipFailures + -SkipFailures:$skipFailures ` + -AdditionalSkips $AdditionalSkips -if ($Triplet -in @('x64-windows', 'x64-osx', 'x64-linux')) +if ($null -ne $OnlyTest) { - # WORKAROUND: These triplets are native-targetting which triggers an issue in how vcpkg handles the skip list. - # The workaround is to pass the skip list as host-excludes as well. - & "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --host-exclude=$skipList --failure-logs=$failureLogs @commonArgs + $OnlyTest | % { + $portName = $_ + & "./vcpkg$executableExtension" install --triplet $Triplet @commonArgs $portName + [System.Console]::Error.WriteLine( ` + "REGRESSION: ${portName}:$triplet. If expected, remove ${portName} from the OnlyTest list." ` + ) + } } else { - & "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --failure-logs=$failureLogs @commonArgs -} + if ($Triplet -in @('x64-windows', 'x64-osx', 'x64-linux')) + { + # WORKAROUND: These triplets are native-targetting which triggers an issue in how vcpkg handles the skip list. + # The workaround is to pass the skip list as host-excludes as well. + & "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --host-exclude=$skipList --failure-logs=$failureLogs @commonArgs + } + else + { + & "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --failure-logs=$failureLogs @commonArgs + } -$failureLogsEmpty = ((Test-Path $failureLogs) -and (Get-ChildItem $failureLogs).count -eq 0) -Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" + $failureLogsEmpty = ((Test-Path $failureLogs) -and (Get-ChildItem $failureLogs).count -eq 0) + Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" -if ($LASTEXITCODE -ne 0) -{ - throw "vcpkg ci failed" -} + if ($LASTEXITCODE -ne 0) + { + throw "vcpkg ci failed" + } -& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults ` - -triplet $Triplet ` - -baselineFile .\scripts\ci.baseline.txt ` - -passingIsPassing:$PassingIsPassing + & "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults ` + -triplet $Triplet ` + -baselineFile .\scripts\ci.baseline.txt ` + -passingIsPassing:$PassingIsPassing +} -- cgit v1.2.3