From 4fb225608532e9fb2fd2f5f1dbe9ec092cdc7c93 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Wed, 3 Jun 2020 19:31:28 -0700 Subject: [vcpkg] Allow CI to pass in all relevant directories and remove use of symbolic links (#11483) --- scripts/azure-pipelines/linux/azure-pipelines.yml | 24 ++++++------ .../linux/initialize-environment.sh | 8 ---- scripts/azure-pipelines/osx/azure-pipelines.yml | 10 +++-- .../azure-pipelines/osx/initialize-environment.sh | 17 +-------- .../azure-pipelines/windows/azure-pipelines.yml | 21 +++++------ .../windows/initialize-environment.ps1 | 44 ---------------------- 6 files changed, 28 insertions(+), 96 deletions(-) delete mode 100755 scripts/azure-pipelines/linux/initialize-environment.sh (limited to 'scripts/azure-pipelines') diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 022fc70e4..fc0977697 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -10,28 +10,26 @@ jobs: clean: resources timeoutInMinutes: 1440 # 1 day steps: - - bash: | - df -h - displayName: 'Report on Disk Space Before Environment' - condition: always() - - task: Bash@3 - displayName: 'Initialize Environment' - inputs: - filePath: scripts/azure-pipelines/linux/initialize-environment.sh + - bash: df -h + displayName: 'Report on Disk Space' - bash: | sudo mkdir /home/agent -m=777 sudo chown `id -u` /home/agent exit 0 displayName: 'Create /home/agent' + # Note: /mnt is the Azure machines' temporary disk. - bash: | - df -h - displayName: 'Report on Disk Space After Environment' - condition: always() + sudo mkdir /mnt/vcpkg-ci -m=777 + sudo mkdir /mnt/vcpkg-ci/downloads -m=777 + exit 0 + displayName: 'Create /mnt/vcpkg-ci/downloads' - task: Bash@3 displayName: 'Build vcpkg' inputs: filePath: bootstrap-vcpkg.sh arguments: "-buildTests" + env: + VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - powershell: | @@ -39,14 +37,14 @@ jobs: $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results $env:HOME = '/home/agent' - ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching + ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads displayName: '** Test Modified Ports **' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "archives/fail" -triplets "x64-linux" -errorOnRegression' + arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "/archives/fail" -triplets "x64-linux" -errorOnRegression' - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh deleted file mode 100755 index 1cbdd3326..000000000 --- a/scripts/azure-pipelines/linux/initialize-environment.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT - -# Cleans up the environment to prevent contamination across builds. -if [ ! -d "archives" ]; then - ln -s /archives archives -fi diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index e51698c45..be410e6a2 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -33,20 +33,22 @@ jobs: inputs: filePath: bootstrap-vcpkg.sh arguments: '-buildTests' + env: + VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - powershell: | - ./vcpkg x-ci-clean - $skip_list = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" + ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads + $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results - ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skip_list --binarycaching + ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads displayName: '** Test Modified Ports **' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir ~/Data/archives/fail -triplets "x64-osx" -errorOnRegression' + arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir /Users/vagrant/Data/archives/fail -triplets "x64-osx" -errorOnRegression' - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/osx/initialize-environment.sh b/scripts/azure-pipelines/osx/initialize-environment.sh index 6f42a809a..e94790668 100755 --- a/scripts/azure-pipelines/osx/initialize-environment.sh +++ b/scripts/azure-pipelines/osx/initialize-environment.sh @@ -4,20 +4,5 @@ # Sets up the environment for MacOS runs of vcpkg CI -mkdir -p ~/Data/installed || true -ln -s ~/Data/installed - -mkdir -p ~/Data/buildtrees || true -ln -s ~/Data/buildtrees - -mkdir -p ~/Data/packages || true -ln -s ~/Data/packages - -rm archives || rm -rf archives || true -ln -s ~/Data/archives - -mkdir -p ~/Data/downloads || true -ln -s ~/Data/downloads - #delete downloaded files that have not been used in 7 days -find downloads/ -maxdepth 1 -type f ! -atime 7 -exec rm -f {} \; +find ~/Data/downloads/ -maxdepth 1 -type f ! -atime 7 -exec rm -f {} \; diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 48f3908d2..f2b14657b 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -15,33 +15,32 @@ jobs: displayName: 'Initialize Environment' inputs: filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' - - task: BatchScript@1 + # Note: D: is the Azure machines' temporary disk. + - task: CmdLine@2 displayName: 'Build vcpkg' inputs: - filename: 'bootstrap-vcpkg.bat' + script: | + set VCPKG_DOWNLOADS=D:\downloads + .\bootstrap-vcpkg.bat - powershell: | - .\vcpkg.exe x-ci-clean - $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" + .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads + $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt mkdir $(System.ArtifactsDirectory)\xml-results # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed. # Install them so the CI succeeds: if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) { - .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows + .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads } - $command = ".\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching" + .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII - displayName: 'Write Test Modified Ports Batch File' - - task: BatchScript@1 displayName: '** Test Modified Ports **' - inputs: - filename: 'run_ci.cmd' - task: PowerShell@2 displayName: 'Analyze results and prepare test logs' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "archives\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"' + arguments: '-baselineFile .\scripts\ci.baseline.txt -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "W:\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"' - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs' inputs: diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1 index 4211a228a..4a252df49 100644 --- a/scripts/azure-pipelines/windows/initialize-environment.ps1 +++ b/scripts/azure-pipelines/windows/initialize-environment.ps1 @@ -22,13 +22,6 @@ Param( $StorageAccountName = $env:StorageAccountName $StorageAccountKey = $env:StorageAccountKey -function Remove-DirectorySymlink { - Param([string]$Path) - if (Test-Path $Path) { - [System.IO.Directory]::Delete($Path, $true) - } -} - Write-Host 'Setting up archives mount' if (-Not (Test-Path W:)) { net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey @@ -52,40 +45,3 @@ if( Test-Path D:\downloads\tools\msys2 ) Write-Host "removing previously installed msys2" Remove-Item D:\downloads\tools\msys2 -Recurse -Force } - -Write-Host 'Setting up archives path...' -if ([string]::IsNullOrWhiteSpace($ForceAllPortsToRebuildKey)) -{ - $archivesPath = 'W:\' -} -else -{ - $archivesPath = "W:\force\$ForceAllPortsToRebuildKey" - if (-Not (Test-Path $fullPath)) { - Write-Host 'Creating $archivesPath' - mkdir $archivesPath - } -} - -Write-Host "Linking archives => $archivesPath" -if (-Not (Test-Path archives)) { - cmd /c "mklink /D archives $archivesPath" -} - -Write-Host 'Linking installed => E:\installed' -if (-Not (Test-Path E:\installed)) { - mkdir E:\installed -} - -if (-Not (Test-Path installed)) { - cmd /c "mklink /D installed E:\installed" -} - -Write-Host 'Linking downloads => D:\downloads' -if (-Not (Test-Path D:\downloads)) { - mkdir D:\downloads -} - -if (-Not (Test-Path downloads)) { - cmd /c "mklink /D downloads D:\downloads" -} -- cgit v1.2.3