[vcpkg] Fix OSX CI by ensuring the downloads directory exists (#11839)

* [vcpkg] Remove do-nothing Set-Content from Windows azure-pipelines.yml.
* [vcpkg] Fix OSX CI by ensuring the downloads directory exists in advance, and extract common command line parameters with powershell splatting.
* [tensorflow-cc] Prevent hang building tensorflow-cc asking to configure iOS.
* Skip ignition-msgs5:x64-osx

6 files changed, 46 insertions, 39 deletions

diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml
index 9ecee265d..dc601bcfc 100644
--- a/scripts/azure-pipelines/clean-tombstones.yml
+++ b/scripts/azure-pipelines/clean-tombstones.yml
@@ -14,27 +14,19 @@ jobs:
       displayName: 'Initialize Environment'

       inputs:

         filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'

-    - script: rmdir /s /q archives\fail

-      displayName: 'Delete archives\fail'

+    - script: rmdir /s /q W:\fail

+      displayName: 'Delete W:\fail'

   - job: linux

     displayName: 'Clean Linux Tombstones'

     pool: $(linux-pool)

     steps:

-    - task: Bash@3

-      displayName: 'Initialize Environment'

-      inputs:

-        filePath: scripts/azure-pipelines/linux/initialize-environment.sh

-    - bash: rm -rf archives/fail

-      displayName: 'Delete archives/fail'

+    - bash: rm -rf /archives/fail

+      displayName: 'Delete /archives/fail'

   - job: osx

     displayName: 'Clean MacOS Tombstones'

     pool:

       name: vcpkgAgentPool

       demands: Agent.OS -equals Darwin

     steps:

-    - task: Bash@3

-      displayName: 'Initialize Environment'

-      inputs:

-        filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'

-    - bash: rm -rf archives/fail

-      displayName: 'Delete archives/fail'

+    - bash: rm -rf /Users/vagrant/Data/archives/fail

+      displayName: 'Delete /Users/vagrant/Data/archives/fail'

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index 83a634919..1e2a653e0 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -33,11 +33,20 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - powershell: |

-      ./vcpkg x-ci-clean --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages

+      $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads'

+      $commonArgs = @(

+        '--binarycaching',

+        '--x-binarysource=clear;files,/archives,upload',

+        '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees',

+        '--x-install-root=/mnt/vcpkg-ci/install',

+        '--x-packages-root=/mnt/vcpkg-ci/packages'

+      )

+

+      ./vcpkg x-ci-clean @commonArgs

       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

       mkdir $(System.ArtifactsDirectory)/xml-results

       $env:HOME = '/home/agent'

-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads

+      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
index a73a102a4..a24ed7d95 100644
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -13,12 +13,8 @@ jobs:
   steps:

   - bash: |

       df -h

-    displayName: 'Report on Disk Space Before Environment'

+    displayName: 'Report on Disk Space'

     condition: always()

-  - task: Bash@3

-    displayName: 'Initialize Environment'

-    inputs:

-      filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'

   - bash: |

       brew list autoconf || brew install autoconf

       brew list automake || brew install automake

@@ -26,9 +22,10 @@ jobs:
       brew list gfortran || brew cask install gfortran

     displayName: 'Install brew dependencies'

   - bash: |

-      df -h

-    displayName: 'Report on Disk Space After Environment'

-    condition: always()

+      sudo mkdir /Users/vagrant/Data/downloads || 0

+      sudo chmod 777 /Users/vagrant/Data/downloads || 0

+      exit 0

+    displayName: 'Create /Users/vagrant/Data/downloads'

   - task: Bash@3

     displayName: 'Build vcpkg'

     inputs:

@@ -39,10 +36,19 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test

     displayName: 'Run vcpkg tests'

   - powershell: |

-      ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads

+      $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads'

+      $commonArgs = @(

+        '--binarycaching',

+        '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload',

+        '--x-buildtrees-root=/Users/vagrant/Data/buildtrees',

+        '--x-install-root=/Users/vagrant/Data/installed',

+        '--x-packages-root=/Users/vagrant/Data/packages'

+      )

+

+      ./vcpkg x-ci-clean @commonArgs

       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

       mkdir $(System.ArtifactsDirectory)/xml-results

-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads

+      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/azure-pipelines/osx/initialize-environment.sh b/scripts/azure-pipelines/osx/initialize-environment.sh
deleted file mode 100755
index e94790668..000000000
--- a/scripts/azure-pipelines/osx/initialize-environment.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: MIT
-
-# Sets up the environment for MacOS runs of vcpkg CI
-
-#delete downloaded files that have not been used in 7 days
-find ~/Data/downloads/  -maxdepth 1 -type f ! -atime 7  -exec rm -f {} \;
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index f2b14657b..24089b674 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -23,17 +23,25 @@ jobs:
         set VCPKG_DOWNLOADS=D:\downloads

         .\bootstrap-vcpkg.bat

   - powershell: |

-      .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

+      $env:VCPKG_DOWNLOADS='D:\downloads'

+      $commonArgs = @(

+        '--binarycaching',

+        '--x-binarysource=clear;files,W:\,upload',

+        '--x-buildtrees-root=E:\buildtrees',

+        '--x-install-root=E:\install',

+        '--x-packages-root=E:\packages'

+      )

+

+      .\vcpkg.exe x-ci-clean @commonArgs

       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt

       mkdir $(System.ArtifactsDirectory)\xml-results

       # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.

       # Install them so the CI succeeds:

       if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {

-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

+          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs

       }

 

-      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads

-      Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII

+      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
index 13a6e9943..df31d1489 100644
--- a/scripts/ci.baseline.txt
+++ b/scripts/ci.baseline.txt
@@ -621,7 +621,7 @@ ignition-msgs5:x64-linux=fail
 ignition-msgs5:arm64-windows=fail

 ignition-msgs5:arm-uwp=fail

 ignition-msgs5:x64-uwp=fail

-ignition-msgs5:x64-osx=fail

+ignition-msgs5:x64-osx=skip

 imgui-sfml:x64-linux=ignore

 intel-ipsec:arm64-windows=fail

 intel-ipsec:arm-uwp=fail