[skia, ompl, capnproto] Update VMs for 2020-08-12 (including VS 16.7) (#12884)

Co-authored-by: Nicole Mazzuca <mazzucan@outlook.com>

8 files changed, 60 insertions, 72 deletions

diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml
index cda29f7e6..01810eb0f 100644
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: MIT

 #

 variables:

-  windows-pool: 'PrWin-2020-06-30'

-  linux-pool: 'PrLin-2020-06-30'

+  windows-pool: 'PrWin-2020-08-12'

+  linux-pool: 'PrLin-2020-08-12'

 

 stages:

 - stage: check_cxx_formatting

@@ -19,6 +19,7 @@ stages:
       inputs:

         filePath: 'scripts/azure-pipelines/windows/Check-CxxFormatting.ps1'

         arguments: '-Root .'

+        pwsh: true

 - stage: check_manifest_formatting

   displayName: Check the formatting of port manifests

   pool: $(windows-pool)

@@ -36,6 +37,7 @@ stages:
       inputs:

         filePath: 'scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1'

         arguments: '-Root .'

+        pwsh: true

 - stage: run_port_ci

   displayName: 'Run the Port CI'

   dependsOn:

diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1
index 84c83dfea..169a80fc9 100755
--- a/scripts/azure-pipelines/linux/create-vmss.ps1
+++ b/scripts/azure-pipelines/linux/create-vmss.ps1
@@ -220,7 +220,7 @@ Write-Progress `
   -Status 'Running provisioning script provision-image.sh in VM' `

   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

 

-Invoke-AzVMRunCommand `

+$ProvisionImageResult = Invoke-AzVMRunCommand `

   -ResourceGroupName $ResourceGroupName `

   -VMName $ProtoVMName `

   -CommandId 'RunShellScript' `

@@ -228,6 +228,8 @@ Invoke-AzVMRunCommand `
   -Parameter @{StorageAccountName=$StorageAccountName; `

     StorageAccountKey=$StorageAccountKey;}

 

+Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"

+

 ####################################################################################################

 Write-Progress `

   -Activity $ProgressActivity `

diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh
index 5a59161e6..4936cf719 100755
--- a/scripts/azure-pipelines/linux/provision-image.sh
+++ b/scripts/azure-pipelines/linux/provision-image.sh
@@ -27,7 +27,10 @@ APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
 APT_PACKAGES="$APT_PACKAGES libkrb5-dev"
 
 # Additionally required by mesa
-APT_PACKAGES="$APT_PACKAGES python3-setuptools"
+APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"
+
+# Additionally required by some packages to install additional python packages
+APT_PACKAGES="$APT_PACKAGES python3-pip"
 
 # Additionally required/installed by Azure DevOps Scale Set Agents
 APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60"
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index 09a95a519..bb8c7435c 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -20,11 +20,13 @@ jobs:
     displayName: 'Initialize Environment'

     inputs:

       filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'

+      pwsh: true

   - task: PowerShell@2

     displayName: 'Report on Disk Space'

     condition: always()

     inputs:

       filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'

+      pwsh: true

   # Note: D: is the Azure machines' temporary disk.

   - script: .\bootstrap-vcpkg.bat

     displayName: 'Build vcpkg'

@@ -49,17 +51,20 @@ jobs:
     inputs:

       filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'

       arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}'

+      pwsh: true

   - task: PowerShell@2

     displayName: '*** Test Modified Ports and Prepare Test Logs ***'

     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

       arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'

+      pwsh: true

   - task: PowerShell@2

     displayName: 'Report on Disk Space After Build'

     condition: always()

     inputs:

       filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'

+      pwsh: true

   - task: PublishBuildArtifacts@1

     displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'

     inputs:

@@ -74,6 +79,7 @@ jobs:
       script: |

         ./vcpkg.exe fetch python3

         & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\

+      pwsh: true

   - task: PublishBuildArtifacts@1

     displayName: 'Publish Artifact: file lists for ${{ parameters.triplet }}'

     condition: always()

diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1
index b1aa5d0ce..3bb89ee25 100644
--- a/scripts/azure-pipelines/windows/create-vmss.ps1
+++ b/scripts/azure-pipelines/windows/create-vmss.ps1
@@ -198,7 +198,7 @@ Write-Progress `
   -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' `

   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

 

-Invoke-AzVMRunCommand `

+$ProvisionImageResult = Invoke-AzVMRunCommand `

   -ResourceGroupName $ResourceGroupName `

   -VMName $ProtoVMName `

   -CommandId 'RunPowerShellScript' `

@@ -207,6 +207,8 @@ Invoke-AzVMRunCommand `
     StorageAccountName=$StorageAccountName; `

     StorageAccountKey=$StorageAccountKey;}

 

+Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)"

+

 ####################################################################################################

 Write-Progress `

   -Activity $ProgressActivity `

@@ -221,12 +223,14 @@ Write-Progress `
   -Status 'Running provisioning script sysprep.ps1 in VM' `

   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

 

-Invoke-AzVMRunCommand `

+$SysprepResult = Invoke-AzVMRunCommand `

   -ResourceGroupName $ResourceGroupName `

   -VMName $ProtoVMName `

   -CommandId 'RunPowerShellScript' `

   -ScriptPath "$PSScriptRoot\sysprep.ps1"

 

+Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)"

+

 ####################################################################################################

 Write-Progress `

   -Activity $ProgressActivity `

diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt
index 798b96e9a..b43b1bf87 100644
--- a/scripts/azure-pipelines/windows/provision-image.txt
+++ b/scripts/azure-pipelines/windows/provision-image.txt
@@ -7,7 +7,7 @@ Sets up a machine to be an image for a scale set.
 

 .DESCRIPTION

 provision-image.ps1 runs on an existing, freshly provisioned virtual machine,

-and sets that virtual machine up as a vcpkg build machine. After this is done,

+and sets up that virtual machine as a build machine. After this is done,

 (outside of this script), we take that machine and make it an image to be copied

 for setting up new VMs in the scale set.

 

@@ -60,10 +60,14 @@ Function Get-TempFilePath {
   return Join-Path $tempPath $tempName

 }

 

-if (-not [string]::IsNullOrEmpty($AdminUserPassword)) {

-  Write-Host "AdminUser password supplied; switching to AdminUser"

+$TranscriptPath = 'C:\provision-image-transcript.txt'

+

+if ([string]::IsNullOrEmpty($AdminUserPassword)) {

+  Start-Transcript -Path $TranscriptPath

+} else {

+  Write-Host 'AdminUser password supplied; switching to AdminUser.'

   $PsExecPath = Get-TempFilePath -Extension 'exe'

-  Write-Host "Downloading psexec to $PsExecPath"

+  Write-Host "Downloading psexec to: $PsExecPath"

   & curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe

   $PsExecArgs = @(

     '-u',

@@ -89,8 +93,11 @@ if (-not [string]::IsNullOrEmpty($AdminUserPassword)) {
     $PsExecArgs += $StorageAccountKey

   }

 

-  Write-Host "Executing $PsExecPath " + @PsExecArgs

-  & $PsExecPath @PsExecArgs > C:\ProvisionLog.txt

+  Write-Host "Executing: $PsExecPath $PsExecArgs"

+

+  $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru

+  Write-Host 'Reading transcript...'

+  Get-Content -Path $TranscriptPath

   Write-Host 'Cleaning up...'

   Remove-Item $PsExecPath

   exit $proc.ExitCode

@@ -118,7 +125,6 @@ $WindowsSDKUrl = 'https://download.microsoft.com/download/1/c/3/1c3d5161-d9e9-4e
 $WindowsWDKUrl = 'https://download.microsoft.com/download/1/a/7/1a730121-7aa7-46f7-8978-7db729aa413d/wdk/wdksetup.exe'

 

 $MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe'

-$LlvmUrl = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/LLVM-10.0.0-win64.exe'

 

 $CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe'

 $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + `

@@ -127,7 +133,7 @@ $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_ad
   'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + `

   'occupancy_calculator_10.1 fortran_examples_10.1'

 

-$BinSkimUrl = 'https://www.nuget.org/api/v2/package/Microsoft.CodeAnalysis.BinSkim/1.6.0'

+$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.msi'

 

 $ErrorActionPreference = 'Stop'

 $ProgressPreference = 'SilentlyContinue'

@@ -191,7 +197,7 @@ Function InstallVisualStudio {
     Write-Host 'Downloading Visual Studio...'

     [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe'

     curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl

-    Write-Host "Installing Visual Studio..."

+    Write-Host 'Installing Visual Studio...'

     $args = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache')

     foreach ($workload in $Workloads) {

       $args += '--add'

@@ -364,59 +370,6 @@ Function InstallWindowsWDK {
 

 <#

 .SYNOPSIS

-Installs LLVM.

-

-.DESCRIPTION

-InstallLLVM installs LLVM from the supplied URL.

-

-.PARAMETER Url

-The URL of the LLVM installer.

-#>

-Function InstallLLVM {

-  Param(

-    [String]$Url

-  )

-

-  try {

-    Write-Host 'Downloading LLVM...'

-    [string]$installerPath = Get-TempFilePath -Extension 'exe'

-    curl.exe -L -o $installerPath -s -S $Url

-    Write-Host 'Installing LLVM...'

-    $proc = Start-Process -FilePath $installerPath -ArgumentList @('/S') -NoNewWindow -Wait -PassThru

-    PrintMsiExitCodeMessage $proc.ExitCode

-  }

-  catch {

-    Write-Error "Failed to install LLVM! $($_.Exception.Message)"

-    throw

-  }

-}

-

-<#

-.SYNOPSIS

-Installs LLVM.

-

-.DESCRIPTION

-InstallLLVM installs LLVM from the supplied URL.

-

-.PARAMETER Url

-The URL of the LLVM installer.

-#>

-Function InstallLLVM {

-  try {

-    Write-Host 'Downloading LLVM...'

-    [string]$installerPath = Get-TempFilePath -Extension 'exe'

-    curl.exe -L -o $installerPath -s -S $Url

-    Write-Host 'Installing LLVM...'

-    $proc = Start-Process -FilePath $installerPath -ArgumentList @('/S') -NoNewWindow -Wait -PassThru

-    PrintMsiExitCodeMessage $proc.ExitCode

-  }

-  catch {

-    Write-Error "Failed to install LLVM! $($_.Exception.Message)"

-  }

-}

-

-<#

-.SYNOPSIS

 Installs MPI

 

 .DESCRIPTION

@@ -458,7 +411,7 @@ Installs NVIDIA's CUDA Toolkit.
 

 .DESCRIPTION

 InstallCuda installs the CUDA Toolkit with the features specified as a

-space separated list of strings in $Features.

+space-separated list of strings in $Features.

 

 .PARAMETER Url

 The URL of the CUDA installer.

@@ -504,6 +457,7 @@ if ($null -eq $av) {
   Write-Host 'AntiVirus not installed, skipping exclusions.'

 } else {

   Write-Host 'Configuring AntiVirus exclusions...'

+  Add-MpPreference -ExclusionPath C:\agent

   Add-MPPreference -ExclusionPath D:\

   Add-MPPreference -ExclusionProcess ninja.exe

   Add-MPPreference -ExclusionProcess clang-cl.exe

@@ -515,10 +469,9 @@ if ($null -eq $av) {
 InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable'

 InstallWindowsSDK -Url $WindowsSDKUrl

 InstallWindowsWDK -Url $WindowsWDKUrl

-InstallLLVM -Url $LlvmUrl

 InstallMpi -Url $MpiUrl

 InstallCuda -Url $CudaUrl -Features $CudaFeatures

-InstallZip -Url $BinSkimUrl -Name 'BinSkim' -Dir 'C:\BinSkim'

+InstallMSI -Url $PwshUrl -Name 'PowerShell Core'

 if ([string]::IsNullOrWhiteSpace($StorageAccountName)) {

   Write-Host 'No storage account name configured.'

 } else {

diff --git a/scripts/azure-pipelines/windows/sysprep.ps1 b/scripts/azure-pipelines/windows/sysprep.ps1
index c0965350d..a29950044 100644
--- a/scripts/azure-pipelines/windows/sysprep.ps1
+++ b/scripts/azure-pipelines/windows/sysprep.ps1
@@ -14,4 +14,4 @@ for more information.
 

 $ErrorActionPreference = 'Stop'

 Write-Host 'Running sysprep'

-& C:\Windows\system32\sysprep\sysprep.exe /oobe /generalize /shutdown

+& C:\Windows\system32\sysprep\sysprep.exe /oobe /generalize /mode:vm /shutdown

diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
index 55f4c33c4..38b5db2d9 100644
--- a/scripts/ci.baseline.txt
+++ b/scripts/ci.baseline.txt
@@ -153,6 +153,11 @@ caf:x64-uwp=fail
 caffe2:x86-windows=fail
 caffe2:arm64-windows=fail
 cairomm:x64-linux=fail
+# capnproto currently triggers an ICE
+capnproto:x86-windows=fail
+capnproto:x64-windows=fail
+capnproto:x64-windows-static=fail
+# capnproto doesn't support arm, arm64, or UWP
 capnproto:arm64-windows=fail
 capnproto:arm-uwp=fail
 capnproto:x64-uwp=fail
@@ -517,6 +522,8 @@ gmmlib:x64-uwp=fail
 gmmlib:x64-windows=fail
 gmmlib:x64-windows-static=fail
 gmmlib:x86-windows=fail
+# VS 2019 16.7 broke gmp UWP https://github.com/microsoft/vcpkg/issues/13172
+gmp:x64-uwp=fail
 google-cloud-cpp:arm64-windows=fail
 google-cloud-cpp:arm-uwp=fail
 google-cloud-cpp:x64-uwp=fail
@@ -1181,6 +1188,17 @@ ogre-next:x86-windows        = skip
 ois:arm64-windows=fail
 ois:arm-uwp=fail
 ois:x64-uwp=fail
+# ompl is vulnerable to some form of race in its dependent ports, and adding 'ode' as a dependency
+# does not resolve the issue
+# src/ompl/CMakeFiles/ompl.dir/extensions/ode/src/OpenDEStateValidityChecker.cpp.o
+# -L/mnt/vcpkg-ci/packages/flann_x64-linux/debug/lib   -L/mnt/vcpkg-ci/packages/ode_x64-linux/debug/lib
+# -Wl,-rpath,/mnt/vcpkg-ci/packages/flann_x64-linux/debug/lib:/mnt/vcpkg-ci/packages/ode_x64-linux/debug/lib::::::::::::::::::::::::::::::::::::::::::::::::
+# -lode  /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_serialization.a 
+# /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_filesystem.a 
+# /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_system.a  -lpthread && :
+# /usr/bin/ld: cannot find -lode
+ompl:x64-osx=fail
+ompl:x64-linux=fail
 open62541:arm-uwp=fail
 open62541:x64-uwp=fail
 openal-soft:arm-uwp=fail