[opentracing] Update VMs 2020-12 (#15151)

* Add meson from https://github.com/microsoft/vcpkg/pull/12860/

* Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/

* Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/

* Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/

* Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/

* Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774

* Remove no longer necessary apt-mark calls.

* Update nasm on Linux.

* Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share

* Delete no longer necessary azure storage firewall rules.

* Install the newer Windows SDK with the VS installer instead of manually.

* Install the VS2015 and VS2017 compilers.

* Update Powershell-Core to 7.1.0.

* Update source of WDK.

* Update pools.

* [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled.

Fixes:
C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001).

Fixes:
D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error]
    try {
    ^

* [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX.

Note that this port sets /GS-, possibly because it may be for authoring shellcode.

* Bump storage API version to 2020-04-08.

4 files changed, 69 insertions, 135 deletions

diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
index 069c2e095..b9bf49e22 100644
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -17,11 +17,6 @@ jobs:
 

   steps:

   - task: PowerShell@2

-    displayName: 'Initialize Environment'

-    inputs:

-      filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'

-      pwsh: true

-  - task: PowerShell@2

     displayName: 'Report on Disk Space'

     condition: always()

     inputs:

@@ -57,7 +52,7 @@ jobs:
     inputs:

       failOnStderr: true

       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'

-      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'

+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -UseEnvironmentSasToken -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'

       pwsh: true

   - task: PowerShell@2

     displayName: 'Report on Disk Space After Build'

diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1
index 1fcec50ed..608465100 100644
--- a/scripts/azure-pipelines/windows/create-vmss.ps1
+++ b/scripts/azure-pipelines/windows/create-vmss.ps1
@@ -70,7 +70,9 @@ Write-Progress `
   -Status 'Creating virtual network' `

   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

 

-$allowHttp = New-AzNetworkSecurityRuleConfig `

+$allFirewallRules = @()

+

+$allFirewallRules += New-AzNetworkSecurityRuleConfig `

   -Name AllowHTTP `

   -Description 'Allow HTTP(S)' `

   -Access Allow `

@@ -82,75 +84,67 @@ $allowHttp = New-AzNetworkSecurityRuleConfig `
   -DestinationAddressPrefix * `

   -DestinationPortRange @(80, 443)

 

-$allowDns = New-AzNetworkSecurityRuleConfig `

+$allFirewallRules += New-AzNetworkSecurityRuleConfig `

+  -Name AllowSFTP `

+  -Description 'Allow (S)FTP' `

+  -Access Allow `

+  -Protocol Tcp `

+  -Direction Outbound `

+  -Priority 1009 `

+  -SourceAddressPrefix * `

+  -SourcePortRange * `

+  -DestinationAddressPrefix * `

+  -DestinationPortRange @(21, 22)

+

+$allFirewallRules += New-AzNetworkSecurityRuleConfig `

   -Name AllowDNS `

   -Description 'Allow DNS' `

   -Access Allow `

   -Protocol * `

   -Direction Outbound `

-  -Priority 1009 `

+  -Priority 1010 `

   -SourceAddressPrefix * `

   -SourcePortRange * `

   -DestinationAddressPrefix * `

   -DestinationPortRange 53

 

-$allowGit = New-AzNetworkSecurityRuleConfig `

+$allFirewallRules += New-AzNetworkSecurityRuleConfig `

   -Name AllowGit `

   -Description 'Allow git' `

   -Access Allow `

   -Protocol Tcp `

   -Direction Outbound `

-  -Priority 1010 `

+  -Priority 1011 `

   -SourceAddressPrefix * `

   -SourcePortRange * `

   -DestinationAddressPrefix * `

   -DestinationPortRange 9418

 

-if (-Not $Unstable) {

-  $allowStorage = New-AzNetworkSecurityRuleConfig `

-    -Name AllowStorage `

-    -Description 'Allow Storage' `

-    -Access Allow `

-    -Protocol * `

-    -Direction Outbound `

-    -Priority 1011 `

-    -SourceAddressPrefix VirtualNetwork `

-    -SourcePortRange * `

-    -DestinationAddressPrefix Storage `

-    -DestinationPortRange *

-}

-

-$denyEverythingElse = New-AzNetworkSecurityRuleConfig `

+$allFirewallRules += New-AzNetworkSecurityRuleConfig `

   -Name DenyElse `

   -Description 'Deny everything else' `

   -Access Deny `

   -Protocol * `

   -Direction Outbound `

-  -Priority 1012 `

+  -Priority 1013 `

   -SourceAddressPrefix * `

   -SourcePortRange * `

   -DestinationAddressPrefix * `

   -DestinationPortRange *

 

 $NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'

-$securityRules = @($allowHttp, $allowDns, $allowGit);

-if (-Not $Unstable) {

-  $securityRules += @($allowStorage)

-}

-

-$securityRules += @($denyEverythingElse)

-

 $NetworkSecurityGroup = New-AzNetworkSecurityGroup `

   -Name $NetworkSecurityGroupName `

   -ResourceGroupName $ResourceGroupName `

   -Location $Location `

-  -SecurityRules $securityRules

+  -SecurityRules $allFirewallRules

 

 $SubnetName = $ResourceGroupName + 'Subnet'

 $Subnet = New-AzVirtualNetworkSubnetConfig `

   -Name $SubnetName `

   -AddressPrefix "10.0.0.0/16" `

-  -NetworkSecurityGroup $NetworkSecurityGroup

+  -NetworkSecurityGroup $NetworkSecurityGroup `

+  -ServiceEndpoint "Microsoft.Storage"

 

 $VirtualNetworkName = $ResourceGroupName + 'Network'

 $VirtualNetwork = New-AzVirtualNetwork `

@@ -186,8 +180,31 @@ if (-Not $Unstable) {
     -StorageAccountName $StorageAccountName `

     -StorageAccountKey $StorageAccountKey

 

-  New-AzStorageShare -Name 'archives' -Context $StorageContext

-  Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048

+  New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off

+  $StartTime = [DateTime]::Now

+  $ExpiryTime = $StartTime.AddMonths(6)

+

+  $SasToken = New-AzStorageAccountSASToken `

+    -Service Blob `

+    -Permission "racwdlup" `

+    -Context $StorageContext `

+    -StartTime $StartTime `

+    -ExpiryTime $ExpiryTime `

+    -ResourceType Service,Container,Object `

+    -Protocol HttpsOnly

+

+  $SasToken = $SasToken.Substring(1) # strip leading ?

+

+  # Note that we put the storage account into the firewall after creating the above SAS token or we

+  # would be denied since the person running this script isn't one of the VMs we're creating here.

+  Set-AzStorageAccount `

+    -ResourceGroupName $ResourceGroupName `

+    -AccountName $StorageAccountName `

+    -NetworkRuleSet ( `

+      @{bypass="AzureServices"; `

+      virtualNetworkRules=( `

+        @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `

+      defaultAction="Deny"})

 }

 

 ####################################################################################################

@@ -234,7 +251,7 @@ Write-Progress `
 $provisionParameters = @{AdminUserPassword = $AdminPW;}

 if (-Not $Unstable) {

   $provisionParameters['StorageAccountName'] = $StorageAccountName

-  $provisionParameters['StorageAccountKey'] = $StorageAccountKey

+  $provisionParameters['StorageAccountSasToken'] = $SasToken

 }

 

 $ProvisionImageResult = Invoke-AzVMRunCommand `

diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1
deleted file mode 100644
index 24520802e..000000000
--- a/scripts/azure-pipelines/windows/initialize-environment.ps1
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright (c) Microsoft Corporation.

-# SPDX-License-Identifier: MIT

-#

-<#

-.SYNOPSIS

-Sets up the environment to run other vcpkg CI steps in an Azure Pipelines job.

-

-.DESCRIPTION

-This script maps network drives from infrastructure and cleans out anything that

-might have been leftover from a previous run.

-#>

-

-if ([string]::IsNullOrWhiteSpace($env:StorageAccountName) -or  [string]::IsNullOrWhiteSpace($env:StorageAccountKey)) {

-    Write-Host 'No storage account and/or key set, skipping mount of W:\'

-} else {

-    $StorageAccountName = $env:StorageAccountName

-    $StorageAccountKey = $env:StorageAccountKey

-

-    Write-Host 'Setting up archives mount'

-    if (-Not (Test-Path W:)) {

-        net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey

-    }

-}

-

-Write-Host 'Creating downloads directory'

-mkdir D:\downloads -ErrorAction SilentlyContinue

-

-# Delete entries in the downloads folder, except:

-#   those in the 'tools' folder

-#   those last accessed in the last 30 days

-Get-ChildItem -Path D:\downloads -Exclude "tools" `

-    | Where-Object{ $_.LastAccessTime -lt (get-Date).AddDays(-30) } `

-    | ForEach-Object{Remove-Item -Path $_ -Recurse -Force}

-

-# Msys sometimes leaves a database lock file laying around, especially if there was a failed job

-# which causes unrelated failures in jobs that run later on the machine.

-# work around this by just removing the vcpkg installed msys2 if it exists

-if( Test-Path D:\downloads\tools\msys2 )

-{

-    Write-Host "removing previously installed msys2"

-    Remove-Item D:\downloads\tools\msys2 -Recurse -Force

-}

diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt
index b43b1bf87..e3e04be28 100644
--- a/scripts/azure-pipelines/windows/provision-image.txt
+++ b/scripts/azure-pipelines/windows/provision-image.txt
@@ -20,17 +20,18 @@ The administrator user's password; if this is $null, or not passed, then the
 script assumes it's running on an administrator account.

 

 .PARAMETER StorageAccountName

-The name of the storage account. Stored in the environment variable %StorageAccountName%.

+The name of the storage account. Stored in the environment variable %PROVISIONED_AZURE_STORAGE_NAME%.

 Used by the CI system to access the global storage.

 

-.PARAMETER StorageAccountKey

-The key of the storage account. Stored in the environment variable %StorageAccountKey%.

+.PARAMETER StorageAccountSasToken

+The SAS token to access the storage account. Stored in the environment variable

+%PROVISIONED_AZURE_STORAGE_SAS_TOKEN%.

 Used by the CI system to access the global storage.

 #>

 param(

   [string]$AdminUserPassword = $null,

   [string]$StorageAccountName = $null,

-  [string]$StorageAccountKey = $null

+  [string]$StorageAccountSasToken = $null

 )

 

 $ErrorActionPreference = 'Stop'

@@ -88,9 +89,9 @@ if ([string]::IsNullOrEmpty($AdminUserPassword)) {
     $PsExecArgs += $StorageAccountName

   }

 

-  if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountKey))) {

-    $PsExecArgs += '-StorageAccountKey'

-    $PsExecArgs += $StorageAccountKey

+  if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountSasToken))) {

+    $PsExecArgs += '-StorageAccountSasToken'

+    $PsExecArgs += $StorageAccountSasToken

   }

 

   Write-Host "Executing: $PsExecPath $PsExecArgs"

@@ -114,15 +115,16 @@ $Workloads = @(
   'Microsoft.VisualStudio.Component.VC.ATLMFC',

   'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre',

   'Microsoft.VisualStudio.Component.Windows10SDK.18362',

+  'Microsoft.VisualStudio.Component.Windows10SDK.19041',

   'Microsoft.Net.Component.4.8.SDK',

   'Microsoft.Component.NetFX.Native',

   'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',

-  'Microsoft.VisualStudio.Component.VC.Llvm.Clang'

+  'Microsoft.VisualStudio.Component.VC.Llvm.Clang',

+  'Microsoft.VisualStudio.Component.VC.v141.x86.x64',

+  'Microsoft.VisualStudio.Component.VC.140'

 )

 

-$WindowsSDKUrl = 'https://download.microsoft.com/download/1/c/3/1c3d5161-d9e9-4e4b-9b43-b70fe8be268c/windowssdk/winsdksetup.exe'

-

-$WindowsWDKUrl = 'https://download.microsoft.com/download/1/a/7/1a730121-7aa7-46f7-8978-7db729aa413d/wdk/wdksetup.exe'

+$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854'

 

 $MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe'

 

@@ -133,7 +135,7 @@ $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_ad
   'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + `

   'occupancy_calculator_10.1 fortran_examples_10.1'

 

-$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.msi'

+$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi'

 

 $ErrorActionPreference = 'Stop'

 $ProgressPreference = 'SilentlyContinue'

@@ -296,43 +298,6 @@ Function InstallZip {
 

 <#

 .SYNOPSIS

-Installs Windows SDK version 2004

-

-.DESCRIPTION

-Downloads the Windows SDK installer located at $Url, and installs it with the

-correct flags.

-

-.PARAMETER Url

-The URL of the installer.

-#>

-Function InstallWindowsSDK {

-  Param(

-    [String]$Url

-  )

-

-  try {

-    Write-Host 'Downloading Windows SDK...'

-    [string]$installerPath = Get-TempFilePath -Extension 'exe'

-    curl.exe -L -o $installerPath -s -S $Url

-    Write-Host 'Installing Windows SDK...'

-    $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru

-    $exitCode = $proc.ExitCode

-    if ($exitCode -eq 0) {

-      Write-Host 'Installation successful!'

-    }

-    else {

-      Write-Error "Installation failed! Exited with $exitCode."

-      throw

-    }

-  }

-  catch {

-    Write-Error "Failed to install Windows SDK! $($_.Exception.Message)"

-    throw

-  }

-}

-

-<#

-.SYNOPSIS

 Installs Windows WDK version 2004

 

 .DESCRIPTION

@@ -467,7 +432,6 @@ if ($null -eq $av) {
 }

 

 InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable'

-InstallWindowsSDK -Url $WindowsSDKUrl

 InstallWindowsWDK -Url $WindowsWDKUrl

 InstallMpi -Url $MpiUrl

 InstallCuda -Url $CudaUrl -Features $CudaFeatures

@@ -477,14 +441,14 @@ if ([string]::IsNullOrWhiteSpace($StorageAccountName)) {
 } else {

   Write-Host 'Storing storage account name to environment'

   Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `

-    -Name StorageAccountName `

+    -Name PROVISIONED_AZURE_STORAGE_NAME `

     -Value $StorageAccountName

 }

-if ([string]::IsNullOrWhiteSpace($StorageAccountKey)) {

+if ([string]::IsNullOrWhiteSpace($StorageAccountSasToken)) {

   Write-Host 'No storage account key configured.'

 } else {

   Write-Host 'Storing storage account key to environment'

   Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `

-    -Name StorageAccountKey `

-    -Value $StorageAccountKey

+    -Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN `

+    -Value $StorageAccountSasToken

 }