From 229f537c9414da5a577484b467a1f7ab0f4d8f34 Mon Sep 17 00:00:00 2001
From: Billy O'Neal <bion@microsoft.com>
Date: Tue, 22 Dec 2020 14:52:58 -0800
Subject: [opentracing] Update VMs 2020-12 (#15151)

* Add meson from https://github.com/microsoft/vcpkg/pull/12860/

* Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/

* Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/

* Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/

* Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/

* Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774

* Remove no longer necessary apt-mark calls.

* Update nasm on Linux.

* Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share

* Delete no longer necessary azure storage firewall rules.

* Install the newer Windows SDK with the VS installer instead of manually.

* Install the VS2015 and VS2017 compilers.

* Update Powershell-Core to 7.1.0.

* Update source of WDK.

* Update pools.

* [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled.

Fixes:
C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001).

Fixes:
D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error]
    try {
    ^

* [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX.

Note that this port sets /GS-, possibly because it may be for authoring shellcode.

* Bump storage API version to 2020-04-08.
---
 scripts/azure-pipelines/linux/azure-pipelines.yml |  2 +-
 scripts/azure-pipelines/linux/create-vmss.ps1     | 98 +++++++++++++++--------
 scripts/azure-pipelines/linux/provision-image.sh  | 42 +++-------
 3 files changed, 80 insertions(+), 62 deletions(-)

(limited to 'scripts/azure-pipelines/linux')

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index d7bd09fe2..018203a6a 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -41,7 +41,7 @@ jobs:
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -UseEnvironmentSasToken -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'
diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1
index 169a80fc9..55484f29b 100755
--- a/scripts/azure-pipelines/linux/create-vmss.ps1
+++ b/scripts/azure-pipelines/linux/create-vmss.ps1
@@ -64,7 +64,9 @@ Write-Progress `
   -Status 'Creating virtual network' `
   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
 
-$allowHttp = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules = @()
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
   -Name AllowHTTP `
   -Description 'Allow HTTP(S)' `
   -Access Allow `
@@ -76,49 +78,49 @@ $allowHttp = New-AzNetworkSecurityRuleConfig `
   -DestinationAddressPrefix * `
   -DestinationPortRange @(80, 443)
 
-$allowDns = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowSFTP `
+  -Description 'Allow (S)FTP' `
+  -Access Allow `
+  -Protocol Tcp `
+  -Direction Outbound `
+  -Priority 1009 `
+  -SourceAddressPrefix * `
+  -SourcePortRange * `
+  -DestinationAddressPrefix * `
+  -DestinationPortRange @(21, 22)
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
   -Name AllowDNS `
   -Description 'Allow DNS' `
   -Access Allow `
   -Protocol * `
   -Direction Outbound `
-  -Priority 1009 `
+  -Priority 1010 `
   -SourceAddressPrefix * `
   -SourcePortRange * `
   -DestinationAddressPrefix * `
   -DestinationPortRange 53
 
-$allowGit = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
   -Name AllowGit `
   -Description 'Allow git' `
   -Access Allow `
   -Protocol Tcp `
   -Direction Outbound `
-  -Priority 1010 `
+  -Priority 1011 `
   -SourceAddressPrefix * `
   -SourcePortRange * `
   -DestinationAddressPrefix * `
   -DestinationPortRange 9418
 
-$allowStorage = New-AzNetworkSecurityRuleConfig `
-  -Name AllowStorage `
-  -Description 'Allow Storage' `
-  -Access Allow `
-  -Protocol * `
-  -Direction Outbound `
-  -Priority 1011 `
-  -SourceAddressPrefix VirtualNetwork `
-  -SourcePortRange * `
-  -DestinationAddressPrefix Storage `
-  -DestinationPortRange *
-
-$denyEverythingElse = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
   -Name DenyElse `
   -Description 'Deny everything else' `
   -Access Deny `
   -Protocol * `
   -Direction Outbound `
-  -Priority 1012 `
+  -Priority 1013 `
   -SourceAddressPrefix * `
   -SourcePortRange * `
   -DestinationAddressPrefix * `
@@ -129,13 +131,14 @@ $NetworkSecurityGroup = New-AzNetworkSecurityGroup `
   -Name $NetworkSecurityGroupName `
   -ResourceGroupName $ResourceGroupName `
   -Location $Location `
-  -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
+  -SecurityRules $allFirewallRules
 
 $SubnetName = $ResourceGroupName + 'Subnet'
 $Subnet = New-AzVirtualNetworkSubnetConfig `
   -Name $SubnetName `
   -AddressPrefix "10.0.0.0/16" `
-  -NetworkSecurityGroup $NetworkSecurityGroup
+  -NetworkSecurityGroup $NetworkSecurityGroup `
+  -ServiceEndpoint "Microsoft.Storage"
 
 $VirtualNetworkName = $ResourceGroupName + 'Network'
 $VirtualNetwork = New-AzVirtualNetwork `
@@ -170,8 +173,31 @@ $StorageContext = New-AzStorageContext `
   -StorageAccountName $StorageAccountName `
   -StorageAccountKey $StorageAccountKey
 
-New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024
+New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
+$StartTime = [DateTime]::Now
+$ExpiryTime = $StartTime.AddMonths(6)
+
+$SasToken = New-AzStorageAccountSASToken `
+  -Service Blob `
+  -Permission "racwdlup" `
+  -Context $StorageContext `
+  -StartTime $StartTime `
+  -ExpiryTime $ExpiryTime `
+  -ResourceType Service,Container,Object `
+  -Protocol HttpsOnly
+
+$SasToken = $SasToken.Substring(1) # strip leading ?
+
+# Note that we put the storage account into the firewall after creating the above SAS token or we
+# would be denied since the person running this script isn't one of the VMs we're creating here.
+Set-AzStorageAccount `
+  -ResourceGroupName $ResourceGroupName `
+  -AccountName $StorageAccountName `
+  -NetworkRuleSet ( `
+    @{bypass="AzureServices"; `
+    virtualNetworkRules=( `
+      @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
+    defaultAction="Deny"})
 
 ####################################################################################################
 Write-Progress `
@@ -220,15 +246,23 @@ Write-Progress `
   -Status 'Running provisioning script provision-image.sh in VM' `
   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
 
-$ProvisionImageResult = Invoke-AzVMRunCommand `
-  -ResourceGroupName $ResourceGroupName `
-  -VMName $ProtoVMName `
-  -CommandId 'RunShellScript' `
-  -ScriptPath "$PSScriptRoot\provision-image.sh" `
-  -Parameter @{StorageAccountName=$StorageAccountName; `
-    StorageAccountKey=$StorageAccountKey;}
-
-Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+$tempScript = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".sh"
+try {
+  $script = Get-Content "$PSScriptRoot\provision-image.sh" -Encoding utf8NoBOM
+  $script += "echo `"PROVISIONED_AZURE_STORAGE_NAME=\`"$StorageAccountName\`"`" | sudo tee -a /etc/environment"
+  $script += "echo `"PROVISIONED_AZURE_STORAGE_SAS_TOKEN=\`"$SasToken\`"`" | sudo tee -a /etc/environment"
+  Set-Content -Path $tempScript -Value $script -Encoding utf8NoBOM
+
+  $ProvisionImageResult = Invoke-AzVMRunCommand `
+    -ResourceGroupName $ResourceGroupName `
+    -VMName $ProtoVMName `
+    -CommandId 'RunShellScript' `
+    -ScriptPath $tempScript
+
+  Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+} finally {
+  Remove-Item $tempScript -Recurse -Force
+}
 
 ####################################################################################################
 Write-Progress `
diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh
index 4936cf719..c04f4b3ca 100755
--- a/scripts/azure-pipelines/linux/provision-image.sh
+++ b/scripts/azure-pipelines/linux/provision-image.sh
@@ -11,11 +11,12 @@ APT_PACKAGES="at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils \
   libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev \
   libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml \
   flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
-  xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config"
+  xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \
+  pkg-config meson"
 
 # Additionally required by qt5-base
 APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \
-  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev"
+  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev"
 
 # Additionally required by qt5-base for qt5-x11extras
 APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
@@ -26,31 +27,29 @@ APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
 # Additionally required by libhdfs3
 APT_PACKAGES="$APT_PACKAGES libkrb5-dev"
 
+# Additionally required by kf5windowsystem
+APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
+
 # Additionally required by mesa
 APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"
 
 # Additionally required by some packages to install additional python packages
 APT_PACKAGES="$APT_PACKAGES python3-pip"
 
+# Additionally required by rtaudio
+APT_PACKAGES="$APT_PACKAGES libasound2-dev"
+
 # Additionally required/installed by Azure DevOps Scale Set Agents
 APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60"
 
 sudo apt -y install $APT_PACKAGES
 
-# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from
-# removing some of the above
-sudo apt-mark hold libcurl4
-sudo apt-mark hold liblttng-ust0
-sudo apt-mark hold libkrb5-3
-sudo apt-mark hold zlib1g
-sudo apt-mark hold libicu60
-
 # Install newer version of nasm than the apt package, required by intel-ipsec
 mkdir /tmp/nasm
 cd /tmp/nasm
-curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz
-tar -xf nasm-2.14.02.tar.gz
-cd nasm-2.14.02/
+curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz
+tar -xf nasm-2.15.05.tar.gz
+cd nasm-2.15.05/
 ./configure --prefix=/usr && make -j
 sudo make install
 cd ~
@@ -75,19 +74,4 @@ sudo apt update
 sudo add-apt-repository universe
 sudo apt install -y powershell
 
-if [ -z "$StorageAccountName" ]; then
-echo "No storage account supplied, skipping."
-else
-echo "Mapping storage account"
-
-# Write SMB credentials
-sudo mkdir /etc/smbcredentials
-smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred
-echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null
-echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null
-sudo chmod 600 $smbCredentialFile
-
-# Mount the archives SMB share to /archives
-sudo mkdir /archives -m=777
-echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab
-fi
+# provision-image.ps1 will append installation of the SAS token here
-- 
cgit v1.2.3