[vcpkg] [llvm] [mpir] Bump Linux VM memory size and do all operations on the temporary disk. (#11174)

* [vcpkg] Bump Linux VM memory size and do all operations on the temporary disk.

* [llvm] disable optional dependencies to libxml2 and zlib

* [vcpkg] Reduce quotas on the share to something reasonable.

* [mpir] Skip on Linux because it conflicts with libgmp.

Co-authored-by: yurybura <yurybura@gmail.com>

8 files changed, 36 insertions, 43 deletions

diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml
index 7b6274845..a0d9f3953 100644
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@@ -3,7 +3,7 @@
 #

 variables:

   windows-pool: 'PrWin-2020-04-28'

-  linux-pool: 'PrLin-2020-04-30'

+  linux-pool: 'PrLin-2020-05-07'

 

 jobs:

   - template: windows/azure-pipelines.yml

diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml
index 3acf736e1..9ecee265d 100644
--- a/scripts/azure-pipelines/clean-tombstones.yml
+++ b/scripts/azure-pipelines/clean-tombstones.yml
@@ -3,7 +3,7 @@
 #

 variables:

   windows-pool: 'PrWin-2020-04-28'

-  linux-pool: 'PrLin-2020-04-30'

+  linux-pool: 'PrLin-2020-05-07'

 

 jobs:

   - job: windows

diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
index 085c6c3be..58e82efef 100644
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -27,8 +27,7 @@ jobs:
   - powershell: |

       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"

       mkdir $(System.ArtifactsDirectory)/xml-results

-      mkdir /ci/home

-      $env:HOME = '/ci/home'

+      $env:HOME = '/home/root'

       ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching

     displayName: '** Test Modified Ports **'

   - task: PowerShell@2

diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1
index dfe2102d9..ac438254a 100755
--- a/scripts/azure-pipelines/linux/create-vmss.ps1
+++ b/scripts/azure-pipelines/linux/create-vmss.ps1
@@ -18,10 +18,9 @@ or are running from Azure Cloud Shell.
 

 $Location = 'westus2'

 $Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd')

-$VMSize = 'Standard_F16s_v2'

+$VMSize = 'Standard_D16a_v4'

 $ProtoVMName = 'PROTOTYPE'

 $LiveVMPrefix = 'BUILD'

-$InstalledDiskSizeInGB = 1024

 $ErrorActionPreference = 'Stop'

 

 $ProgressActivity = 'Creating Scale Set'

@@ -259,7 +258,7 @@ $StorageContext = New-AzStorageContext `
   -StorageAccountKey $StorageAccountKey

 

 New-AzStorageShare -Name 'archives' -Context $StorageContext

-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120

+Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024

 

 ####################################################################################################

 Write-Progress `

@@ -273,7 +272,7 @@ $Nic = New-AzNetworkInterface `
   -Location $Location `

   -Subnet $VirtualNetwork.Subnets[0]

 

-$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize

+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1

 $VM = Set-AzVMOperatingSystem `

   -VM $VM `

   -Linux `

@@ -377,8 +376,7 @@ $Vmss = Set-AzVmssOsProfile `
   -VirtualMachineScaleSet $Vmss `

   -ComputerNamePrefix $LiveVMPrefix `

   -AdminUsername AdminUser `

-  -AdminPassword $AdminPW `

-  -CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n")))

+  -AdminPassword $AdminPW

 

 $Vmss = Set-AzVmssStorageProfile `

   -VirtualMachineScaleSet $Vmss `

@@ -386,14 +384,6 @@ $Vmss = Set-AzVmssStorageProfile `
   -OsDiskCaching ReadWrite `

   -ImageReferenceId $Image.Id

 

-$Vmss = Add-AzVmssDataDisk `

-  -VirtualMachineScaleSet $Vmss `

-  -Lun 0 `

-  -Caching 'ReadWrite' `

-  -CreateOption Empty `

-  -DiskSizeGB 1024 `

-  -StorageAccountType 'StandardSSD_LRS'

-

 New-AzVmss `

   -ResourceGroupName $ResourceGroupName `

   -Name $VmssName `

diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh
index 82b52bd2e..5346345a5 100755
--- a/scripts/azure-pipelines/linux/initialize-environment.sh
+++ b/scripts/azure-pipelines/linux/initialize-environment.sh
@@ -4,12 +4,9 @@
 
 # Cleans up the environment to prevent contamination across builds.
 if [ ! -d "archives" ]; then
-    ln -s /ci/archives archives
-fi
-if [ ! -d "installed" ]; then
-    ln -s /ci/installed installed
+    ln -s /archives archives
 fi
 
-rm -rf installed/*
+rm -rf installed
 rm -rf buildtrees
-rm -rf packages
+rm -rf packages/*
diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh
index dbe10cf78..54e6c55ac 100755
--- a/scripts/azure-pipelines/linux/provision-image.sh
+++ b/scripts/azure-pipelines/linux/provision-image.sh
@@ -50,19 +50,12 @@ echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null
 echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null
 sudo chmod 600 $smbCredentialFile
 
-# Write script to provision disks used by cloud-init
-echo "if [ ! -d \"/ci\" ]; then" > /etc/provision-disks.sh
-echo "sudo parted /dev/sdc mklabel gpt" >> /etc/provision-disks.sh
-echo "sudo parted /dev/sdc mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh
-echo "sudo mkfs -t ext4 /dev/sdc1" >> /etc/provision-disks.sh
-echo "sudo mkdir /ci -m=777" >> /etc/provision-disks.sh
-echo "sudo mkdir /ci/installed -m=777" >> /etc/provision-disks.sh
-echo "sudo mkdir /ci/archives -m=777" >> /etc/provision-disks.sh
-echo "echo \"/dev/sdc1 /ci/installed ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh
-echo "echo \"//$StorageAccountName.file.core.windows.net/archives /ci/archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh
-echo "sudo mount -a" >> /etc/provision-disks.sh
-echo "fi" >> /etc/provision-disks.sh
-sudo chmod 700 /etc/provision-disks.sh
+# Mount the archives SMB share to /archives
+sudo mkdir /archives -m=777
+echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab
+
+# Create 'home' directory for haskell stack bits that want this
+sudo mkdir -p /home/root -m=777
 
 # Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above
 sudo rm /etc/debian_version
@@ -71,3 +64,8 @@ sudo rm /etc/debian_version
 # https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux
 # (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs)
 apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60
+
+# Create work trees for the Azure Pipelines agent so that it puts its work tree into temporary storage.
+sudo chmod 777 /mnt
+sudo mkdir /agent -m=777
+sudo ln -s /mnt /agent/_work
diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1
index 8c05cd51d..6a3f01fc1 100644
--- a/scripts/azure-pipelines/windows/create-vmss.ps1
+++ b/scripts/azure-pipelines/windows/create-vmss.ps1
@@ -294,7 +294,7 @@ $StorageContext = New-AzStorageContext `
   -StorageAccountKey $StorageAccountKey

 

 New-AzStorageShare -Name 'archives' -Context $StorageContext

-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120

+Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048

 

 ####################################################################################################

 Write-Progress `

@@ -308,7 +308,7 @@ $Nic = New-AzNetworkInterface `
   -Location $Location `

   -Subnet $VirtualNetwork.Subnets[0]

 

-$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize

+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1

 $VM = Set-AzVMOperatingSystem `

   -VM $VM `

   -Windows `

diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
index 192325a80..19588471b 100644
--- a/scripts/ci.baseline.txt
+++ b/scripts/ci.baseline.txt
@@ -1006,7 +1006,6 @@ llgl:arm-uwp=fail
 llgl:x64-uwp=fail

 llvm:arm-uwp=fail

 llvm:arm64-windows=fail

-llvm:x64-linux=fail

 llvm:x64-uwp=fail

 lmdb:arm64-windows=fail

 lmdb:arm-uwp=fail

@@ -1134,7 +1133,17 @@ mpg123:x64-uwp=fail
 mpir:arm64-windows=fail

 mpir:arm-uwp=fail

 mpir:x64-uwp=fail

-

+# Building package gmp[core]:x64-linux... done

+# Installing package gmp[core]:x64-linux...

+# The following files are already installed in /mnt/1/s/installed/x64-linux and are in conflict with gmp:x64-linux

+#

+# Installed by mpir:x64-linux

+#     debug/lib/libgmp.a

+#     debug/lib/libgmp.la

+#     include/gmp.h

+#     lib/libgmp.a

+#     lib/libgmp.la

+mpir:x64-linux=skip

 #Conflicts with angle and qt-5base

 ms-angle:arm64-windows      = skip

 ms-angle:arm-uwp            = skip

@@ -1868,8 +1877,8 @@ usd:x64-linux=ignore
 usd:x86-windows=fail

 usrsctp:arm-uwp=fail

 usrsctp:x64-uwp=fail

-uthenticode:arm-uwp=fail
-uthenticode:x64-uwp=fail
+uthenticode:arm-uwp=fail

+uthenticode:x64-uwp=fail

 uvatlas:arm64-windows=fail

 uvatlas:arm-uwp=fail

 uvatlas:x64-linux=fail