From 6920b49047f2de6880df8ef9724cc508c5f491ca Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Sun, 10 May 2020 15:17:27 -0700 Subject: [vcpkg] [llvm] [mpir] Bump Linux VM memory size and do all operations on the temporary disk. (#11174) * [vcpkg] Bump Linux VM memory size and do all operations on the temporary disk. * [llvm] disable optional dependencies to libxml2 and zlib * [vcpkg] Reduce quotas on the share to something reasonable. * [mpir] Skip on Linux because it conflicts with libgmp. Co-authored-by: yurybura --- scripts/azure-pipelines/azure-pipelines.yml | 2 +- scripts/azure-pipelines/clean-tombstones.yml | 2 +- scripts/azure-pipelines/linux/azure-pipelines.yml | 3 +-- scripts/azure-pipelines/linux/create-vmss.ps1 | 18 ++++------------ .../linux/initialize-environment.sh | 9 +++----- scripts/azure-pipelines/linux/provision-image.sh | 24 ++++++++++------------ scripts/azure-pipelines/windows/create-vmss.ps1 | 4 ++-- scripts/ci.baseline.txt | 17 +++++++++++---- 8 files changed, 36 insertions(+), 43 deletions(-) (limited to 'scripts') diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 7b6274845..a0d9f3953 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -3,7 +3,7 @@ # variables: windows-pool: 'PrWin-2020-04-28' - linux-pool: 'PrLin-2020-04-30' + linux-pool: 'PrLin-2020-05-07' jobs: - template: windows/azure-pipelines.yml diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml index 3acf736e1..9ecee265d 100644 --- a/scripts/azure-pipelines/clean-tombstones.yml +++ b/scripts/azure-pipelines/clean-tombstones.yml @@ -3,7 +3,7 @@ # variables: windows-pool: 'PrWin-2020-04-28' - linux-pool: 'PrLin-2020-04-30' + linux-pool: 'PrLin-2020-05-07' jobs: - job: windows diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 085c6c3be..58e82efef 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -27,8 +27,7 @@ jobs: - powershell: | $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results - mkdir /ci/home - $env:HOME = '/ci/home' + $env:HOME = '/home/root' ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching displayName: '** Test Modified Ports **' - task: PowerShell@2 diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 index dfe2102d9..ac438254a 100755 --- a/scripts/azure-pipelines/linux/create-vmss.ps1 +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -18,10 +18,9 @@ or are running from Azure Cloud Shell. $Location = 'westus2' $Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd') -$VMSize = 'Standard_F16s_v2' +$VMSize = 'Standard_D16a_v4' $ProtoVMName = 'PROTOTYPE' $LiveVMPrefix = 'BUILD' -$InstalledDiskSizeInGB = 1024 $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' @@ -259,7 +258,7 @@ $StorageContext = New-AzStorageContext ` -StorageAccountKey $StorageAccountKey New-AzStorageShare -Name 'archives' -Context $StorageContext -Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120 +Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024 #################################################################################################### Write-Progress ` @@ -273,7 +272,7 @@ $Nic = New-AzNetworkInterface ` -Location $Location ` -Subnet $VirtualNetwork.Subnets[0] -$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize +$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1 $VM = Set-AzVMOperatingSystem ` -VM $VM ` -Linux ` @@ -377,8 +376,7 @@ $Vmss = Set-AzVmssOsProfile ` -VirtualMachineScaleSet $Vmss ` -ComputerNamePrefix $LiveVMPrefix ` -AdminUsername AdminUser ` - -AdminPassword $AdminPW ` - -CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n"))) + -AdminPassword $AdminPW $Vmss = Set-AzVmssStorageProfile ` -VirtualMachineScaleSet $Vmss ` @@ -386,14 +384,6 @@ $Vmss = Set-AzVmssStorageProfile ` -OsDiskCaching ReadWrite ` -ImageReferenceId $Image.Id -$Vmss = Add-AzVmssDataDisk ` - -VirtualMachineScaleSet $Vmss ` - -Lun 0 ` - -Caching 'ReadWrite' ` - -CreateOption Empty ` - -DiskSizeGB 1024 ` - -StorageAccountType 'StandardSSD_LRS' - New-AzVmss ` -ResourceGroupName $ResourceGroupName ` -Name $VmssName ` diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh index 82b52bd2e..5346345a5 100755 --- a/scripts/azure-pipelines/linux/initialize-environment.sh +++ b/scripts/azure-pipelines/linux/initialize-environment.sh @@ -4,12 +4,9 @@ # Cleans up the environment to prevent contamination across builds. if [ ! -d "archives" ]; then - ln -s /ci/archives archives -fi -if [ ! -d "installed" ]; then - ln -s /ci/installed installed + ln -s /archives archives fi -rm -rf installed/* +rm -rf installed rm -rf buildtrees -rm -rf packages +rm -rf packages/* diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index dbe10cf78..54e6c55ac 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -50,19 +50,12 @@ echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null sudo chmod 600 $smbCredentialFile -# Write script to provision disks used by cloud-init -echo "if [ ! -d \"/ci\" ]; then" > /etc/provision-disks.sh -echo "sudo parted /dev/sdc mklabel gpt" >> /etc/provision-disks.sh -echo "sudo parted /dev/sdc mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh -echo "sudo mkfs -t ext4 /dev/sdc1" >> /etc/provision-disks.sh -echo "sudo mkdir /ci -m=777" >> /etc/provision-disks.sh -echo "sudo mkdir /ci/installed -m=777" >> /etc/provision-disks.sh -echo "sudo mkdir /ci/archives -m=777" >> /etc/provision-disks.sh -echo "echo \"/dev/sdc1 /ci/installed ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh -echo "echo \"//$StorageAccountName.file.core.windows.net/archives /ci/archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh -echo "sudo mount -a" >> /etc/provision-disks.sh -echo "fi" >> /etc/provision-disks.sh -sudo chmod 700 /etc/provision-disks.sh +# Mount the archives SMB share to /archives +sudo mkdir /archives -m=777 +echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab + +# Create 'home' directory for haskell stack bits that want this +sudo mkdir -p /home/root -m=777 # Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above sudo rm /etc/debian_version @@ -71,3 +64,8 @@ sudo rm /etc/debian_version # https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux # (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs) apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60 + +# Create work trees for the Azure Pipelines agent so that it puts its work tree into temporary storage. +sudo chmod 777 /mnt +sudo mkdir /agent -m=777 +sudo ln -s /mnt /agent/_work diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 8c05cd51d..6a3f01fc1 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -294,7 +294,7 @@ $StorageContext = New-AzStorageContext ` -StorageAccountKey $StorageAccountKey New-AzStorageShare -Name 'archives' -Context $StorageContext -Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120 +Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048 #################################################################################################### Write-Progress ` @@ -308,7 +308,7 @@ $Nic = New-AzNetworkInterface ` -Location $Location ` -Subnet $VirtualNetwork.Subnets[0] -$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize +$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1 $VM = Set-AzVMOperatingSystem ` -VM $VM ` -Windows ` diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 192325a80..19588471b 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1006,7 +1006,6 @@ llgl:arm-uwp=fail llgl:x64-uwp=fail llvm:arm-uwp=fail llvm:arm64-windows=fail -llvm:x64-linux=fail llvm:x64-uwp=fail lmdb:arm64-windows=fail lmdb:arm-uwp=fail @@ -1134,7 +1133,17 @@ mpg123:x64-uwp=fail mpir:arm64-windows=fail mpir:arm-uwp=fail mpir:x64-uwp=fail - +# Building package gmp[core]:x64-linux... done +# Installing package gmp[core]:x64-linux... +# The following files are already installed in /mnt/1/s/installed/x64-linux and are in conflict with gmp:x64-linux +# +# Installed by mpir:x64-linux +# debug/lib/libgmp.a +# debug/lib/libgmp.la +# include/gmp.h +# lib/libgmp.a +# lib/libgmp.la +mpir:x64-linux=skip #Conflicts with angle and qt-5base ms-angle:arm64-windows = skip ms-angle:arm-uwp = skip @@ -1868,8 +1877,8 @@ usd:x64-linux=ignore usd:x86-windows=fail usrsctp:arm-uwp=fail usrsctp:x64-uwp=fail -uthenticode:arm-uwp=fail -uthenticode:x64-uwp=fail +uthenticode:arm-uwp=fail +uthenticode:x64-uwp=fail uvatlas:arm64-windows=fail uvatlas:arm-uwp=fail uvatlas:x64-linux=fail -- cgit v1.2.3