diff options
| author | Billy Robert O'Neal III <bion@microsoft.com> | 2020-04-28 22:16:12 -0700 |
|---|---|---|
| committer | Billy O'Neal <billy.oneal@gmail.com> | 2020-04-30 21:51:31 -0700 |
| commit | b8755728ab92f43d3056091324d7135b1c9fc390 (patch) | |
| tree | 61b68858eaee5e547c2cdc95c7aa601d334627ab | |
| parent | 43d664ab53233acc1afacaefbdc8ece98b75d02a (diff) | |
| download | vcpkg-b8755728ab92f43d3056091324d7135b1c9fc390.tar.gz vcpkg-b8755728ab92f43d3056091324d7135b1c9fc390.zip | |
[vcpkg] Onboard Linux to VMSS, open 'git' port, and switch back to Azure Spot
* Adds scripts to generate scale sets for testing Linux.
* Note workaround for https://github.com/microsoft/azure-pipelines-agent/pull/2929
* Switches Windows validation to 'Spot' VMs.
* Opens the git port 9418.
* Removes provisioning of the no longer used 'logs' file share.
* Changes Azure region to 'westus2', which is cheaper.
* Adds +x to all the scripts in scripts/azure-pipelines.
* Use 'xml-results' for all platforms instead of 'raw xml results' on Windows.
| -rw-r--r-- | README.md | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/azure-pipelines/analyze-test-results.ps1 | 0 | ||||
| -rw-r--r-- | scripts/azure-pipelines/azure-pipelines.yml | 9 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/azure-pipelines/generate-skip-list.ps1 | 0 | ||||
| -rw-r--r-- | scripts/azure-pipelines/linux/azure-pipelines.yml | 39 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/linux/create-vmss.ps1 | 408 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/linux/initialize-environment.sh | 15 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/linux/provision-image.sh | 75 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/azure-pipelines.yml (renamed from scripts/azure-pipelines/macos/azure-pipelines.yml) | 9 | ||||
| -rw-r--r-- | scripts/azure-pipelines/windows/azure-pipelines.yml | 18 | ||||
| -rw-r--r-- | scripts/azure-pipelines/windows/create-vmss.ps1 | 35 | ||||
| -rw-r--r-- | scripts/azure-pipelines/windows/initialize-environment.ps1 | 5 | ||||
| -rw-r--r-- | scripts/ci.baseline.txt | 30 |
13 files changed, 604 insertions, 43 deletions
@@ -9,9 +9,7 @@ For short description of available commands, run `vcpkg help`. * Slack: [https://cppalliance.org/slack/](https://cppalliance.org/slack/), the #vcpkg channel * Docs: [Documentation](docs/index.md) -| Windows (x86, x64, arm, uwp) | MacOS | Linux | -| ------------- | ------------- | ------------- | -| [](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=9&branchName=master) | [](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=11&branchName=master) | [](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=6&branchName=master) | +[](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=27&branchName=master) ## Quick Start Prerequisites: diff --git a/scripts/azure-pipelines/analyze-test-results.ps1 b/scripts/azure-pipelines/analyze-test-results.ps1 index 9e6d09d20..9e6d09d20 100644..100755 --- a/scripts/azure-pipelines/analyze-test-results.ps1 +++ b/scripts/azure-pipelines/analyze-test-results.ps1 diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index f810a2e6a..7b6274845 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,7 +2,8 @@ # SPDX-License-Identifier: MIT
#
variables:
- windows-pool: 'PrWin-2020-04-22'
+ windows-pool: 'PrWin-2020-04-28'
+ linux-pool: 'PrLin-2020-04-30'
jobs:
- template: windows/azure-pipelines.yml
@@ -41,4 +42,8 @@ jobs: jobName: arm_uwp
poolName: $(windows-pool)
- - template: macos/azure-pipelines.yml
+ - template: osx/azure-pipelines.yml
+
+ - template: linux/azure-pipelines.yml
+ parameters:
+ poolName: $(linux-pool)
diff --git a/scripts/azure-pipelines/generate-skip-list.ps1 b/scripts/azure-pipelines/generate-skip-list.ps1 index 98c868eb9..98c868eb9 100644..100755 --- a/scripts/azure-pipelines/generate-skip-list.ps1 +++ b/scripts/azure-pipelines/generate-skip-list.ps1 diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml new file mode 100644 index 000000000..c975626b4 --- /dev/null +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+jobs:
+- job: x64_linux
+ pool:
+ name: ${{ parameters.poolName }}
+
+ timeoutInMinutes: 1440 # 1 day
+
+ steps:
+ - task: Bash@3
+ displayName: 'Initialize Environment'
+ inputs:
+ filePath: scripts/azure-pipelines/linux/initialize-environment.sh
+ - task: Bash@3
+ displayName: 'Build vcpkg'
+ inputs:
+ filePath: bootstrap-vcpkg.sh
+ - powershell: |
+ $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
+ mkdir $(System.ArtifactsDirectory)/xml-results
+ mkdir /ci/home
+ $env:HOME = '/ci/home'
+ ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching
+ displayName: '** Test Modified Ports **'
+ - task: PowerShell@2
+ displayName: 'Analyze results and prepare test logs'
+ inputs:
+ failOnStderr: true
+ filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'
+ arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "archives/fail" -triplets "x64-linux" -errorOnRegression'
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifact: x86-linux Build Failure Logs'
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/failureLogs'
+ ArtifactName: 'x64-linux port build failure logs'
+ condition: failed()
diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 new file mode 100755 index 000000000..dfe2102d9 --- /dev/null +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -0,0 +1,408 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+<#
+.SYNOPSIS
+Creates a Linux virtual machine scale set, set up for vcpkg's CI.
+
+.DESCRIPTION
+create-vmss.ps1 creates an Azure Linux VM scale set, set up for vcpkg's CI
+system. See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/overview
+for more information.
+
+This script assumes you have installed Azure tools into PowerShell by following the instructions
+at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
+or are running from Azure Cloud Shell.
+#>
+
+$Location = 'westus2'
+$Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd')
+$VMSize = 'Standard_F16s_v2'
+$ProtoVMName = 'PROTOTYPE'
+$LiveVMPrefix = 'BUILD'
+$InstalledDiskSizeInGB = 1024
+$ErrorActionPreference = 'Stop'
+
+$ProgressActivity = 'Creating Scale Set'
+$TotalProgress = 10
+$CurrentProgress = 1
+
+<#
+.SYNOPSIS
+Returns whether there's a name collision in the resource group.
+
+.DESCRIPTION
+Find-ResourceGroupNameCollision takes a list of resources, and checks if $Test
+collides names with any of the resources.
+
+.PARAMETER Test
+The name to test.
+
+.PARAMETER Resources
+The list of resources.
+#>
+function Find-ResourceGroupNameCollision {
+ [CmdletBinding()]
+ Param([string]$Test, $Resources)
+
+ foreach ($resource in $Resources) {
+ if ($resource.ResourceGroupName -eq $Test) {
+ return $true
+ }
+ }
+
+ return $false
+}
+
+<#
+.SYNOPSIS
+Attempts to find a name that does not collide with any resources in the resource group.
+
+.DESCRIPTION
+Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the
+first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with
+any of the resources in the resource group.
+
+.PARAMETER Prefix
+The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?"
+#>
+function Find-ResourceGroupName {
+ [CmdletBinding()]
+ Param([string] $Prefix)
+
+ $resources = Get-AzResourceGroup
+ $result = $Prefix
+ $suffix = 0
+ while (Find-ResourceGroupNameCollision -Test $result -Resources $resources) {
+ $suffix++
+ $result = "$Prefix-$suffix"
+ }
+
+ return $result
+}
+
+<#
+.SYNOPSIS
+Creates a randomly generated password.
+
+.DESCRIPTION
+New-Password generates a password, randomly, of length $Length, containing
+only alphanumeric characters (both uppercase and lowercase).
+
+.PARAMETER Length
+The length of the returned password.
+#>
+function New-Password {
+ Param ([int] $Length = 32)
+
+ $Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ $result = ''
+ for ($idx = 0; $idx -lt $Length; $idx++) {
+ # NOTE: this should probably use RNGCryptoServiceProvider
+ $result += $Chars[(Get-Random -Minimum 0 -Maximum $Chars.Length)]
+ }
+
+ return $result
+}
+
+<#
+.SYNOPSIS
+Sanitizes a name to be used in a storage account.
+
+.DESCRIPTION
+Sanitize-Name takes a string, and removes all of the '-'s and
+lowercases the string, since storage account names must have no
+'-'s and must be completely lowercase alphanumeric. It then makes
+certain that the length of the string is not greater than 24,
+since that is invalid.
+
+.PARAMETER RawName
+The name to sanitize.
+#>
+function Sanitize-Name {
+ [CmdletBinding()]
+ Param(
+ [string]$RawName
+ )
+
+ $result = $RawName.Replace('-', '').ToLowerInvariant()
+ if ($result.Length -gt 24) {
+ Write-Error 'Sanitized name for storage account $result was too long.'
+ }
+
+ return $result
+}
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating resource group' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$ResourceGroupName = Find-ResourceGroupName $Prefix
+$AdminPW = New-Password
+New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
+$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating virtual network' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$allowHttp = New-AzNetworkSecurityRuleConfig `
+ -Name AllowHTTP `
+ -Description 'Allow HTTP(S)' `
+ -Access Allow `
+ -Protocol Tcp `
+ -Direction Outbound `
+ -Priority 1008 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange @(80, 443)
+
+$allowDns = New-AzNetworkSecurityRuleConfig `
+ -Name AllowDNS `
+ -Description 'Allow DNS' `
+ -Access Allow `
+ -Protocol * `
+ -Direction Outbound `
+ -Priority 1009 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange 53
+
+$allowGit = New-AzNetworkSecurityRuleConfig `
+ -Name AllowGit `
+ -Description 'Allow git' `
+ -Access Allow `
+ -Protocol Tcp `
+ -Direction Outbound `
+ -Priority 1010 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange 9418
+
+$allowStorage = New-AzNetworkSecurityRuleConfig `
+ -Name AllowStorage `
+ -Description 'Allow Storage' `
+ -Access Allow `
+ -Protocol * `
+ -Direction Outbound `
+ -Priority 1011 `
+ -SourceAddressPrefix VirtualNetwork `
+ -SourcePortRange * `
+ -DestinationAddressPrefix Storage `
+ -DestinationPortRange *
+
+$denyEverythingElse = New-AzNetworkSecurityRuleConfig `
+ -Name DenyElse `
+ -Description 'Deny everything else' `
+ -Access Deny `
+ -Protocol * `
+ -Direction Outbound `
+ -Priority 1012 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange *
+
+$NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
+$NetworkSecurityGroup = New-AzNetworkSecurityGroup `
+ -Name $NetworkSecurityGroupName `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
+
+$SubnetName = $ResourceGroupName + 'Subnet'
+$Subnet = New-AzVirtualNetworkSubnetConfig `
+ -Name $SubnetName `
+ -AddressPrefix "10.0.0.0/16" `
+ -NetworkSecurityGroup $NetworkSecurityGroup
+
+$VirtualNetworkName = $ResourceGroupName + 'Network'
+$VirtualNetwork = New-AzVirtualNetwork `
+ -Name $VirtualNetworkName `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -AddressPrefix "10.0.0.0/16" `
+ -Subnet $Subnet
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating archives storage account' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$StorageAccountName = Sanitize-Name $ResourceGroupName
+
+New-AzStorageAccount `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -Name $StorageAccountName `
+ -SkuName 'Standard_LRS' `
+ -Kind StorageV2
+
+$StorageAccountKeys = Get-AzStorageAccountKey `
+ -ResourceGroupName $ResourceGroupName `
+ -Name $StorageAccountName
+
+$StorageAccountKey = $StorageAccountKeys[0].Value
+
+$StorageContext = New-AzStorageContext `
+ -StorageAccountName $StorageAccountName `
+ -StorageAccountKey $StorageAccountKey
+
+New-AzStorageShare -Name 'archives' -Context $StorageContext
+Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120
+
+####################################################################################################
+Write-Progress `
+ -Activity 'Creating prototype VM' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$NicName = $ResourceGroupName + 'NIC'
+$Nic = New-AzNetworkInterface `
+ -Name $NicName `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -Subnet $VirtualNetwork.Subnets[0]
+
+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize
+$VM = Set-AzVMOperatingSystem `
+ -VM $VM `
+ -Linux `
+ -ComputerName $ProtoVMName `
+ -Credential $Credential
+
+$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
+$VM = Set-AzVMSourceImage `
+ -VM $VM `
+ -PublisherName 'Canonical' `
+ -Offer 'UbuntuServer' `
+ -Skus '18.04-LTS' `
+ -Version latest
+
+$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
+New-AzVm `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -VM $VM
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Running provisioning script provision-image.sh in VM' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Invoke-AzVMRunCommand `
+ -ResourceGroupName $ResourceGroupName `
+ -VMName $ProtoVMName `
+ -CommandId 'RunShellScript' `
+ -ScriptPath "$PSScriptRoot\provision-image.sh" `
+ -Parameter @{StorageAccountName=$StorageAccountName; `
+ StorageAccountKey=$StorageAccountKey;}
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Restarting VM' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Converting VM to Image' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Stop-AzVM `
+ -ResourceGroupName $ResourceGroupName `
+ -Name $ProtoVMName `
+ -Force
+
+Set-AzVM `
+ -ResourceGroupName $ResourceGroupName `
+ -Name $ProtoVMName `
+ -Generalized
+
+$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+$PrototypeOSDiskName = $VM.StorageProfile.OsDisk.Name
+$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID
+$Image = New-AzImage -Image $ImageConfig -ImageName $ProtoVMName -ResourceGroupName $ResourceGroupName
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Deleting unused VM and disk' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Remove-AzVM -Id $VM.ID -Force
+Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $PrototypeOSDiskName -Force
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating scale set' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig'
+$VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id -Primary -Name $VmssIpConfigName
+$VmssName = $ResourceGroupName + 'Vmss'
+$Vmss = New-AzVmssConfig `
+ -Location $Location `
+ -SkuCapacity 0 `
+ -SkuName $VMSize `
+ -SkuTier 'Standard' `
+ -Overprovision $false `
+ -UpgradePolicyMode Manual `
+ -EvictionPolicy Delete `
+ -Priority Spot `
+ -MaxPrice -1
+
+$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
+ -VirtualMachineScaleSet $Vmss `
+ -Primary $true `
+ -IpConfiguration $VmssIpConfig `
+ -NetworkSecurityGroupId $NetworkSecurityGroup.Id `
+ -Name $NicName
+
+$Vmss = Set-AzVmssOsProfile `
+ -VirtualMachineScaleSet $Vmss `
+ -ComputerNamePrefix $LiveVMPrefix `
+ -AdminUsername AdminUser `
+ -AdminPassword $AdminPW `
+ -CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n")))
+
+$Vmss = Set-AzVmssStorageProfile `
+ -VirtualMachineScaleSet $Vmss `
+ -OsDiskCreateOption 'FromImage' `
+ -OsDiskCaching ReadWrite `
+ -ImageReferenceId $Image.Id
+
+$Vmss = Add-AzVmssDataDisk `
+ -VirtualMachineScaleSet $Vmss `
+ -Lun 0 `
+ -Caching 'ReadWrite' `
+ -CreateOption Empty `
+ -DiskSizeGB 1024 `
+ -StorageAccountType 'StandardSSD_LRS'
+
+New-AzVmss `
+ -ResourceGroupName $ResourceGroupName `
+ -Name $VmssName `
+ -VirtualMachineScaleSet $Vmss
+
+####################################################################################################
+Write-Progress -Activity $ProgressActivity -Completed
+Write-Host "Location: $Location"
+Write-Host "Resource group name: $ResourceGroupName"
+Write-Host "User name: AdminUser"
+Write-Host "Using generated password: $AdminPW"
+Write-Host 'Finished!'
diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh new file mode 100755 index 000000000..82b52bd2e --- /dev/null +++ b/scripts/azure-pipelines/linux/initialize-environment.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# Cleans up the environment to prevent contamination across builds. +if [ ! -d "archives" ]; then + ln -s /ci/archives archives +fi +if [ ! -d "installed" ]; then + ln -s /ci/installed installed +fi + +rm -rf installed/* +rm -rf buildtrees +rm -rf packages diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh new file mode 100755 index 000000000..b281529e1 --- /dev/null +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +sudo apt -y update +sudo apt -y dist-upgrade +# Install common build dependencies and partitioning tools +sudo apt -y install at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils build-essential zip libx11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxcursor-dev yasm nasm libnuma1 libnuma-dev python-six python3-six python-yaml flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config +# Required by qt5-x11extras +sudo apt -y install libxkbcommon-dev libxkbcommon-x11-dev +# Required by libhdfs3 +sudo apt -y install libkrb5-dev +# Required by intel-ipsec +sudo apt -y install nasm + +# Install newer version of nasm than the apt package +mkdir /tmp/nasm +cd /tmp/nasm +curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz +tar -xf nasm-2.14.02.tar.gz +cd nasm-2.14.02/ +./configure --prefix=/usr && make +sudo make install +cd ~ + +# Install the latest Haskell stack +curl -sSL https://get.haskellstack.org/ | sh + +# Install CUDA +wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb +sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +sudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb +wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +sudo apt -y update +sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 +sudo apt install -y --no-install-recommends libcudnn7-dev +sudo ln -s /usr/local/cuda-10.1/lib64/stubs/libcuda.so /usr/local/cuda-10.1/lib64/stubs/libcuda.so.1 + +# Install PowerShell +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +sudo apt update +sudo add-apt-repository universe +sudo apt install -y powershell + +# Write SMB credentials +sudo mkdir /etc/smbcredentials +smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred +echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null +echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null +sudo chmod 600 $smbCredentialFile + +# Write script to provision disks used by cloud-init +echo "if [ ! -d \"/ci\" ]; then" > /etc/provision-disks.sh +echo "sudo parted /dev/sdc mklabel gpt" >> /etc/provision-disks.sh +echo "sudo parted /dev/sdc mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh +echo "sudo mkfs -t ext4 /dev/sdc1" >> /etc/provision-disks.sh +echo "sudo mkdir /ci -m=777" >> /etc/provision-disks.sh +echo "sudo mkdir /ci/installed -m=777" >> /etc/provision-disks.sh +echo "sudo mkdir /ci/archives -m=777" >> /etc/provision-disks.sh +echo "echo \"/dev/sdc1 /ci/installed ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh +echo "echo \"//$StorageAccountName.file.core.windows.net/archives /ci/archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh +echo "sudo mount -a" >> /etc/provision-disks.sh +echo "fi" >> /etc/provision-disks.sh +sudo chmod 700 /etc/provision-disks.sh + +# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above +sudo rm /etc/debian_version + +# Install dependencies that the Azure Pipelines agent will want later to make launching VMs faster +# https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux +# (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs) +apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60 diff --git a/scripts/azure-pipelines/macos/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 68ced441c..45d9c0a7f 100644 --- a/scripts/azure-pipelines/macos/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -47,10 +47,11 @@ jobs: brew list libtool || brew install libtool
displayName: 'Install brew dependencies'
- - bash: |
- ./bootstrap-vcpkg.sh -allowAppleClang
+ - task: Bash@3
displayName: 'Build vcpkg'
-
+ inputs:
+ filePath: bootstrap-vcpkg.sh
+ arguments: '-allowAppleClang'
- powershell: |
$skip_list = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
mkdir $(System.ArtifactsDirectory)/xml-results
@@ -68,5 +69,5 @@ jobs: displayName: 'Publish Artifact: x64-osx port build failure logs'
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/failureLogs'
- ArtifactName: 'Port build failure logs'
+ ArtifactName: 'x64-osx port build failure logs'
condition: failed()
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 2e2e38c96..559037e3b 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -7,9 +7,6 @@ jobs: pool:
name: ${{ parameters.poolName }}
- variables:
- triplet: '${{ parameters.triplet }}'
-
timeoutInMinutes: 1440 # 1 day
steps:
@@ -20,17 +17,17 @@ jobs: - powershell: |
$baselineFile = "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt"
- $skipList = $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\generate-skip-list.ps1 -Triplet "$(triplet)" -BaselineFile $baselineFile
+ $skipList = $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile $baselineFile
Write-Host "baseline file: $baselineFile"
Write-Host "skip list: $skipList"
- $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\windows\ci-step.ps1 -Triplet "$(triplet)" -ExcludePorts $skipList
+ $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\windows\ci-step.ps1 -Triplet "${{ parameters.triplet }}" -ExcludePorts $skipList
Write-Host "CI test script is complete"
errorActionPreference: continue
displayName: '** Build vcpkg and Test Modified Ports **'
- powershell: |
- $baseName = "$(triplet)"
- $outputPathRoot = "$(System.ArtifactsDirectory)\raw xml results"
+ $baseName = "${{ parameters.triplet }}"
+ $outputPathRoot = "$(System.ArtifactsDirectory)\xml-results"
if(-not (Test-Path $outputPathRoot))
{
Write-Host "creating $outputPathRoot"
@@ -56,11 +53,10 @@ jobs: inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'
- arguments: '-baselineFile ''$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt'' -logDir ''$(System.ArtifactsDirectory)\raw xml results'' -failurelogDir ''archives\fail'' -outputDir ''$(Build.ArtifactStagingDirectory)'' -errorOnRegression -triplets ''$(triplet)'''
-
+ arguments: '-baselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "archives\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"'
- task: PublishBuildArtifacts@1
- displayName: 'Publish Artifact: $(triplet) port build failure logs'
+ displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs'
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)\failureLogs'
- ArtifactName: '$(triplet) port build failure logs'
+ ArtifactName: '${{ parameters.triplet }} port build failure logs'
condition: failed()
diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 099c7dbfb..8c05cd51d 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: MIT
#
-#
<#
.SYNOPSIS
@@ -17,7 +16,7 @@ at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6 or are running from Azure Cloud Shell.
#>
-$Location = 'SouthCentralUS'
+$Location = 'westus2'
$Prefix = 'PrWin-' + (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_F16s_v2'
$ProtoVMName = 'PROTOTYPE'
@@ -212,13 +211,25 @@ $allowDns = New-AzNetworkSecurityRuleConfig ` -DestinationAddressPrefix * `
-DestinationPortRange 53
+$allowGit = New-AzNetworkSecurityRuleConfig `
+ -Name AllowGit `
+ -Description 'Allow git' `
+ -Access Allow `
+ -Protocol Tcp `
+ -Direction Outbound `
+ -Priority 1010 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange 9418
+
$allowStorage = New-AzNetworkSecurityRuleConfig `
-Name AllowStorage `
-Description 'Allow Storage' `
-Access Allow `
-Protocol * `
-Direction Outbound `
- -Priority 1010 `
+ -Priority 1011 `
-SourceAddressPrefix VirtualNetwork `
-SourcePortRange * `
-DestinationAddressPrefix Storage `
@@ -230,7 +241,7 @@ $denyEverythingElse = New-AzNetworkSecurityRuleConfig ` -Access Deny `
-Protocol * `
-Direction Outbound `
- -Priority 1011 `
+ -Priority 1012 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
@@ -241,7 +252,7 @@ $NetworkSecurityGroup = New-AzNetworkSecurityGroup ` -Name $NetworkSecurityGroupName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
- -SecurityRules @($allowHttp, $allowDns, $allowStorage, $denyEverythingElse)
+ -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
$SubnetName = $ResourceGroupName + 'Subnet'
$Subnet = New-AzVirtualNetworkSubnetConfig `
@@ -282,10 +293,8 @@ $StorageContext = New-AzStorageContext ` -StorageAccountName $StorageAccountName `
-StorageAccountKey $StorageAccountKey
-$ArchivesFiles = New-AzStorageShare -Name 'archives' -Context $StorageContext
+New-AzStorageShare -Name 'archives' -Context $StorageContext
Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120
-$LogFiles = New-AzStorageShare -Name 'logs' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'logs' -Context $StorageContext -Quota 64
####################################################################################################
Write-Progress `
@@ -305,8 +314,7 @@ $VM = Set-AzVMOperatingSystem ` -Windows `
-ComputerName $ProtoVMName `
-Credential $Credential `
- -ProvisionVMAgent `
- -EnableAutoUpdate
+ -ProvisionVMAgent
$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
$VM = Set-AzVMSourceImage `
@@ -417,11 +425,14 @@ $VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id $VmssName = $ResourceGroupName + 'Vmss'
$Vmss = New-AzVmssConfig `
-Location $Location `
- -SkuCapacity 6 `
+ -SkuCapacity 0 `
-SkuName $VMSize `
-SkuTier 'Standard' `
-Overprovision $false `
- -UpgradePolicyMode Manual
+ -UpgradePolicyMode Manual `
+ -EvictionPolicy Delete `
+ -Priority Spot `
+ -MaxPrice -1
$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
-VirtualMachineScaleSet $Vmss `
diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1 index b86006a9c..4bbc15665 100644 --- a/scripts/azure-pipelines/windows/initialize-environment.ps1 +++ b/scripts/azure-pipelines/windows/initialize-environment.ps1 @@ -34,11 +34,6 @@ if (-Not (Test-Path W:)) { net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey
}
-Write-Host 'Setting up logs mount'
-if (-Not (Test-Path L:)) {
- net use L: "\\$StorageAccountName.file.core.windows.net\logs" /u:"AZURE\$StorageAccountName" $StorageAccountKey
-}
-
Write-Host 'Creating downloads directory'
mkdir D:\downloads -ErrorAction SilentlyContinue
diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 3dcc55941..b8695e15f 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -340,12 +340,13 @@ dmlc:arm64-windows=ignore dmlc:x64-uwp=fail
dmlc:x64-windows-static=ignore
dmlc:x86-windows=ignore
-dpdk:arm64-windows=fail
dpdk:arm-uwp=fail
+dpdk:arm64-windows=fail
+dpdk:x64-linux=fail
dpdk:x64-osx=fail
dpdk:x64-uwp=fail
-dpdk:x64-windows=fail
dpdk:x64-windows-static=fail
+dpdk:x64-windows=fail
dpdk:x86-windows=fail
duckx:arm64-windows = skip
duckx:arm-uwp = skip
@@ -560,10 +561,15 @@ graphqlparser:x64-uwp=fail grpc:arm-uwp=fail
gsl:arm-uwp=fail
gsl:x64-uwp=fail
-gsoap:arm-uwp=ignore
+# https://github.com/microsoft/vcpkg/pull/11048/files
+gsoap:arm-uwp=fail
+gsoap:arm64-windows=fail
gsoap:x64-linux=fail
gsoap:x64-osx=fail
-gsoap:x86-windows=ignore
+gsoap:x64-uwp=fail
+gsoap:x64-windows-static=fail
+gsoap:x64-windows=fail
+gsoap:x86-windows=fail
gtk:x64-linux=fail
gtk:x86-windows=ignore
guetzli:x64-osx=fail
@@ -1116,6 +1122,11 @@ mozjpeg:x64-uwp = skip mozjpeg:x64-windows = skip
mozjpeg:x64-windows-static = skip
mozjpeg:x86-windows = skip
+# mp3lame emits error LNK2019: unresolved external symbol __imp_RoInitialize referenced in function
+# __scrt_initialize_winrt; likely missing an import library
+mp3lame:arm-uwp=fail
+mp3lame:arm64-uwp=fail
+mp3lame:x64-uwp=fail
mpg123:arm-uwp=fail
mpg123:x64-uwp=fail
mpir:arm64-windows=fail
@@ -1346,8 +1357,12 @@ pangomm:x86-windows=ignore pangomm:arm64-windows=fail
parmetis:x64-linux=fail
parmetis:x64-osx=fail
+# https://github.com/microsoft/vcpkg/issues/11050
pbc:arm-uwp=fail
+pbc:x86-windows=fail
+pbc:x64-windows=fail
pbc:x64-uwp=fail
+pbc:x64-windows-static=fail
pcre2:arm-uwp=fail
pdal:x64-linux=fail
pdal:x64-osx=fail
@@ -1364,6 +1379,9 @@ pfring:x64-windows=fail pfring:x64-windows-static=fail
pfring:x86-windows=fail
pfring:x64-osx=fail
+# pfring on Linux currently fails because its build scripts enable warnings as
+# errors, and warnings trigger with the Linux kernel headers in the Azure images.
+pfring:x64-linux=fail
physfs:arm64-windows=fail
physx:arm64-windows=fail
physx:x64-linux=fail
@@ -1393,12 +1411,13 @@ plibsys:x64-uwp=fail plplot:arm64-windows=fail
plplot:arm-uwp=fail
plplot:x64-uwp=fail
-pmdk:arm64-windows=fail
pmdk:arm-uwp=fail
+pmdk:arm64-windows=fail
pmdk:x64-linux=fail
pmdk:x64-osx=fail
pmdk:x64-uwp=fail
pmdk:x64-windows-static=fail
+pmdk:x64-windows=fail
pmdk:x86-windows=fail
pngwriter:arm-uwp=fail
pngwriter:x64-uwp=fail
@@ -1550,7 +1569,6 @@ redis-plus-plus:x64-windows-static=fail redis-plus-plus:arm64-windows=fail
replxx:arm-uwp=fail
replxx:x64-uwp=fail
-replxx:x64-windows-static=fail
reproc:arm-uwp=fail
reproc:x64-uwp=fail
restbed:arm-uwp=fail
|