diff options
Diffstat (limited to 'scripts')
8 files changed, 667 insertions, 0 deletions
diff --git a/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 new file mode 100755 index 000000000..f2c0b3713 --- /dev/null +++ b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 @@ -0,0 +1,80 @@ +#!pwsh +#Requires -Version 6.0 + +<# +.SYNOPSIS +Installs the set of prerequisites for the macOS CI hosts. + +.DESCRIPTION +Install-Prerequisites.ps1 installs all of the necessary prerequisites +to run the vcpkg macOS CI in a vagrant virtual machine, +skipping all prerequisites that are already installed. + +.PARAMETER Force +Don't skip the prerequisites that are already installed. + +.INPUTS +None + +.OUTPUTS +None +#> +[CmdletBinding()] +Param( + [Parameter()] + [Switch]$Force +) + +Set-StrictMode -Version 2 + +if (-not $IsMacOS) { + Write-Error 'This script should only be run on a macOS host' + throw +} + +Import-Module "$PSScriptRoot/Utilities.psm1" + +$Installables = Get-Content "$PSScriptRoot/configuration/installables.json" | ConvertFrom-Json + +$Installables.Applications | ForEach-Object { + if (-not (Get-CommandExists $_.TestCommand)) { + Write-Host "$($_.Name) not installed; installing now" + } elseif ($Force) { + Write-Host "$($_.Name) found; attempting to upgrade or re-install" + } else { + Write-Host "$($_.Name) already installed" + return + } + + $pathToDmg = "~/Downloads/$($_.Name).dmg" + Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256 + + hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer + sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target / + hdiutil detach /Volumes/setup-installer +} + +# Install plugins +$installedExtensionPacks = Get-InstalledVirtualBoxExtensionPacks + +$Installables.VBoxExtensions | ForEach-Object { + $extension = $_ + $installedExts = $installedExtensionPacks | Where-Object { $_.Pack -eq $extension.FullName -and $_.Usable -eq 'true' } + + if ($null -eq $installedExts) { + Write-Host "VBox extension: $($extension.Name) not installed; installing now" + } elseif ($Force) { + Write-Host "VBox extension: $($extension.Name) found; attempting to upgrade or re-install" + } else { + Write-Host "VBox extension: $($extension.Name) already installed" + return + } + + $pathToExt = "~/Downloads/$($extension.FullName -replace ' ','_').vbox-extpack" + + Get-RemoteFile -OutFile $pathToExt -Uri $extension.Url -Sha256 $extension.Sha256 | Out-Null + + Write-Host 'Attempting to install extension with sudo; you may need to enter your password' + sudo VBoxManage extpack install --replace $pathToExt + sudo VBoxManage extpack cleanup +} diff --git a/scripts/azure-pipelines/osx/README.md b/scripts/azure-pipelines/osx/README.md new file mode 100644 index 000000000..b0fdde434 --- /dev/null +++ b/scripts/azure-pipelines/osx/README.md @@ -0,0 +1,69 @@ +# `vcpkg-eg-mac` VMs + +## Table of Contents + +- [`vcpkg-eg-mac` VMs](#vcpkg-eg-mac-vms) + - [Table of Contents](#table-of-contents) + - [Basic Usage](#basic-usage) + - [Setting up a new macOS machine](#setting-up-a-new-macos-machine) + +## Basic Usage + +The simplest usage, and one which should be used for when spinning up +new VMs, and when restarting old ones, is a simple: + +``` +$ cd ~/vagrant/vcpkg-eg-mac +$ vagrant up +``` + +Any modifications to the machines should be made in `configuration/VagrantFile` +and `Setup-VagrantMachines.ps1`, and make sure to push any changes! + +## Setting up a new macOS machine + +Before anything else, one must download `brew` and `powershell`. + +```sh +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +$ brew cask install powershell +``` + +Then, we need to download the `vcpkg` repository: + +```sh +$ git clone https://github.com/microsoft/vcpkg +``` + +And now all we need to do is set it up! Replace `XX` with the number of +the virtual machine. Generally, that should be the same as the number +for the physical machine; i.e., vcpkgmm-04 will have vcpkg-eg-mac-04. + +```sh +$ cd vcpkg/scripts/azure-pipelines/osx +$ ./Install-Prerequisites.ps1 -Force + # NOTE: you may get an error about CoreCLR; see the following paragraph if you do +$ ./Setup-VagrantMachines.ps1 XX \ + -Pat '<get this from azure>' \ + -ArchivesUsername '<get this from the archives share>' \ + -ArchivesAccessKey '<get this from the archives share>' \ + -ArchivesUrn '<something>.file.core.windows.net' \ + -ArchivesShare 'archives' +$ cd ~/vagrant/vcpkg-eg-mac +$ vagrant up +``` + +If you see the following error: + +``` +Failed to initialize CoreCLR, HRESULT: 0x8007001F +``` + +You have to reboot the machine; run + +```sh +$ sudo shutdown -r now +``` + +and wait for the machine to start back up. Then, start again from +`Install-Prerequisites.ps1`. diff --git a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 new file mode 100755 index 000000000..fb153a28a --- /dev/null +++ b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 @@ -0,0 +1,113 @@ +#!pwsh +#Requires -Version 6.0 + +<# +.SYNOPSIS +Sets up the configuration for the vagrant virtual machines. + +.DESCRIPTION +Setup-VagrantMachines.ps1 sets up the virtual machines for +vcpkg's macOS CI. It puts the VagrantFile and necessary +configuration JSON file into ~/vagrant/vcpkg-eg-mac. + +.PARAMETER Pat +The personal access token which has Read & Manage permissions on the ADO pool. + +.PARAMETER ArchivesUsername +The username for the archives share. + +.PARAMETER ArchivesAccessKey +The access key for the archives share. + +.PARAMETER ArchivesUri +The URN of the archives share; looks like `foo.windows.core.net`. + +.PARAMETER ArchivesShare +The archives share name. + +.PARAMETER BaseName +The base name for the vagrant VM; the machine name is $BaseName-$MachineIdentifiers. +Defaults to 'vcpkg-eg-mac'. + +.PARAMETER Force +Delete any existing vagrant/vcpkg-eg-mac directory. + +.PARAMETER DiskSize +The size to make the temporary disks in gigabytes. Defaults to 425. + +.PARAMETER MachineIdentifiers +The numbers to give the machines; should match [0-9]{2}. + +.INPUTS +None + +.OUTPUTS +None +#> +[CmdletBinding(PositionalBinding=$False)] +Param( + [Parameter(Mandatory=$True)] + [String]$Pat, + + [Parameter(Mandatory=$True)] + [String]$ArchivesUsername, + + [Parameter(Mandatory=$True)] + [String]$ArchivesAccessKey, + + [Parameter(Mandatory=$True)] + [String]$ArchivesUrn, + + [Parameter(Mandatory=$True)] + [String]$ArchivesShare, + + [Parameter()] + [String]$BaseName = 'vcpkg-eg-mac', + + [Parameter()] + [Switch]$Force, + + [Parameter()] + [Int]$DiskSize = 425, + + [Parameter(Mandatory=$True, ValueFromRemainingArguments)] + [String[]]$MachineIdentifiers +) + +Set-StrictMode -Version 2 + +if (-not $IsMacOS) { + throw 'This script should only be run on a macOS host' +} + +if (Test-Path '~/vagrant') { + if ($Force) { + Write-Host 'Deleting existing directories' + Remove-Item -Recurse -Force -Path '~/vagrant' | Out-Null + } else { + throw '~/vagrant already exists; try re-running with -Force' + } +} + +Write-Host 'Creating new directories' +New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null +New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null + +Copy-Item ` + -Path "$PSScriptRoot/configuration/VagrantFile" ` + -Destination '~/vagrant/vcpkg-eg-mac/VagrantFile' + +$configuration = @{ + pat = $Pat; + base_name = $BaseName; + machine_identifiers = $MachineIdentifiers; + disk_size = $DiskSize; + archives = @{ + username = $ArchivesUsername; + access_key = $ArchivesAccessKey; + url = $ArchivesUri; + share = $ArchivesShare; + }; +} +ConvertTo-Json -InputObject $configuration -Depth 5 ` + | Set-Content -Path '~/vagrant/vcpkg-eg-mac/vagrant-configuration.json' diff --git a/scripts/azure-pipelines/osx/Utilities.psm1 b/scripts/azure-pipelines/osx/Utilities.psm1 new file mode 100644 index 000000000..6fffa1522 --- /dev/null +++ b/scripts/azure-pipelines/osx/Utilities.psm1 @@ -0,0 +1,150 @@ +#Requires -Version 6.0 +Set-StrictMode -Version 2 + +<# +.SYNOPSIS +Returns whether the specified command exists in the current environment. + +.DESCRIPTION +Get-CommandExists takes a string as a parameter, +and returns whether it exists in the current environment; +either a function, alias, or an executable in the path. +It's somewhat equivalent to `which`. + +.PARAMETER Name +Specifies the name of the command which may or may not exist. + +.INPUTS +System.String + The name of the command. + +.OUTPUTS +System.Boolean + Whether the command exists. +#> +function Get-CommandExists +{ + [CmdletBinding()] + [OutputType([Boolean])] + Param( + [Parameter(ValueFromPipeline)] + [String]$Name + ) + + $null -ne (Get-Command -Name $Command -ErrorAction SilentlyContinue) +} + +<# +.SYNOPSIS +Downloads a file and checks its hash. + +.DESCRIPTION +Get-RemoteFile takes a URI and a hash, +downloads the file at that URI to OutFile, +and checks that the hash of the downloaded file. +It then returns a FileInfo object corresponding to the downloaded file. + +.PARAMETER OutFile +Specifies the file path to download to. + +.PARAMETER Uri +The URI to download from. + +.PARAMETER Sha256 +The expected SHA256 of the downloaded file. + +.INPUTS +None + +.OUTPUTS +System.IO.FileInfo + The FileInfo for the downloaded file. +#> +function Get-RemoteFile +{ + [CmdletBinding(PositionalBinding=$False)] + [OutputType([System.IO.FileInfo])] + Param( + [Parameter(Mandatory=$True)] + [String]$OutFile, + [Parameter(Mandatory=$True)] + [String]$Uri, + [Parameter(Mandatory=$True)] + [String]$Sha256 + ) + + Invoke-WebRequest -OutFile $OutFile -Uri $Uri + $actualHash = Get-FileHash -Algorithm SHA256 -Path $OutFile + + if ($actualHash.Hash -ne $Sha256) { + throw @" +Invalid hash for file $OutFile; + expected: $Hash + found: $($actualHash.Hash) +Please make sure that the hash in the powershell file is correct. +"@ + } + + Get-Item $OutFile +} + +<# +.SYNOPSIS +Gets the list of installed extensions as powershell objects. + +.DESCRIPTION +Get-InstalledVirtualBoxExtensionPacks gets the installed extensions, +returning objects that look like: + +{ + Pack = 'Oracle VM VirtualBox Extension Pack'; + Version = '6.1.10'; + ... +} + +.INPUTS +None + +.OUTPUTS +PSCustomObject + The list of VBox Extension objects that are installed. +#> +function Get-InstalledVirtualBoxExtensionPacks +{ + [CmdletBinding()] + [OutputType([PSCustomObject])] + Param() + + $lines = VBoxManage list extpacks + + $result = @() + + $currentObject = $null + $currentKey = "" + $currentString = "" + + $lines | ForEach-Object { + if ($Line[0] -eq ' ') { + $currentString += "`n$($Line.Trim())" + } else { + if ($null -ne $currentObject) { + $currentObject.$currentKey = $currentString + } + $currentKey, $currentString = $Line -Split ':' + $currentString = $currentString.Trim() + + if ($currentKey.StartsWith('Pack no')) { + $currentKey = 'Pack' + if ($null -ne $currentObject) { + Write-Output ([PSCustomObject]$currentObject) + } + $currentObject = @{} + } + } + } + + if ($null -ne $currentObject) { + $currentObject.$currentKey = $currentString + Write-Output ([PSCustomObject]$currentObject) + } +} diff --git a/scripts/azure-pipelines/osx/configuration/VagrantFile b/scripts/azure-pipelines/osx/configuration/VagrantFile new file mode 100644 index 000000000..b142b8189 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/VagrantFile @@ -0,0 +1,118 @@ +require 'json' + +require "erb" +include ERB::Util + +configuration = JSON.parse(File.read('./vagrant-configuration.json')) + +servers = configuration['machine_identifiers'].map do |id| + { + :hostname => "#{configuration['base_name']}-#{id}", + :box => 'ramsey/macos-catalina', + :ram => 12000, + :cpu => 5 + } +end + +brew_formulas = [ + 'autoconf', + 'automake', + 'libtool', + 'bison' ] + +brew_cask_formulas = [ + 'powershell', + 'gfortran' ] + +azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.171.1/vsts-agent-osx-x64-2.171.1.tar.gz' +devops_url = 'https://dev.azure.com/vcpkg' +agent_pool = 'vcpkgAgentPool' +pat = configuration['pat'] +archives = configuration['archives'] +archives_url = "//#{archives['username']}:#{url_encode(archives['access_key'])}@#{archives['url']}/#{archives['share']}" + +Vagrant.configure('2') do |config| + # give them extra time to boot up + config.vm.boot_timeout = 600 + + servers.each do |machine| + config.vm.define machine[:hostname] do |node| + + node.vm.box = machine[:box] + node.vm.hostname = machine[:hostname] + node.vm.synced_folder '.', '/vagrant', disabled: true + + node.vm.disk :disk, name: "#{machine[:hostname]}-data", size: "#{config['disk_size']}GB" + + node.vm.provision 'shell', + run: 'once', + name: 'Format and mount the data filesystem', + inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0', + privileged: true + + node.vm.provision 'shell', + run: 'once', + name: 'Link the data filesystem to the home directory', + inline: "ln -s /Volumes/data ~/Data", + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Download azure agent', + inline: "curl -s -o ~/Downloads/azure-agent.tar.gz #{azure_agent_url}", + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Unpack azure agent', + inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz', + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Install brew and xcode command line tools', + inline: '/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"', + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Install brew applications', + inline: "brew install #{brew_formulas.join(' ')} && brew cask install #{brew_cask_formulas.join(' ')}", + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Create archives mountpoint', + inline: 'mkdir ~/Data/archives', + privileged: false + + node.vm.provision "shell", + run: 'once', + name: 'Mount archives directory', + inline: "mount_smbfs -d 777 -f 777 #{archives_url} ~/Data/archives", + privileged: false + + node.vm.provision 'shell', + run: 'once', + name: 'Add VM to azure agent pool', + inline: "cd ~/myagent;\ + ./config.sh --unattended \ + --url #{devops_url} \ + --work ~/Data/work \ + --auth pat --token #{pat} \ + --pool #{agent_pool} \ + --agent `hostname` \ + --replace \ + --acceptTeeEula", + privileged: false + + # Start listening for jobs + node.vm.provision 'shell', + run: 'always', + name: 'Start running azure pipelines', + inline: 'cd /Users/vagrant/myagent;\ + nohup ./run.sh&', + privileged: false + end + end +end diff --git a/scripts/azure-pipelines/osx/configuration/installables.json b/scripts/azure-pipelines/osx/configuration/installables.json new file mode 100644 index 000000000..bd9b33ae4 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/installables.json @@ -0,0 +1,28 @@ +{ + "$schema": "./installables.schema.json", + + "Applications": [ + { + "Name": "VirtualBox", + "TestCommand": "VBoxManage", + "DmgUrl": "https://download.virtualbox.org/virtualbox/6.1.10/VirtualBox-6.1.10-138449-OSX.dmg", + "Sha256": "EF0CA4924922514B6AD71469998821F2CF7C596B4B8B59736C3699759E0F1DF8", + "InstallerPath": "VirtualBox.pkg" + }, + { + "Name": "vagrant", + "TestCommand": "vagrant", + "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.dmg", + "Sha256": "529CDE2A78E6DF38EC906B65C70B36A087E2601EAB42E25856E35B20CCB027C0", + "InstallerPath": "vagrant.pkg" + } + ], + "VBoxExtensions": [ + { + "Name": "Extension Pack", + "FullName": "Oracle VM VirtualBox Extension Pack", + "Url": "https://download.virtualbox.org/virtualbox/6.1.10/Oracle_VM_VirtualBox_Extension_Pack-6.1.10.vbox-extpack", + "Sha256": "03067F27F4DA07C5D0FDAFC56D27E3EA23A60682B333B2A1010FB74EF9A40C28" + } + ] +} diff --git a/scripts/azure-pipelines/osx/configuration/installables.schema.json b/scripts/azure-pipelines/osx/configuration/installables.schema.json new file mode 100644 index 000000000..9f7734cc6 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/installables.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "type": "object", + "definitions": { + "sha256": { + "type": "string", + "pattern": "[A-Z0-9]{64}" + } + }, + "required": [ + "Applications", + "VBoxExtensions" + ], + "properties": { + "Applications": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Name": { + "type": "string" + }, + "TestCommand": { + "type": "string" + }, + "DmgUrl": { + "type": "string", + "format": "uri" + }, + "Sha256": { + "$ref": "#/definitions/sha256" + }, + "InstallerPath": { + "type": "string" + } + } + } + }, + "VBoxExtensions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Name": { + "type": "string" + }, + "FullName": { + "type": "string" + }, + "Url": { + "type": "string", + "format": "uri" + }, + "Sha256": { + "$ref": "#/definitions/sha256" + } + } + } + } + } +} diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json new file mode 100644 index 000000000..0518df3b6 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + + "type": "object", + + "required": [ + "pat", + "base_name", + "disk_size", + "machine_identifiers", + "archives" + ], + + "properties": { + "pat": { + "type": "string" + }, + "base_name": { + "type": "string" + }, + "disk_size": { + "type": "integer" + }, + "machine_identifiers": { + "type": "array", + + "items": { + "type": "string", + "pattern": "[0-9]{2}" + } + }, + "archives": { + "type": "object", + "required": [ + "username", + "access_key", + "url", + "share" + ], + "properties": { + "username": { "type": "string" }, + "access_key": { "type": "string" }, + "url": { "type": "string" }, + "share": { "type": "string" } + } + } + } +} |