diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/azure-pipelines/azure-pipelines.yml | 3 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 | 32 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/README.md | 65 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 | 63 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/azure-pipelines.yml | 13 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/VagrantFile | 136 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/Vagrantfile | 129 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json | 22 |
8 files changed, 290 insertions, 173 deletions
diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 01810eb0f..e59ab4652 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -4,6 +4,7 @@ variables:
windows-pool: 'PrWin-2020-08-12'
linux-pool: 'PrLin-2020-08-12'
+ osx-pool: 'PrOsx-2020-09-17'
stages:
- stage: check_cxx_formatting
@@ -81,6 +82,8 @@ stages: poolName: $(windows-pool)
- template: osx/azure-pipelines.yml
+ parameters:
+ poolName: $(osx-pool)
- template: linux/azure-pipelines.yml
parameters:
diff --git a/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 b/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 new file mode 100644 index 000000000..07b3f9a9b --- /dev/null +++ b/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 @@ -0,0 +1,32 @@ +#!pwsh +#Requires -Version 6.0 + +<# +.SYNOPSIS +Installs the base box at the specified version from the share. + +.PARAMETER StorageAccountAccessKey +An access key for the storage account. + +.PARAMETER BaseBoxVersion +The version of the base box to import; this should be a date, i.e. 2020-09-17 +#> +[CmdletBinding(PositionalBinding=$False)] +Param( + [Parameter(Mandatory=$True)] + [String]$StorageAccountAccessKey, + [Parameter(Mandatory=$True)] + [String]$BaseBoxVersion +) + +Set-StrictMode -Version 2 + +if (-not $IsMacOS) { + throw 'This script should only be run on a macOS host' +} + +$encodedAccessKey = [System.Web.HttpUtility]::UrlEncode($StorageAccountAccessKey) + +# TODO: finish this, once I have access to a mac again +# mount_smbfs +# vagrant box add diff --git a/scripts/azure-pipelines/osx/README.md b/scripts/azure-pipelines/osx/README.md index 0395908f0..5c6dd74e7 100644 --- a/scripts/azure-pipelines/osx/README.md +++ b/scripts/azure-pipelines/osx/README.md @@ -67,3 +67,68 @@ $ sudo shutdown -r now and wait for the machine to start back up. Then, start again from `Install-Prerequisites.ps1`. + +## Creating a new Vagrant box + +Whenever we want to install updated versions of the command line tools, +or of macOS, we need to create a new vagrant box. +This is pretty easy, but the results of the creation are not public, +since we're concerned about licensing. +However, if you're sure you're following Apple's licensing, +you can set up your own vagrant boxes that are the same as ours by doing the following: + +You'll need some prerequisites: + +- macinbox - installable via `sudo gem install macinbox` +- vagrant - found at <https://www.vagrantup.com/> +- VirtualBox - found at <https://www.virtualbox.org/> +- A macOS installer application - you can get this from the App Store (although I believe only the latest is available) +- An Xcode Command Line Tools installer - you can get this from Apple's developer website, + although you'll need to sign in first: <https://developer.apple.com/downloads> + +First, you'll need to create a base box; +this is where you determine what version of macOS is installed. + +``` +> sudo macinbox \ + --box-format virtualbox \ + --name macos-ci-base \ + --installer <path to macOS installer> \ + --no-gui +``` + +Once you've done that, create a Vagrantfile that looks like the following: + +```rb +Vagrant.configure('2') do |config| + config.vm.box = 'macos-ci-base' + config.vm.boot_timeout = 600 +end +``` + +then, run the following in that vagrant directory: + +```sh +$ vagrant up +$ vagrant scp <path to Command Line Tools for Xcode installer> :clt.dmg +$ vagrant ssh -c 'hdiutil attach clt.dmg -mountpoint /Volumes/setup-installer' +$ vagrant ssh -c 'sudo installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /' +$ vagrant ssh -c 'hdiutil detach /Volumes/setup-installer' +$ vagrant ssh -c '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"' +$ vagrant package +``` + +This will create a `package.box`, which is the box file for the base VM. +Then, you can `vagrant box add <package.box> --name <name for the box>`, +and you'll have the base vcpkg box added for purposes of `Setup-VagrantMachines.ps1`! + +Once you've created the base box, if you're making it the new base box for the CI, +upload it to the `vcpkgvagrant` storage account, to the `vagrant-boxes` share. +Then, add the metadata about the box (the name and version) to the JSON file there. +Once you've done that, add the software versions below. + +### VM Software Versions + +* 2020-09-17: + * macOS: 10.15.6 + * Xcode CLTs: 12 diff --git a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 index 5600cb1ed..22d2677d2 100755 --- a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 +++ b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 @@ -10,6 +10,18 @@ Setup-VagrantMachines.ps1 sets up the virtual machines for vcpkg's macOS CI. It puts the VagrantFile and necessary configuration JSON file into ~/vagrant/vcpkg-eg-mac. +.PARAMETER MachineId +The number to give the machine; should match [0-9]{2}. + +.PARAMETER BoxVersion +The version of the box to use. + +.PARAMETER AgentPool +The agent pool to add the machine to. + +.PARAMETER DevopsUrl +The URL of the ADO instance; for example, https://dev.azure.com/vcpkg is the vcpkg ADO instance. + .PARAMETER Pat The personal access token which has Read & Manage permissions on the ADO pool. @@ -26,18 +38,19 @@ The URN of the archives share; looks like `foo.windows.core.net`. The archives share name. .PARAMETER BaseName -The base name for the vagrant VM; the machine name is $BaseName-$MachineIdentifiers. +The base name for the vagrant VM; the machine name is $BaseName-$MachineId. Defaults to 'vcpkg-eg-mac'. +.PARAMETER BoxName +The name of the box to use. Defaults to 'vcpkg/macos-ci', +which is only available internally. + .PARAMETER Force Delete any existing vagrant/vcpkg-eg-mac directory. .PARAMETER DiskSize The size to make the temporary disks in gigabytes. Defaults to 425. -.PARAMETER MachineIdentifiers -The numbers to give the machines; should match [0-9]{2}. - .INPUTS None @@ -47,6 +60,18 @@ None [CmdletBinding(PositionalBinding=$False)] Param( [Parameter(Mandatory=$True)] + [String]$MachineId, + + [Parameter(Mandatory=$True)] + [String]$BoxVersion, + + [Parameter(Mandatory=$True)] + [String]$AgentPool, + + [Parameter(Mandatory=$True)] + [String]$DevopsUrl, + + [Parameter(Mandatory=$True)] [String]$Pat, [Parameter(Mandatory=$True)] @@ -65,13 +90,13 @@ Param( [String]$BaseName = 'vcpkg-eg-mac', [Parameter()] - [Switch]$Force, + [String]$BoxName = 'vcpkg/macos-ci', [Parameter()] - [Int]$DiskSize = 425, + [Switch]$Force, - [Parameter(Mandatory=$True, ValueFromRemainingArguments)] - [String[]]$MachineIdentifiers + [Parameter()] + [Int]$DiskSize = 425 ) Set-StrictMode -Version 2 @@ -80,27 +105,33 @@ if (-not $IsMacOS) { throw 'This script should only be run on a macOS host' } -if (Test-Path '~/vagrant') { +if (Test-Path '~/vagrant/vcpkg-eg-mac') { if ($Force) { Write-Host 'Deleting existing directories' - Remove-Item -Recurse -Force -Path '~/vagrant' | Out-Null + Remove-Item -Recurse -Force -Path '~/vagrant/vcpkg-eg-mac' | Out-Null } else { - throw '~/vagrant already exists; try re-running with -Force' + throw '~/vagrant/vcpkg-eg-mac already exists; try re-running with -Force' } } Write-Host 'Creating new directories' -New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null +if (-not (Test-Path -Path '~/vagrant')) +{ + New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null +} New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null Copy-Item ` - -Path "$PSScriptRoot/configuration/VagrantFile" ` - -Destination '~/vagrant/vcpkg-eg-mac/VagrantFile' + -Path "$PSScriptRoot/configuration/Vagrantfile" ` + -Destination '~/vagrant/vcpkg-eg-mac/Vagrantfile' $configuration = @{ pat = $Pat; - base_name = $BaseName; - machine_identifiers = $MachineIdentifiers; + agent_pool = $AgentPool; + devops_url = $DevopsUrl; + machine_name = "${BaseName}-${MachineId}"; + box_name = $BoxName; + box_version = $BoxVersion; disk_size = $DiskSize; archives = @{ username = $ArchivesUsername; diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 1b601fa7d..d3229f83c 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -5,8 +5,7 @@ jobs:
- job: x64_osx
pool:
- name: vcpkgAgentPool
- demands: Agent.OS -equals Darwin
+ name: ${{ parameters.poolName }}
workspace:
clean: resources
timeoutInMinutes: 1440 # 1 day
@@ -20,16 +19,6 @@ jobs: - bash: df -h
displayName: 'Report on Disk Space'
- bash: |
- brew list autoconf || brew install autoconf
- brew list automake || brew install automake
- brew list pkg-config || brew install pkg-config
- brew list libtool || brew install libtool
- brew list bison || brew install bison
- brew list gfortran || brew cask install gfortran
- brew list mono || brew install mono
- brew list yasm || brew install yasm
- displayName: 'Install brew dependencies'
- - bash: |
sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
exit 0
diff --git a/scripts/azure-pipelines/osx/configuration/VagrantFile b/scripts/azure-pipelines/osx/configuration/VagrantFile deleted file mode 100644 index 1e1cce50f..000000000 --- a/scripts/azure-pipelines/osx/configuration/VagrantFile +++ /dev/null @@ -1,136 +0,0 @@ -require 'json' - -require "erb" -include ERB::Util - -configuration = JSON.parse(File.read('./vagrant-configuration.json')) - -servers = configuration['machine_identifiers'].map do |id| - { - :hostname => "#{configuration['base_name']}-#{id}", - :box => 'ramsey/macos-catalina', - :ram => 12000, - :cpu => 5 - } -end - -brew_formulas = [ - 'autoconf', - 'automake', - 'libtool', - 'bison' ] - -brew_cask_formulas = [ - 'powershell', - 'gfortran' ] - -azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.171.1/vsts-agent-osx-x64-2.171.1.tar.gz' -devops_url = 'https://dev.azure.com/vcpkg' -agent_pool = 'vcpkgAgentPool' -pat = configuration['pat'] -archives = configuration['archives'] -archives_url = "//#{archives['username']}:#{url_encode(archives['access_key'])}@#{archives['urn']}/#{archives['share']}" - -Vagrant.configure('2') do |config| - # give them extra time to boot up - config.vm.boot_timeout = 600 - - servers.each do |machine| - config.vm.define machine[:hostname] do |node| - - node.vm.box = machine[:box] - node.vm.hostname = machine[:hostname] - node.vm.synced_folder '.', '/vagrant', disabled: true - - diskname = "#{machine[:hostname]}-data.vmdk" - - # I don't like this, but as far as I can tell, it's the only way - # to do this. When vagrant finishes the `disk` feature, switch - # over to that -- Nicole Mazzuca - if (not File.exists? diskname) then - system "VBoxManage createmedium --filename #{diskname} --size #{1024 * 220}" - end - - node.vm.provider 'virtualbox' do |vb| - vb.memory = machine[:ram] - vb.cpus = machine[:cpu] - vb.customize ['modifyvm', :id, '--ioapic', 'on'] - vb.customize ['storageattach', :id, - '--storagectl', 'SATA Controller', - '--port', '1', '--device', '0', '--type', 'hdd', - '--medium', "#{diskname}" - ] - end - - node.vm.provision 'shell', - run: 'once', - name: 'Format and mount the data filesystem', - inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0', - privileged: true - - node.vm.provision 'shell', - run: 'once', - name: 'Link the data filesystem to the home directory', - inline: "ln -s /Volumes/data ~/Data", - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Download azure agent', - inline: "curl -s -o ~/Downloads/azure-agent.tar.gz #{azure_agent_url}", - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Unpack azure agent', - inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz', - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Install brew and xcode command line tools', - inline: '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"', - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Install brew applications', - inline: "brew install #{brew_formulas.join(' ')} && brew cask install #{brew_cask_formulas.join(' ')}", - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Create archives mountpoint', - inline: 'mkdir ~/Data/archives', - privileged: false - - node.vm.provision "shell", - run: 'once', - name: 'Mount archives directory', - inline: "mount_smbfs -d 777 -f 777 #{archives_url} ~/Data/archives", - privileged: false - - node.vm.provision 'shell', - run: 'once', - name: 'Add VM to azure agent pool', - inline: "cd ~/myagent;\ - ./config.sh --unattended \ - --url #{devops_url} \ - --work ~/Data/work \ - --auth pat --token #{pat} \ - --pool #{agent_pool} \ - --agent `hostname` \ - --replace \ - --acceptTeeEula", - privileged: false - - # Start listening for jobs - node.vm.provision 'shell', - run: 'always', - name: 'Start running azure pipelines', - inline: 'cd /Users/vagrant/myagent;\ - nohup ./run.sh&', - privileged: false - end - end -end diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile b/scripts/azure-pipelines/osx/configuration/Vagrantfile new file mode 100644 index 000000000..91a59860c --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile @@ -0,0 +1,129 @@ +require 'json' + +require "erb" +include ERB::Util + +configuration = JSON.parse(File.read('./vagrant-configuration.json')) + +server = { + :hostname => configuration['machine_name'], + :box => configuration['box_name'], + :box_version => configuration['box_version'], + :disk_size => configuration['disk_size'], + :ram => 12000, + :cpu => 5 +} + +brew_formulas = [ + 'autoconf', + 'automake', + 'bison', + 'libtool', + 'mono', + 'pkg-config', + 'yasm' ] + +brew_cask_formulas = [ + 'powershell', + 'gfortran' ] + +azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.171.1/vsts-agent-osx-x64-2.171.1.tar.gz' +devops_url = configuration['devops_url'] +agent_pool = configuration['agent_pool'] +pat = configuration['pat'] +archives = configuration['archives'] +archives_url = "//#{archives['username']}:#{url_encode(archives['access_key'])}@#{archives['urn']}/#{archives['share']}" + +Vagrant.configure('2') do |config| + # give them extra time to boot up + config.vm.boot_timeout = 600 + + config.vm.box = server[:box] + config.vm.box_version = server[:box_version] + config.vm.hostname = server[:hostname] + config.vm.synced_folder '.', '/vagrant', disabled: true + + diskname = "#{server[:hostname]}-data.vmdk" + + # I don't like this, but as far as I can tell, it's the only way + # to do this. When vagrant finishes the `disk` feature, switch + # over to that -- Nicole Mazzuca + if (not File.exists? diskname) then + system "VBoxManage createmedium --filename #{diskname} --size #{1024 * server[:disk_size]}" + end + + config.vm.provider 'virtualbox' do |vb| + vb.memory = server[:ram] + vb.cpus = server[:cpu] + vb.customize ['modifyvm', :id, '--ioapic', 'on'] + vb.customize ['storageattach', :id, + '--storagectl', 'SATA Controller', + '--port', '1', '--device', '0', '--type', 'hdd', + '--medium', "#{diskname}" + ] + end + + config.vm.provision 'shell', + run: 'once', + name: 'Format and mount the data filesystem', + inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0', + privileged: true + + config.vm.provision 'shell', + run: 'once', + name: 'Link the data filesystem to the home directory', + inline: "ln -s /Volumes/data ~/Data", + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Download azure agent', + inline: "curl -s -o ~/Downloads/azure-agent.tar.gz #{azure_agent_url}", + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Unpack azure agent', + inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz', + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Install brew applications', + inline: "brew install #{brew_formulas.join(' ')} && brew cask install #{brew_cask_formulas.join(' ')}", + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Create archives mountpoint', + inline: 'mkdir ~/Data/archives', + privileged: false + + config.vm.provision "shell", + run: 'once', + name: 'Mount archives directory', + inline: "mount_smbfs -d 777 -f 777 #{archives_url} ~/Data/archives", + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Add VM to azure agent pool', + inline: "cd ~/myagent;\ + ./config.sh --unattended \ + --url #{devops_url} \ + --work ~/Data/work \ + --auth pat --token #{pat} \ + --pool #{agent_pool} \ + --agent `hostname` \ + --replace \ + --acceptTeeEula", + privileged: false + + # Start listening for jobs + config.vm.provision 'shell', + run: 'always', + name: 'Start running azure pipelines', + inline: 'cd /Users/vagrant/myagent;\ + nohup ./run.sh&', + privileged: false +end diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json index f0abc1673..d806f4861 100644 --- a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json +++ b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json @@ -15,20 +15,24 @@ "pat": { "type": "string" }, - "base_name": { + "agent_pool": { + "type": "string" + }, + "devops_url": { + "type": "string" + }, + "machine_name": { + "type": "string" + }, + "box_name": { + "type": "string" + }, + "box_version": { "type": "string" }, "disk_size": { "type": "integer" }, - "machine_identifiers": { - "type": "array", - - "items": { - "type": "string", - "pattern": "[0-9]{2}" - } - }, "archives": { "type": "object", "required": [ |