diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/azure-pipelines/azure-pipelines.yml | 2 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 | 53 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/osx/Install-Prerequisites.ps1 | 14 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/README.md | 67 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 | 83 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/Vagrantfile | 6 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/installables.json | 9 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/installables.schema.json | 19 | ||||
| -rw-r--r-- | scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json | 6 | ||||
| -rw-r--r-- | scripts/ci.baseline.txt | 1 |
10 files changed, 194 insertions, 66 deletions
diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index e59ab4652..1a2852ced 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -4,7 +4,7 @@ variables:
windows-pool: 'PrWin-2020-08-12'
linux-pool: 'PrLin-2020-08-12'
- osx-pool: 'PrOsx-2020-09-17'
+ osx-pool: 'PrOsx-2020-09-28'
stages:
- stage: check_cxx_formatting
diff --git a/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 b/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 index 07b3f9a9b..2660ba9e1 100644..100755 --- a/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 +++ b/scripts/azure-pipelines/osx/Get-InternalBaseBox.ps1 @@ -5,18 +5,19 @@ .SYNOPSIS Installs the base box at the specified version from the share. -.PARAMETER StorageAccountAccessKey -An access key for the storage account. +.PARAMETER FileshareMachine +The machine which is acting as a fileshare -.PARAMETER BaseBoxVersion -The version of the base box to import; this should be a date, i.e. 2020-09-17 +.PARAMETER BoxVersion +The version of the box to add. Defaults to latest if nothing is passed. #> -[CmdletBinding(PositionalBinding=$False)] +[CmdletBinding()] Param( [Parameter(Mandatory=$True)] - [String]$StorageAccountAccessKey, - [Parameter(Mandatory=$True)] - [String]$BaseBoxVersion + [String]$FileshareMachine, + + [Parameter()] + [String]$BoxVersion ) Set-StrictMode -Version 2 @@ -25,8 +26,36 @@ if (-not $IsMacOS) { throw 'This script should only be run on a macOS host' } -$encodedAccessKey = [System.Web.HttpUtility]::UrlEncode($StorageAccountAccessKey) +$mountPoint = '/Users/vcpkg/vagrant/share' + +if (mount | grep "on $mountPoint (") { + umount $mountPoint + if (-not $?) { + Write-Error "umount $mountPoint failed with return code $LASTEXITCODE." + throw + } +} + +sshfs "fileshare@${FileshareMachine}:/Users/fileshare/share" $mountPoint +if ($LASTEXITCODE -eq 1) { + Write-Error 'sshfs returned 1. +This means that the osxfuse kernel extension was not allowed to load. +Please open System Preferences > Security & Privacy > General, +and allow the kernel extension to load. +Then, rerun this script. + +If you''ve already done this, you probably need to add your ssh keys to the fileshare machine.' + throw +} elseif (-not $?) { + Write-Error "sshfs failed with return code $LASTEXITCODE." + throw +} + +if (-not [String]::IsNullOrEmpty($BoxVersion)) { + $versionArgs = @("--box-version", $BoxVersion) +} else { + $versionArgs = @() +} + +vagrant box add "$mountPoint/vcpkg-boxes/macos-ci.json" @versionArgs -# TODO: finish this, once I have access to a mac again -# mount_smbfs -# vagrant box add diff --git a/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 index f2c0b3713..546367574 100755 --- a/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 +++ b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 @@ -54,6 +54,20 @@ $Installables.Applications | ForEach-Object { hdiutil detach /Volumes/setup-installer } +$Installables.Brew | ForEach-Object { + $installable = $_ + if ($null -eq (Get-Member -InputObject $installable -Name 'Kind')) { + brew install $installable.Name + } else { + switch ($installable.Kind) { + 'cask' { brew cask install $installable.Name } + default { + Write-Error "Invalid kind: $_. Expected either empty, or 'cask'." + } + } + } +} + # Install plugins $installedExtensionPacks = Get-InstalledVirtualBoxExtensionPacks diff --git a/scripts/azure-pipelines/osx/README.md b/scripts/azure-pipelines/osx/README.md index 5c6dd74e7..96177365a 100644 --- a/scripts/azure-pipelines/osx/README.md +++ b/scripts/azure-pipelines/osx/README.md @@ -91,10 +91,10 @@ this is where you determine what version of macOS is installed. ``` > sudo macinbox \ - --box-format virtualbox \ - --name macos-ci-base \ - --installer <path to macOS installer> \ - --no-gui + --box-format virtualbox \ + --name macos-ci-base \ + --installer <path to macOS installer> \ + --no-gui ``` Once you've done that, create a Vagrantfile that looks like the following: @@ -102,7 +102,8 @@ Once you've done that, create a Vagrantfile that looks like the following: ```rb Vagrant.configure('2') do |config| config.vm.box = 'macos-ci-base' - config.vm.boot_timeout = 600 + config.vm.boot_timeout = 600 + config.vm.synced_folder ".", "/vagrant", disabled: true end ``` @@ -114,7 +115,36 @@ $ vagrant scp <path to Command Line Tools for Xcode installer> :clt.dmg $ vagrant ssh -c 'hdiutil attach clt.dmg -mountpoint /Volumes/setup-installer' $ vagrant ssh -c 'sudo installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /' $ vagrant ssh -c 'hdiutil detach /Volumes/setup-installer' +$ vagrant ssh -c 'rm clt.dmg' $ vagrant ssh -c '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"' +$ vagrant ssh -c 'brew cask install osxfuse && brew install sshfs' +$ vagrant scp <path to ssh key for fileshare> :.ssh/id_rsa +$ vagrant scp <path to ssh public key for fileshare> :.ssh/id_rsa.pub +$ vagrant reload +``` + +We also now need to make sure that osxfuse is set up correctly; +macOS requires the user to accept that this signed binary is "okay to be loaded" by the kernel. +We can get `sshfs` to try to start the `osxfuse` kernel module by attempting to start it: + +```sh +$ vagrant ssh -c 'mkdir testmnt && sshfs <fileshare ssh>:/Users/fileshare/share testmnt' +``` + +Then, you'll need to open the VM in VirtualBox, go to System Preferences, +go to Security & Privacy, General, unlock the settings, +and allow apps from the osxfuse developer to run. + +Then, retry the above, and see if it works: + +```sh +$ vagrant ssh -c 'sshfs <fileshare ssh>:/Users/fileshare/share testmnt' +``` + +if that works, you can now package the box: + +```sh +$ vagrant ssh -c 'umount testmnt && rmdir testmnt' $ vagrant package ``` @@ -123,12 +153,35 @@ Then, you can `vagrant box add <package.box> --name <name for the box>`, and you'll have the base vcpkg box added for purposes of `Setup-VagrantMachines.ps1`! Once you've created the base box, if you're making it the new base box for the CI, -upload it to the `vcpkgvagrant` storage account, to the `vagrant-boxes` share. +upload it to the fileshare, under `share/vcpkg-boxes`. Then, add the metadata about the box (the name and version) to the JSON file there. Once you've done that, add the software versions below. ### VM Software Versions -* 2020-09-17: +* 2020-09-28: * macOS: 10.15.6 * Xcode CLTs: 12 + +### (Internal) Accessing the macOS fileshare + +The fileshare is located on `vcpkgmm-01`, under the `fileshare` user, in the `share` directory. +In order to get `sshfs` working on the physical machine, +you'll need to do the same thing one needs to do for building the base box: + +```sh +$ brew cask install osxfuse && brew install sshfs +$ sudo shutdown -r now +``` + +Then, once you've ssh'd back in: + +```sh +$ mkdir vagrant/share +$ sshfs fileshare@<vcpkgmm-01 URN>:/Users/fileshare/share vagrant/share +``` + +If you get an error, that means that gatekeeper has prevented the kernel extension from loading, +so you'll need to access the GUI of the machine, go to System Preferences, +Security & Privacy, General, unlock the settings, and allow apps from the osxfuse developer to run. +Then, you'll be able to add the fileshare as an sshfs. diff --git a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 index 22d2677d2..d29ef599b 100755 --- a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 +++ b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 @@ -13,29 +13,30 @@ configuration JSON file into ~/vagrant/vcpkg-eg-mac. .PARAMETER MachineId The number to give the machine; should match [0-9]{2}. +.PARAMETER DevopsPat +The personal access token which has Read & Manage permissions on the ADO pool. + +.PARAMETER Date +The date on which this pool is being created. Sets the default values for BoxVersion and AgentPool. + .PARAMETER BoxVersion -The version of the box to use. +The version of the box to use. If -Date is passed, uses that as the version. .PARAMETER AgentPool -The agent pool to add the machine to. +The agent pool to add the machine to. If -Date is passed, uses "PrOsx-$Date" as the pool. .PARAMETER DevopsUrl -The URL of the ADO instance; for example, https://dev.azure.com/vcpkg is the vcpkg ADO instance. - -.PARAMETER Pat -The personal access token which has Read & Manage permissions on the ADO pool. +The URL of the ADO instance; defaults to vcpkg's, which is https://dev.azure.com/vcpkg. -.PARAMETER ArchivesUsername -The username for the archives share. - -.PARAMETER ArchivesAccessKey -The access key for the archives share. +.PARAMETER ArchivesMachine +The machine where the archives are located; a URN. -.PARAMETER ArchivesUrn -The URN of the archives share; looks like `foo.windows.core.net`. +.PARAMETER ArchivesPath +The path to where the archives are located on the machine. If -Date is passed, +uses "/Users/${ArchivesUsername}/share/archives/${Date}". -.PARAMETER ArchivesShare -The archives share name. +.PARAMETER ArchivesUsername +The user to log in to on the archives machine. Defaults to 'fileshare'. .PARAMETER BaseName The base name for the vagrant VM; the machine name is $BaseName-$MachineId. @@ -49,7 +50,7 @@ which is only available internally. Delete any existing vagrant/vcpkg-eg-mac directory. .PARAMETER DiskSize -The size to make the temporary disks in gigabytes. Defaults to 425. +The size to make the temporary disks in gigabytes. Defaults to 350. .INPUTS None @@ -57,34 +58,34 @@ None .OUTPUTS None #> -[CmdletBinding(PositionalBinding=$False)] +[CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='DefineDate')] Param( [Parameter(Mandatory=$True)] [String]$MachineId, [Parameter(Mandatory=$True)] - [String]$BoxVersion, + [String]$DevopsPat, - [Parameter(Mandatory=$True)] - [String]$AgentPool, + [Parameter(Mandatory=$True, ParameterSetName='DefineDate')] + [String]$Date, - [Parameter(Mandatory=$True)] - [String]$DevopsUrl, + [Parameter(Mandatory=$True, ParameterSetName='DefineVersionAndAgentPool')] + [String]$BoxVersion, - [Parameter(Mandatory=$True)] - [String]$Pat, + [Parameter(Mandatory=$True, ParameterSetName='DefineVersionAndAgentPool')] + [String]$AgentPool, - [Parameter(Mandatory=$True)] - [String]$ArchivesUsername, + [Parameter(Mandatory=$False)] + [String]$DevopsUrl = 'https://dev.azure.com/vcpkg', [Parameter(Mandatory=$True)] - [String]$ArchivesAccessKey, + [String]$ArchivesMachine, - [Parameter(Mandatory=$True)] - [String]$ArchivesUrn, + [Parameter(Mandatory=$True, ParameterSetName='DefineVersionAndAgentPool')] + [String]$ArchivesPath, - [Parameter(Mandatory=$True)] - [String]$ArchivesShare, + [Parameter(Mandatory=$False)] + [String]$ArchivesUsername = 'archivesshare', [Parameter()] [String]$BaseName = 'vcpkg-eg-mac', @@ -93,10 +94,10 @@ Param( [String]$BoxName = 'vcpkg/macos-ci', [Parameter()] - [Switch]$Force, + [Int]$DiskSize = 350, [Parameter()] - [Int]$DiskSize = 425 + [Switch]$Force ) Set-StrictMode -Version 2 @@ -105,6 +106,12 @@ if (-not $IsMacOS) { throw 'This script should only be run on a macOS host' } +if (-not [String]::IsNullOrEmpty($Date)) { + $BoxVersion = $Date + $AgentPool = "PrOsx-$Date" + $ArchivesPath = "/Users/${ArchivesUsername}/share/archives/${Date}" +} + if (Test-Path '~/vagrant/vcpkg-eg-mac') { if ($Force) { Write-Host 'Deleting existing directories' @@ -115,8 +122,7 @@ if (Test-Path '~/vagrant/vcpkg-eg-mac') { } Write-Host 'Creating new directories' -if (-not (Test-Path -Path '~/vagrant')) -{ +if (-not (Test-Path -Path '~/vagrant')) { New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null } New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null @@ -126,7 +132,7 @@ Copy-Item ` -Destination '~/vagrant/vcpkg-eg-mac/Vagrantfile' $configuration = @{ - pat = $Pat; + pat = $DevopsPat; agent_pool = $AgentPool; devops_url = $DevopsUrl; machine_name = "${BaseName}-${MachineId}"; @@ -135,9 +141,8 @@ $configuration = @{ disk_size = $DiskSize; archives = @{ username = $ArchivesUsername; - access_key = $ArchivesAccessKey; - urn = $ArchivesUrn; - share = $ArchivesShare; + urn = $ArchivesMachine; + path = $ArchivesPath; }; } ConvertTo-Json -InputObject $configuration -Depth 5 ` diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile b/scripts/azure-pipelines/osx/configuration/Vagrantfile index 91a59860c..d13a5f652 100644 --- a/scripts/azure-pipelines/osx/configuration/Vagrantfile +++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile @@ -32,7 +32,9 @@ devops_url = configuration['devops_url'] agent_pool = configuration['agent_pool'] pat = configuration['pat'] archives = configuration['archives'] -archives_url = "//#{archives['username']}:#{url_encode(archives['access_key'])}@#{archives['urn']}/#{archives['share']}" +archives_username = archives['username'] +archives_urn = archives['urn'] +archives_path = archives['path'] Vagrant.configure('2') do |config| # give them extra time to boot up @@ -102,7 +104,7 @@ Vagrant.configure('2') do |config| config.vm.provision "shell", run: 'once', name: 'Mount archives directory', - inline: "mount_smbfs -d 777 -f 777 #{archives_url} ~/Data/archives", + inline: "sshfs #{archives_username}@#{archives_urn}:#{archives_path} ~/Data/archives", privileged: false config.vm.provision 'shell', diff --git a/scripts/azure-pipelines/osx/configuration/installables.json b/scripts/azure-pipelines/osx/configuration/installables.json index bd9b33ae4..28e24e6a3 100644 --- a/scripts/azure-pipelines/osx/configuration/installables.json +++ b/scripts/azure-pipelines/osx/configuration/installables.json @@ -17,6 +17,15 @@ "InstallerPath": "vagrant.pkg" } ], + "Brew": [ + { + "Name": "osxfuse", + "Kind": "cask" + }, + { + "Name": "sshfs" + } + ], "VBoxExtensions": [ { "Name": "Extension Pack", diff --git a/scripts/azure-pipelines/osx/configuration/installables.schema.json b/scripts/azure-pipelines/osx/configuration/installables.schema.json index 9f7734cc6..13476e776 100644 --- a/scripts/azure-pipelines/osx/configuration/installables.schema.json +++ b/scripts/azure-pipelines/osx/configuration/installables.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2019-09/schema", + "$schema": "https://json-schema.org/draft-07/schema", "type": "object", "definitions": { "sha256": { @@ -9,6 +9,7 @@ }, "required": [ "Applications", + "Brew", "VBoxExtensions" ], "properties": { @@ -36,6 +37,22 @@ } } }, + "Brew": { + "type": "array", + "items": { + "type": "object", + "required": [ "Name" ], + "properties": { + "Name": { + "type": "string" + }, + "Kind": { + "type": "string", + "enum": [ "cask" ] + } + } + } + }, "VBoxExtensions": { "type": "array", "items": { diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json index d806f4861..0b1ed2f32 100644 --- a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json +++ b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json @@ -37,15 +37,13 @@ "type": "object", "required": [ "username", - "access_key", "urn", - "share" + "path" ], "properties": { "username": { "type": "string" }, - "access_key": { "type": "string" }, "urn": { "type": "string" }, - "share": { "type": "string" } + "path": { "type": "string" } } } } diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index d7e96911d..e98a277ee 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -809,6 +809,7 @@ libpq:x64-uwp=fail libqcow:arm-uwp=fail libqcow:x64-uwp=fail libqcow:x64-windows-static=fail +libqcow:x64-osx=fail libraqm:x64-windows-static=fail librdkafka:arm-uwp=fail librdkafka:x64-uwp=fail |