[ci] Update macOS to 11 (#17376)

* start 2021-04-16 process

Major new things:
* update to macOS Big Sur (11.*)
* switch from VirtualBox to Parallels due to ^ (and also ARM)

Minor new things:
* update from xcode CLT 12 to 12.4

This PR includes new stuff for creating the base box for parallels.
Still to do: using the new box type.

* update the vagrantfile stuff

* link the CI to the new version

* modify how macOS boxes are made

the Vagrantfile for the final VM will only download the azure pipeline stuff

this allows us to keep the versions of brew applications stable

* fix cpus and memory

* add vagrant plugins to installables

* Skip cppgraphqlgen ICE.

* [sdformat6] Remove unneeded include(FindBoost) which causes problems when the system cmake version doesn't match the one deployed by vcpkg.

* switch to dmg for installing osxfuse/sshfs

* Set cores to 11 (leaving 1 thread for host)

Co-authored-by: Billy Robert ONeal III <bion@microsoft.com>

7 files changed, 126 insertions, 94 deletions

diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb b/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb
new file mode 100644
index 000000000..90ad6c4c5
--- /dev/null
+++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb
@@ -0,0 +1,36 @@
+require 'json'
+
+configuration = JSON.parse(File.read("#{__dir__}/vagrant-box-configuration.json"))
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'vcpkg/macos-ci-base'
+  config.vm.box_version = configuration['box_version']
+  config.vm.synced_folder '.', '/Users/vagrant/shared'
+
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install Xcode Command Line Tools: attach dmg file',
+    inline: 'hdiutil attach shared/clt.dmg -mountpoint /Volumes/setup-installer',
+    privileged: false
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install Xcode Command Line Tools: run installer',
+    inline: 'installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /',
+    privileged: true
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install XCode Command Line Tools: detach dmg file',
+    inline: 'hdiutil detach /Volumes/setup-installer',
+    privileged: false
+
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install brew',
+    inline: '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"',
+    privileged: false
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install brew applications',
+    inline: "brew install #{configuration['brew'].join(' ')} && brew install --cask #{configuration['brew-cask'].join(' ')}",
+    privileged: false
+end
diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile b/scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb
index 08c472fe6..ed689ab7b 100644
--- a/scripts/azure-pipelines/osx/configuration/Vagrantfile
+++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb
@@ -1,79 +1,35 @@
 require 'json'
 
-require "erb"
-include ERB::Util
-
 configuration = JSON.parse(File.read("#{__dir__}/vagrant-configuration.json"))
 
 server = {
   :hostname => configuration['machine_name'],
   :box => configuration['box_name'],
   :box_version => configuration['box_version'],
-  :disk_size => configuration['disk_size'],
   :ram => 12000,
-  :cpu => 5
+  :cpu => 11
 }
 
-brew_formulas = [
-  'autoconf',
-  'automake',
-  'bison',
-  'gfortran',
-  'gperf',
-  'libtool',
-  'meson',
-  'mono',
-  'nasm',
-  'pkg-config',
-  'yasm' ]
-
-brew_cask_formulas = [
-  'powershell' ]
-
-azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.179.0/vsts-agent-osx-x64-2.179.0.tar.gz'
+azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.185.1/vsts-agent-osx-x64-2.185.1.tar.gz'
 devops_url = configuration['devops_url']
 agent_pool = configuration['agent_pool']
 pat = configuration['pat']
 
 Vagrant.configure('2') do |config|
-  # give them extra time to boot up
-  config.vm.boot_timeout = 600
-
   config.vm.box = server[:box]
   config.vm.box_version = server[:box_version]
   config.vm.hostname = server[:hostname]
   config.vm.synced_folder '.', '/vagrant', disabled: true
 
-  diskname = "#{server[:hostname]}-data.vmdk"
-
-  # I don't like this, but as far as I can tell, it's the only way
-  # to do this. When vagrant finishes the `disk` feature, switch
-  # over to that -- Nicole Mazzuca
-  if (not File.exists? diskname) then
-    system "VBoxManage createmedium --filename #{diskname} --size #{1024 * server[:disk_size]} --variant Fixed"
-  end
-
-  config.vm.provider 'virtualbox' do |vb|
-    vb.memory = server[:ram]
-    vb.cpus = server[:cpu]
-    vb.customize ['modifyvm', :id, '--ioapic', 'on']
-    vb.customize ['storageattach', :id,
-      '--storagectl', 'SATA Controller',
-      '--port', '1', '--device', '0', '--type', 'hdd',
-      '--medium', "#{diskname}"
-    ]
+  config.vm.provider 'parallels' do |prl|
+    prl.memory = server[:ram]
+    prl.cpus = server[:cpu]
   end
 
   config.vm.provision 'shell',
     run: 'once',
-    name: 'Format and mount the data filesystem',
-    inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0',
-    privileged: true
-
-  config.vm.provision 'shell',
-    run: 'once',
-    name: 'Link the data filesystem to the home directory',
-    inline: "ln -s /Volumes/data ~/Data",
+    name: 'Create the data directory',
+    inline: "mkdir ~/Data",
     privileged: false
 
   config.vm.provision 'shell',
@@ -90,12 +46,6 @@ Vagrant.configure('2') do |config|
 
   config.vm.provision 'shell',
     run: 'once',
-    name: 'Install brew applications',
-    inline: "brew install #{brew_formulas.join(' ')} && brew install --cask #{brew_cask_formulas.join(' ')}",
-    privileged: false
-
-  config.vm.provision 'shell',
-    run: 'once',
     name: 'Add VM to azure agent pool',
     inline: "cd ~/myagent;\
       ./config.sh --unattended \
diff --git a/scripts/azure-pipelines/osx/configuration/installables.json b/scripts/azure-pipelines/osx/configuration/installables.json
index 7a7ba400c..1b2c2d015 100644
--- a/scripts/azure-pipelines/osx/configuration/installables.json
+++ b/scripts/azure-pipelines/osx/configuration/installables.json
@@ -3,35 +3,38 @@
 
   "Applications": [
     {
-      "Name": "VirtualBox",
-      "TestCommand": "VBoxManage",
-      "DmgUrl": "https://download.virtualbox.org/virtualbox/6.1.16/VirtualBox-6.1.16-140961-OSX.dmg",
-      "Sha256": "D7DF0F05D9A9E7CBA50EA01DA264AC20948B1C9C0E0CCCD2D628085C9F434D45",
-      "InstallerPath": "VirtualBox.pkg"
-    },
-    {
       "Name": "vagrant",
-      "TestCommand": "vagrant",
-      "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.dmg",
-      "Sha256": "76B849B26E6D6187A7829212B05545D3B424E05F1BCD0F7163DA1E5117084FA6",
+      "VersionCommand": [ "vagrant", "-v" ],
+      "VersionRegex": "Vagrant (.*)",
+      "Version": "2.2.15",
+      "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.15/vagrant_2.2.15_x86_64.dmg",
+      "Sha256": "5C2B290C4FA2371E255C56B1E96DED3D0C838D54CB7F0E8E6CF154E9F206A20E",
       "InstallerPath": "vagrant.pkg"
-    }
-  ],
-  "Brew": [
+    },
     {
       "Name": "osxfuse",
-      "Kind": "cask"
+      "VersionCommand": [ "cat", "/Library/Filesystems/macfuse.fs/Contents/version.plist" ],
+      "VersionRegex": "<key>CFBundleVersion</key>[\\n\\t ]*<string>([0-9.]+)</string>",
+      "Version": "4.1.0",
+      "DmgUrl": "https://github.com/osxfuse/osxfuse/releases/download/macfuse-4.1.0/macfuse-4.1.0.dmg",
+      "Sha256": "3CB6A49406FD036C50EF1B4AD717A377F4DCF182811BDE172D69F1C289791085",
+      "InstallerPath": "Install macFUSE.pkg"
     },
     {
-      "Name": "sshfs"
+      "Name": "sshfs",
+      "VersionCommand": [ "sshfs", "--version" ],
+      "VersionRegex": "SSHFS version [0-9.]* \\(OSXFUSE SSHFS (.*)\\)",
+      "Version": "2.5.0",
+      "PkgUrl": "https://github.com/osxfuse/sshfs/releases/download/osxfuse-sshfs-2.5.0/sshfs-2.5.0.pkg",
+      "Sha256": "F8F4F71814273EA42DBE6CD92199F7CFF418571FFD1B10C0608878D3472D2162"
     }
   ],
-  "VBoxExtensions": [
+  "Brew": [
+  ],
+  "VagrantPlugins": [
     {
-      "Name": "Extension Pack",
-      "FullName": "Oracle VM VirtualBox Extension Pack",
-      "Url": "https://download.virtualbox.org/virtualbox/6.1.16/Oracle_VM_VirtualBox_Extension_Pack-6.1.16.vbox-extpack",
-      "Sha256": "9802482B77B95A954CB5111793DA10D009009A4E9A9C4EAA4BD1AE5DAFE9DB46"
+      "Name": "vagrant-parallels",
+      "Version": "2.2.1"
     }
   ]
 }
diff --git a/scripts/azure-pipelines/osx/configuration/installables.schema.json b/scripts/azure-pipelines/osx/configuration/installables.schema.json
index 13476e776..b7ec7ae80 100644
--- a/scripts/azure-pipelines/osx/configuration/installables.schema.json
+++ b/scripts/azure-pipelines/osx/configuration/installables.schema.json
@@ -10,7 +10,7 @@
   "required": [
     "Applications",
     "Brew",
-    "VBoxExtensions"
+    "VagrantPlugins"
   ],
   "properties": {
     "Applications": {
@@ -21,7 +21,18 @@
           "Name": {
             "type": "string"
           },
-          "TestCommand": {
+          "VersionCommand": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1
+          },
+          "VersionRegex": {
+            "type": "string",
+            "format": "regex"
+          },
+          "Version": {
             "type": "string"
           },
           "DmgUrl": {
@@ -53,23 +64,17 @@
         }
       }
     },
-    "VBoxExtensions": {
+    "VagrantPlugins": {
       "type": "array",
       "items": {
         "type": "object",
+        "required": [ "Name", "Version" ],
         "properties": {
           "Name": {
             "type": "string"
           },
-          "FullName": {
+          "Version": {
             "type": "string"
-          },
-          "Url": {
-            "type": "string",
-            "format": "uri"
-          },
-          "Sha256": {
-            "$ref": "#/definitions/sha256"
           }
         }
       }
diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json
new file mode 100644
index 000000000..d800c257a
--- /dev/null
+++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "./vagrant-vm-configuration.schema.json",
+  "brew": [
+    "autoconf",
+    "automake",
+    "bison",
+    "gettext",
+    "gfortran",
+    "gperf",
+    "gtk-doc",
+    "libtool",
+    "meson",
+    "mono",
+    "nasm",
+    "pkg-config",
+    "yasm"
+  ],
+  "brew-cask": [
+    "powershell"
+  ]
+}
diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json
new file mode 100644
index 000000000..80845f70f
--- /dev/null
+++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json
@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema",
+  "type": "object",
+  "required": [
+    "brew",
+    "brew-cask"
+  ],
+  "properties": {
+    "brew": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "brew-cask": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  }
+}
diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json
index 3d6b619e8..91540d77c 100644
--- a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json
+++ b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json
@@ -5,9 +5,11 @@
 
   "required": [
     "pat",
-    "base_name",
-    "disk_size",
-    "machine_identifiers"
+    "agent_pool",
+    "devops_url",
+    "machine_name",
+    "box_name",
+    "box_version"
   ],
 
   "properties": {
@@ -28,9 +30,6 @@
     },
     "box_version": {
       "type": "string"
-    },
-    "disk_size": {
-      "type": "integer"
     }
   }
 }