diff options
Diffstat (limited to 'scripts/azure-pipelines/linux')
| -rw-r--r-- | scripts/azure-pipelines/linux/azure-pipelines.yml | 2 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/linux/create-vmss.ps1 | 98 | ||||
| -rwxr-xr-x | scripts/azure-pipelines/linux/provision-image.sh | 42 |
3 files changed, 80 insertions, 62 deletions
diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index d7bd09fe2..018203a6a 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
- arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+ arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -UseEnvironmentSasToken -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
- bash: |
df -h
displayName: 'Report on Disk Space After Build'
diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 index 169a80fc9..55484f29b 100755 --- a/scripts/azure-pipelines/linux/create-vmss.ps1 +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -64,7 +64,9 @@ Write-Progress ` -Status 'Creating virtual network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
-$allowHttp = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules = @()
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowHTTP `
-Description 'Allow HTTP(S)' `
-Access Allow `
@@ -76,49 +78,49 @@ $allowHttp = New-AzNetworkSecurityRuleConfig ` -DestinationAddressPrefix * `
-DestinationPortRange @(80, 443)
-$allowDns = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+ -Name AllowSFTP `
+ -Description 'Allow (S)FTP' `
+ -Access Allow `
+ -Protocol Tcp `
+ -Direction Outbound `
+ -Priority 1009 `
+ -SourceAddressPrefix * `
+ -SourcePortRange * `
+ -DestinationAddressPrefix * `
+ -DestinationPortRange @(21, 22)
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowDNS `
-Description 'Allow DNS' `
-Access Allow `
-Protocol * `
-Direction Outbound `
- -Priority 1009 `
+ -Priority 1010 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 53
-$allowGit = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowGit `
-Description 'Allow git' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
- -Priority 1010 `
+ -Priority 1011 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 9418
-$allowStorage = New-AzNetworkSecurityRuleConfig `
- -Name AllowStorage `
- -Description 'Allow Storage' `
- -Access Allow `
- -Protocol * `
- -Direction Outbound `
- -Priority 1011 `
- -SourceAddressPrefix VirtualNetwork `
- -SourcePortRange * `
- -DestinationAddressPrefix Storage `
- -DestinationPortRange *
-
-$denyEverythingElse = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name DenyElse `
-Description 'Deny everything else' `
-Access Deny `
-Protocol * `
-Direction Outbound `
- -Priority 1012 `
+ -Priority 1013 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
@@ -129,13 +131,14 @@ $NetworkSecurityGroup = New-AzNetworkSecurityGroup ` -Name $NetworkSecurityGroupName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
- -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
+ -SecurityRules $allFirewallRules
$SubnetName = $ResourceGroupName + 'Subnet'
$Subnet = New-AzVirtualNetworkSubnetConfig `
-Name $SubnetName `
-AddressPrefix "10.0.0.0/16" `
- -NetworkSecurityGroup $NetworkSecurityGroup
+ -NetworkSecurityGroup $NetworkSecurityGroup `
+ -ServiceEndpoint "Microsoft.Storage"
$VirtualNetworkName = $ResourceGroupName + 'Network'
$VirtualNetwork = New-AzVirtualNetwork `
@@ -170,8 +173,31 @@ $StorageContext = New-AzStorageContext ` -StorageAccountName $StorageAccountName `
-StorageAccountKey $StorageAccountKey
-New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024
+New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
+$StartTime = [DateTime]::Now
+$ExpiryTime = $StartTime.AddMonths(6)
+
+$SasToken = New-AzStorageAccountSASToken `
+ -Service Blob `
+ -Permission "racwdlup" `
+ -Context $StorageContext `
+ -StartTime $StartTime `
+ -ExpiryTime $ExpiryTime `
+ -ResourceType Service,Container,Object `
+ -Protocol HttpsOnly
+
+$SasToken = $SasToken.Substring(1) # strip leading ?
+
+# Note that we put the storage account into the firewall after creating the above SAS token or we
+# would be denied since the person running this script isn't one of the VMs we're creating here.
+Set-AzStorageAccount `
+ -ResourceGroupName $ResourceGroupName `
+ -AccountName $StorageAccountName `
+ -NetworkRuleSet ( `
+ @{bypass="AzureServices"; `
+ virtualNetworkRules=( `
+ @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
+ defaultAction="Deny"})
####################################################################################################
Write-Progress `
@@ -220,15 +246,23 @@ Write-Progress ` -Status 'Running provisioning script provision-image.sh in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
-$ProvisionImageResult = Invoke-AzVMRunCommand `
- -ResourceGroupName $ResourceGroupName `
- -VMName $ProtoVMName `
- -CommandId 'RunShellScript' `
- -ScriptPath "$PSScriptRoot\provision-image.sh" `
- -Parameter @{StorageAccountName=$StorageAccountName; `
- StorageAccountKey=$StorageAccountKey;}
-
-Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+$tempScript = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".sh"
+try {
+ $script = Get-Content "$PSScriptRoot\provision-image.sh" -Encoding utf8NoBOM
+ $script += "echo `"PROVISIONED_AZURE_STORAGE_NAME=\`"$StorageAccountName\`"`" | sudo tee -a /etc/environment"
+ $script += "echo `"PROVISIONED_AZURE_STORAGE_SAS_TOKEN=\`"$SasToken\`"`" | sudo tee -a /etc/environment"
+ Set-Content -Path $tempScript -Value $script -Encoding utf8NoBOM
+
+ $ProvisionImageResult = Invoke-AzVMRunCommand `
+ -ResourceGroupName $ResourceGroupName `
+ -VMName $ProtoVMName `
+ -CommandId 'RunShellScript' `
+ -ScriptPath $tempScript
+
+ Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+} finally {
+ Remove-Item $tempScript -Recurse -Force
+}
####################################################################################################
Write-Progress `
diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index 4936cf719..c04f4b3ca 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -11,11 +11,12 @@ APT_PACKAGES="at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils \ libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev \ libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml \ flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \ - xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config" + xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \ + pkg-config meson" # Additionally required by qt5-base APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \ - libxcb1-dev libx11-xcb-dev libxcb-glx0-dev" + libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev" # Additionally required by qt5-base for qt5-x11extras APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \ @@ -26,31 +27,29 @@ APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \ # Additionally required by libhdfs3 APT_PACKAGES="$APT_PACKAGES libkrb5-dev" +# Additionally required by kf5windowsystem +APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev" + # Additionally required by mesa APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako" # Additionally required by some packages to install additional python packages APT_PACKAGES="$APT_PACKAGES python3-pip" +# Additionally required by rtaudio +APT_PACKAGES="$APT_PACKAGES libasound2-dev" + # Additionally required/installed by Azure DevOps Scale Set Agents APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60" sudo apt -y install $APT_PACKAGES -# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from -# removing some of the above -sudo apt-mark hold libcurl4 -sudo apt-mark hold liblttng-ust0 -sudo apt-mark hold libkrb5-3 -sudo apt-mark hold zlib1g -sudo apt-mark hold libicu60 - # Install newer version of nasm than the apt package, required by intel-ipsec mkdir /tmp/nasm cd /tmp/nasm -curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz -tar -xf nasm-2.14.02.tar.gz -cd nasm-2.14.02/ +curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz +tar -xf nasm-2.15.05.tar.gz +cd nasm-2.15.05/ ./configure --prefix=/usr && make -j sudo make install cd ~ @@ -75,19 +74,4 @@ sudo apt update sudo add-apt-repository universe sudo apt install -y powershell -if [ -z "$StorageAccountName" ]; then -echo "No storage account supplied, skipping." -else -echo "Mapping storage account" - -# Write SMB credentials -sudo mkdir /etc/smbcredentials -smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred -echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null -echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null -sudo chmod 600 $smbCredentialFile - -# Mount the archives SMB share to /archives -sudo mkdir /archives -m=777 -echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab -fi +# provision-image.ps1 will append installation of the SAS token here |