diff options
| author | Billy O'Neal <bion@microsoft.com> | 2021-04-26 10:27:45 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-04-26 10:27:45 -0700 |
| commit | a9b27ed5dffbf70b135eddb0c4729f3ca87f106c (patch) | |
| tree | cc562b219c82e40567925a48248cbc17b2cc2b59 | |
| parent | 84e5852cfaa8e7875027e79a29a5534bb5d4a475 (diff) | |
| download | vcpkg-a9b27ed5dffbf70b135eddb0c4729f3ca87f106c.tar.gz vcpkg-a9b27ed5dffbf70b135eddb0c4729f3ca87f106c.zip | |
Update VMs, CMake to 3.20.1, CUDA to 11.3, and pwsh to 7.1.3 (#17331)
* Update to CUDA 11.3 on Windows.
* Update PowerShell to 7.1.3.
* Update CUDA to 11.3 on Linux.
* "Explode" VM provisioning scripts for more consistent feedback during deploy. This resolves the deployment script often hanging with no feedback as to why.
* [libdatachannel] Fix use of deprecated result_type typedef.
Submitted upstream as https://github.com/paullouisageneau/libdatachannel/pull/413
* [libvpx] Get the libvpx outputs from the correct place.
(Perhaps VS2019 version 16.10 moved where these are placed? I've been defensive and left an attempt to find from the old location in place.)
* [chromaprint] Support implementations where lrintf is an intrinsic.
* Add provision-entire-image script.
* [cudnn] Disable download-on-the-fly due to licensing concerns.
* Add libnccl to Linux VMs.
* [wangle] Disable x64-windows due to an ICE.
* [cmake] Update cmake to 3.20.1 to avoid https://gitlab.kitware.com/cmake/cmake/-/issues/21571 race
* [libudis86] Fix passing a bogus working directory which fails on CMake 3.20.x
* [dartsim] Disable unit tests, examples, and tutorials, some of which have CMake authoring errors rejected by 3.20.1.
* Add thrust to the cuda installees.
* [tensorflow] Put .bzl in CURRENT_BUILDTREES_DIR to avoid running out of disk space in CI and to respect --clean-after-build.
* [dimcli] Skip port broken by changes in VS2019 project system.
* [upb] Disable an additional warning.
* [libhv] Fix typo DISABLE_PARALLEL => DISABLE_PARALLEL_CONFIGURE
* Update pools
48 files changed, 862 insertions, 657 deletions
diff --git a/ports/chromaprint/fix_lrintf_detection.patch b/ports/chromaprint/fix_lrintf_detection.patch new file mode 100644 index 000000000..27248f0c8 --- /dev/null +++ b/ports/chromaprint/fix_lrintf_detection.patch @@ -0,0 +1,23 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 3a0eecb..4fa5e8d 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -17,6 +17,7 @@ set_property(CACHE FFT_LIB PROPERTY STRINGS avfft fftw3 fftw3f kissfft vdsp)
+
+ include(CMakePushCheckState)
+ include(CheckFunctionExists)
++include(CheckSymbolExists)
+ include(CheckCXXCompilerFlag)
+
+ find_package(Threads)
+@@ -37,8 +38,8 @@ endif()
+
+ cmake_push_check_state(RESET)
+ set(CMAKE_REQUIRED_LIBRARIES -lm)
+-check_function_exists(lrintf HAVE_LRINTF)
+-check_function_exists(round HAVE_ROUND)
++check_symbol_exists(lrintf math.h HAVE_LRINTF)
++check_symbol_exists(round math.h HAVE_ROUND)
+ cmake_pop_check_state()
+
+ add_definitions(
diff --git a/ports/chromaprint/portfile.cmake b/ports/chromaprint/portfile.cmake index da430faa5..286edaeb3 100644 --- a/ports/chromaprint/portfile.cmake +++ b/ports/chromaprint/portfile.cmake @@ -3,6 +3,8 @@ vcpkg_from_github( REPO acoustid/chromaprint REF v1.5.0 SHA512 333114949928abdf5d4b11aba1db6ec487eebe526324c68d903b3fa80a3af87a28d942af765a2f873e63a1bf222b658b6438cd10cde4446f61b26ea91f537469 + PATCHES + fix_lrintf_detection.patch # submitted upstream as https://github.com/acoustid/chromaprint/pull/85 ) vcpkg_configure_cmake( diff --git a/ports/chromaprint/vcpkg.json b/ports/chromaprint/vcpkg.json index 4808008c9..271532380 100644 --- a/ports/chromaprint/vcpkg.json +++ b/ports/chromaprint/vcpkg.json @@ -1,6 +1,7 @@ { "name": "chromaprint", - "version-string": "1.5.0", + "version": "1.5.0", + "port-version": 1, "description": "C library for generating audio fingerprints used by AcoustID", "homepage": "https://github.com/acoustid/chromaprint", "license": "LGPL-2.1-or-later", diff --git a/ports/cudnn/CONTROL b/ports/cudnn/CONTROL index ebb814349..db910a7bc 100644 --- a/ports/cudnn/CONTROL +++ b/ports/cudnn/CONTROL @@ -1,6 +1,6 @@ Source: cudnn Version: 7.6.5 -Port-Version: 1 +Port-Version: 2 Description: NVIDIA's cuDNN deep neural network acceleration library Build-Depends: cuda Supports: (windows|linux)&x64 diff --git a/ports/cudnn/portfile.cmake b/ports/cudnn/portfile.cmake index 1c1f0641e..6a0083130 100644 --- a/ports/cudnn/portfile.cmake +++ b/ports/cudnn/portfile.cmake @@ -7,7 +7,6 @@ if(NOT VCPKG_TARGET_ARCHITECTURE STREQUAL "x64") endif() vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT) -set(VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES enabled) # only release bits are provided set(MINIMUM_CUDNN_VERSION "7.6.5") @@ -64,74 +63,10 @@ if(CUDNN_HEADER_CONTENTS) endif() if (CUDNN_INCLUDE_DIR AND CUDNN_LIBRARY AND _CUDNN_VERSION VERSION_GREATER_EQUAL MINIMUM_CUDNN_VERSION) - set(CUDNN_FOUND TRUE) -else() - set(CUDNN_FOUND FALSE) -endif() - -# Download CUDNN if not found -if (CUDNN_FOUND) message(STATUS "Found CUDNN ${_CUDNN_VERSION} located on system: (include ${CUDNN_INCLUDE_DIR} lib: ${CUDNN_LIBRARY})") set(VCPKG_POLICY_EMPTY_PACKAGE enabled) -elseif(CUDA_VERSION VERSION_EQUAL "10.1" OR CUDA_VERSION VERSION_EQUAL "10.2") - message(STATUS "CUDNN not found on system - downloading a version compatible with your CUDA v${CUDA_VERSION}...") - if(${CUDA_VERSION} VERSION_EQUAL "10.1") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.1_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "0fb18dd49de877ad6bae24b53ffe007a99915cc9601697a556897e018cc6d99d3aa68716ea99248cf6a9dfaeeb1a551453c606d04e8bbb3e9315bf768184f15b") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "4d901d96ee8f37e3977240b9e6e6eeecb33848388db953a789be47de8f357d815c3a106ceab04297c4df0d8ed9c2795b2a22304e93cd1e53322307d3f3cd668e") - set(CUDNN_OS "linux") - endif() - elseif(${CUDA_VERSION} VERSION_EQUAL "10.2") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.2_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "86ca2f5f510d4fbeb37548d0bcab42474a1c4041be2cf96c4964f1e51c3641dc4bf25e8434cd5ff99fac9f53946e5f0e83bd845613144731d136cd60913d4aaa") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "b15b554c2ec004105cec8ee2a99f33fab0f9aed12128522730be4fa6204a5b2dff29e42901b5c4840b5ebf35607e8a54f35eb30525885067165b05dd95aa391b") - set(CUDNN_OS "linux") - endif() - endif() - - vcpkg_download_distfile(ARCHIVE - URLS ${CUDNN_DOWNLOAD_LINK} - FILENAME "cudnn-${CUDNN_FULL_VERSION}-${CUDNN_OS}.tar.bz2" - SHA512 ${SHA512_CUDNN} - ) - - vcpkg_extract_source_archive_ex( - OUT_SOURCE_PATH SOURCE_PATH - ARCHIVE ${ARCHIVE} - NO_REMOVE_ONE_LEVEL - ) - - if(VCPKG_TARGET_IS_WINDOWS) - file(INSTALL "${SOURCE_PATH}/Library/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/Library/lib/x64/cudnn.lib" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/Library/bin/cudnn64_${CUDNN_VERSION_MAJOR}.dll" DESTINATION ${CURRENT_PACKAGES_DIR}/bin) - set(LICENSE_PATH "${SOURCE_PATH}/info/LICENSE.txt") - elseif(VCPKG_TARGET_IS_LINUX) - file(INSTALL "${SOURCE_PATH}/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - set(LICENSE_PATH "${SOURCE_PATH}/info/licenses/NVIDIA_SLA_cuDNN_Support.txt") - endif() - - file(INSTALL "${LICENSE_PATH}" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindCUDNN.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - -else() # CUDNN NOT FOUND AND NOT AUTO-DOWNLOADABLE - message(FATAL_ERROR "Please install manually cuDNN for your CUDA v${CUDA_VERSION}") +elseif(VCPKG_TARGET_IS_WINDOWS) + message(FATAL_ERROR "Please download CUDNN from official sources (such as https://developer.nvidia.com/rdp/cudnn-download ) and extract the zip into your CUDA_TOOLKIT_ROOT (${CUDA_TOOLKIT_ROOT}). (For example: tar.exe -xvf cudnn-11.2-windows-x64-v8.1.1.33.zip --strip 1 --directory \"${CUDA_TOOLKIT_ROOT}\"") +else() + message(FATAL_ERROR "Please install CUDNN using your system package manager (the same way you installed CUDA). For example: apt install libcudnn8-dev.") endif() diff --git a/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch new file mode 100644 index 000000000..08119ce39 --- /dev/null +++ b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch @@ -0,0 +1,20 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e32147d..5c85186 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -279,6 +279,7 @@ add_subdirectory(dart) + + set(DART_IN_SOURCE_BUILD TRUE) + ++if (0) + if(TARGET dart AND NOT DART_BUILD_DARTPY) + + # Add a "tests" target to build unit tests. +@@ -330,6 +331,7 @@ if(TARGET dart AND NOT DART_BUILD_DARTPY) + endif(DART_VERBOSE) + + endif() ++endif() + + if (DART_BUILD_DARTPY) + add_subdirectory(python) diff --git a/ports/dartsim/portfile.cmake b/ports/dartsim/portfile.cmake index 0af5f8e9a..c208482f1 100644 --- a/ports/dartsim/portfile.cmake +++ b/ports/dartsim/portfile.cmake @@ -7,20 +7,23 @@ vcpkg_from_github( REF v6.9.4
SHA512 a9e8712a294286772ee1e3c3899aac7d202f1d3c2b9242ebeaddb2a142787b192d5421a6e3f889dd9ff9ca9e9106b1f893a0c8ab2e1656b04fac6d0be13494ba
HEAD_REF release-6.9
- PATCHES 1478.patch
- 1497.patch
+ PATCHES
+ 1478.patch
+ 1497.patch
+ disable_unit_tests_examples_and_tutorials.patch
)
vcpkg_configure_cmake(
SOURCE_PATH ${SOURCE_PATH}
- OPTIONS -DDART_VERBOSE=ON
- -DDART_MSVC_DEFAULT_OPTIONS=ON
- -DDART_SKIP_DOXYGEN=ON
- -DDART_SKIP_FLANN=ON
- -DDART_SKIP_IPOPT=ON
- -DDART_SKIP_NLOPT=ON
- -DDART_SKIP_OPENGL=ON
- -DDART_SKIP_pagmo=ON
+ OPTIONS
+ -DDART_VERBOSE=ON
+ -DDART_MSVC_DEFAULT_OPTIONS=ON
+ -DDART_SKIP_DOXYGEN=ON
+ -DDART_SKIP_FLANN=ON
+ -DDART_SKIP_IPOPT=ON
+ -DDART_SKIP_NLOPT=ON
+ -DDART_SKIP_OPENGL=ON
+ -DDART_SKIP_pagmo=ON
)
vcpkg_install_cmake()
@@ -36,5 +39,4 @@ file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/share)
# Handle copyright
-file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/dartsim RENAME copyright)
-
+file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
diff --git a/ports/dartsim/vcpkg.json b/ports/dartsim/vcpkg.json index 0029e6c3c..5f4c00b18 100644 --- a/ports/dartsim/vcpkg.json +++ b/ports/dartsim/vcpkg.json @@ -1,6 +1,7 @@ { "name": "dartsim", - "version-string": "6.9.4", + "version": "6.9.4", + "port-version": 1, "description": "Dynamic Animation and Robotics Toolkit", "homepage": "https://dartsim.github.io/", "dependencies": [ diff --git a/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch new file mode 100644 index 000000000..f41945c28 --- /dev/null +++ b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch @@ -0,0 +1,25 @@ +diff --git a/include/rtc/utils.hpp b/include/rtc/utils.hpp +index f1a82acb..e882042c 100644 +--- a/include/rtc/utils.hpp ++++ b/include/rtc/utils.hpp +@@ -25,6 +25,7 @@ + #include <functional> + #include <memory> + #include <mutex> ++#include <utility> + + namespace rtc { + +@@ -34,11 +35,10 @@ template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + // weak_ptr bind helper + template <typename F, typename T, typename... Args> auto weak_bind(F &&f, T *t, Args &&..._args) { + return [bound = std::bind(f, t, _args...), weak_this = t->weak_from_this()](auto &&...args) { +- using result_type = typename decltype(bound)::result_type; + if (auto shared_this = weak_this.lock()) + return bound(args...); + else +- return static_cast<result_type>(false); ++ return static_cast<decltype(bound(args...))>(false); + }; + } + diff --git a/ports/libdatachannel/portfile.cmake b/ports/libdatachannel/portfile.cmake index 8b1a6c3a9..563224524 100644 --- a/ports/libdatachannel/portfile.cmake +++ b/ports/libdatachannel/portfile.cmake @@ -1,11 +1,12 @@ vcpkg_from_github(
OUT_SOURCE_PATH SOURCE_PATH
REPO paullouisageneau/libdatachannel
- REF v0.12.0
- SHA512 878d0461255620c21597a6b7132ac0c9d1bcba19e32809cdd007039041eacc1f30a73b1c11c43b8b57a82ca1efeae2090822e6c5e13bd98f5afa086ec147e6e7
+ REF 655175d21e58281031c940a94042d5d1fd46efb3 # v 0.12.2
+ SHA512 e1e228bf720ef57130fbb9cc33310cebbdbd16c001455cd56e8746b6ee41bac56da5e5a90235e0a826b52711dc3c95b9d9f56d9e406999f9fd384aee2892578d
HEAD_REF master
PATCHES
fix-for-vcpkg.patch
+ CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch # submitted upstream as https://github.com/paullouisageneau/libdatachannel/pull/413
)
vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
diff --git a/ports/libdatachannel/vcpkg.json b/ports/libdatachannel/vcpkg.json index 6dddc4dc7..7ba32db68 100644 --- a/ports/libdatachannel/vcpkg.json +++ b/ports/libdatachannel/vcpkg.json @@ -1,6 +1,6 @@ { "name": "libdatachannel", - "version-semver": "0.12.0", + "version-semver": "0.12.2", "description": "libdatachannel is a standalone implementation of WebRTC Data Channels, WebRTC Media Transport, and WebSockets in C++17 with C bindings for POSIX platforms (including GNU/Linux, Android, and Apple macOS) and Microsoft Windows.", "homepage": "https://github.com/paullouisageneau/libdatachannel", "dependencies": [ diff --git a/ports/libhv/portfile.cmake b/ports/libhv/portfile.cmake index 01c6f83a5..f54897107 100644 --- a/ports/libhv/portfile.cmake +++ b/ports/libhv/portfile.cmake @@ -14,7 +14,7 @@ string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" BUILD_SHARED) vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA - DISABLE_PARALLEL + DISABLE_PARALLEL_CONFIGURE OPTIONS -DBUILD_EXAMPLES=OFF -DBUILD_UNITTEST=OFF diff --git a/ports/libhv/vcpkg.json b/ports/libhv/vcpkg.json index c6e1a50f7..d16fbeccf 100644 --- a/ports/libhv/vcpkg.json +++ b/ports/libhv/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libhv", "version": "1.0.0", - "port-version": 1, + "port-version": 2, "description": "Libhv is a C/C++ network library similar to libevent/libuv.", "homepage": "https://github.com/ithewei/libhv", "supports": "!(arm | uwp)" diff --git a/ports/libudis86/CONTROL b/ports/libudis86/CONTROL deleted file mode 100644 index 8e67b89cc..000000000 --- a/ports/libudis86/CONTROL +++ /dev/null @@ -1,4 +0,0 @@ -Source: libudis86 -Version: 2018-01-28-56ff6c87 -Homepage: https://github.com/vmt/udis86 -Description: Disassembler Library for x86 and x86-64 diff --git a/ports/libudis86/portfile.cmake b/ports/libudis86/portfile.cmake index a1ae6e539..42d99d38a 100644 --- a/ports/libudis86/portfile.cmake +++ b/ports/libudis86/portfile.cmake @@ -11,8 +11,8 @@ file(COPY ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt DESTINATION ${SOURCE_PATH}) vcpkg_find_acquire_program(PYTHON2) vcpkg_execute_required_process( - COMMAND ${PYTHON2} ${SOURCE_PATH}/scripts/ud_itab.py ${SOURCE_PATH}/docs/x86/optable.xml ${SOURCE_PATH}/libudis86/ - WORKING_DIRECTORY SOURCE_PATH + COMMAND "${PYTHON2}" "${SOURCE_PATH}/scripts/ud_itab.py" "${SOURCE_PATH}/docs/x86/optable.xml" "${SOURCE_PATH}/libudis86/" + WORKING_DIRECTORY "${SOURCE_PATH}" LOGNAME python-${TARGET_TRIPLET}-generate-sources ) @@ -20,12 +20,12 @@ vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA OPTIONS_DEBUG - -DDISABLE_INSTALL_HEADERS=ON - -DDISABLE_INSTALL_TOOLS=ON + -DDISABLE_INSTALL_HEADERS=ON + -DDISABLE_INSTALL_TOOLS=ON ) vcpkg_install_cmake() vcpkg_copy_pdbs() vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/libudis86) -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/libudis86 RENAME copyright) +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/libudis86/vcpkg.json b/ports/libudis86/vcpkg.json new file mode 100644 index 000000000..a687a4104 --- /dev/null +++ b/ports/libudis86/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "libudis86", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1, + "description": "Disassembler Library for x86 and x86-64", + "homepage": "https://github.com/vmt/udis86" +} diff --git a/ports/libvpx/portfile.cmake b/ports/libvpx/portfile.cmake index e888d7fc4..2f3f3680d 100644 --- a/ports/libvpx/portfile.cmake +++ b/ports/libvpx/portfile.cmake @@ -99,14 +99,23 @@ if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) ) # note: pdb file names are hardcoded in the lib file, cannot rename + set(LIBVPX_OUTPUT_PREFIX "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}") if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + endif() endif() if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + endif() endif() if (VCPKG_TARGET_ARCHITECTURE STREQUAL arm64) @@ -130,13 +139,13 @@ else() set(OPTIONS_DEBUG "--enable-debug-libs --enable-debug --prefix=${CURRENT_PACKAGES_DIR}/debug") set(OPTIONS_RELEASE "--prefix=${CURRENT_PACKAGES_DIR}") - + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") set(OPTIONS "${OPTIONS} --disable-static --enable-shared") else() set(OPTIONS "${OPTIONS} --enable-static --disable-shared") endif() - + if(VCPKG_TARGET_ARCHITECTURE STREQUAL x86) set(LIBVPX_TARGET_ARCH "x86") elseif(VCPKG_TARGET_ARCHITECTURE STREQUAL x64) diff --git a/ports/libvpx/vcpkg.json b/ports/libvpx/vcpkg.json index ce3de0341..f0991561c 100644 --- a/ports/libvpx/vcpkg.json +++ b/ports/libvpx/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libvpx", "version-semver": "1.9.0", - "port-version": 8, + "port-version": 9, "description": "The reference software implementation for the video coding formats VP8 and VP9.", "homepage": "https://github.com/webmproject/libvpx" } diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake index 18011257e..4746fcda4 100644 --- a/ports/tensorflow-common/tensorflow-common.cmake +++ b/ports/tensorflow-common/tensorflow-common.cmake @@ -45,7 +45,7 @@ set(ENV{PYTHON_LIB_PATH} "${PYTHON_LIB_PATH}") vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-numpy-${TARGET_TRIPLET}) # tensorflow has long file names, which will not work on windows -set(ENV{TEST_TMPDIR} ${BUILDTREES_DIR}/.bzl) +set(ENV{TEST_TMPDIR} "${CURRENT_BUILDTREES_DIR}/.bzl") set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1) set(ENV{TF_NEED_KAFKA} 0) @@ -439,4 +439,4 @@ else() endif() endif() -message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} and ${BUILDTREES_DIR}/.bzl to free diskspace.") +message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} to free diskspace.") diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json index 7477461b3..447d665b5 100644 --- a/ports/tensorflow-common/vcpkg.json +++ b/ports/tensorflow-common/vcpkg.json @@ -1,6 +1,7 @@ { "name": "tensorflow-common", "version-semver": "2.4.1", + "port-version": 1, "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.", "homepage": "https://github.com/tensorflow/tensorflow" } diff --git a/ports/upb/fix-uwp.patch b/ports/upb/fix-uwp.patch index cbd62f602..c681fa336 100644 --- a/ports/upb/fix-uwp.patch +++ b/ports/upb/fix-uwp.patch @@ -7,7 +7,7 @@ index a6b432f..003615e 100644 endif() +if (MSVC) -+ add_compile_options(/wd4146 -D_CRT_SECURE_NO_WARNINGS) ++ add_compile_options(/wd4146 /wd4703 -D_CRT_SECURE_NO_WARNINGS) +endif() + enable_testing() diff --git a/ports/upb/vcpkg.json b/ports/upb/vcpkg.json index 3213d796f..058833064 100644 --- a/ports/upb/vcpkg.json +++ b/ports/upb/vcpkg.json @@ -1,6 +1,7 @@ { "name": "upb", "version-date": "2020-12-19", + "port-version": 1, "description": "μpb (often written 'upb') is a small protobuf implementation written in C.", "homepage": "https://github.com/protocolbuffers/upb/" } diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 27db6a93d..9f6526cb7 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: MIT
#
variables:
- windows-pool: 'PrWin-2020-12-17'
- linux-pool: 'PrLin-2020-12-16'
+ windows-pool: 'PrWin-2021-04-23'
+ linux-pool: 'PrLin-2021-04-25'
osx-pool: 'PrOsx-2020-09-28'
stages:
diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index c04f4b3ca..6663baed8 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -58,14 +58,13 @@ cd ~ curl -sSL https://get.haskellstack.org/ | sudo sh # Install CUDA -wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -sudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb -sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin +sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo apt -y update -sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 -sudo apt install -y --no-install-recommends libcudnn7-dev +sudo apt install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 \ + cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev libnccl2 libnccl-dev # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 608465100..fddba0eec 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -21,11 +21,18 @@ the compiler rather than for testing vcpkg. Differences: * The machine prefix is changed to VcpkgUnstable instead of PrWin.
* No storage account or "archives" share is provisioned.
* The firewall is not opened to allow communication with Azure Storage.
+
+.PARAMETER CudnnPath
+The path to a CUDNN zip file downloaded from NVidia official sources
+(e.g. https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.1.1.33/11.2_20210301/cudnn-11.2-windows-x64-v8.1.1.33.zip
+downloaded in a browser with an NVidia account logged in.)
#>
[CmdLetBinding()]
Param(
- [switch]$Unstable = $false
+ [switch]$Unstable = $false,
+ [parameter(Mandatory=$true)]
+ [string]$CudnnPath
)
$Location = 'westus2'
@@ -43,15 +50,16 @@ $WindowsServerSku = '2019-Datacenter' $ErrorActionPreference = 'Stop'
$ProgressActivity = 'Creating Scale Set'
-$TotalProgress = 12
-if ($Unstable) {
- $TotalProgress -= 1 # skipping the archives share part
-}
-
+$TotalProgress = 18
$CurrentProgress = 1
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
+if (-Not $CudnnPath.EndsWith('.zip')) {
+ Write-Error 'Expected CudnnPath to be a zip file.'
+ return
+}
+
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
@@ -155,57 +163,84 @@ $VirtualNetwork = New-AzVirtualNetwork ` -Subnet $Subnet
####################################################################################################
-if (-Not $Unstable) {
- Write-Progress `
- -Activity $ProgressActivity `
- -Status 'Creating archives storage account' `
- -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating storage account' `
+ -CurrentOperation 'Initial setup' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
- $StorageAccountName = Sanitize-Name $ResourceGroupName
+$StorageAccountName = Sanitize-Name $ResourceGroupName
- New-AzStorageAccount `
- -ResourceGroupName $ResourceGroupName `
- -Location $Location `
- -Name $StorageAccountName `
- -SkuName 'Standard_LRS' `
- -Kind StorageV2
+New-AzStorageAccount `
+ -ResourceGroupName $ResourceGroupName `
+ -Location $Location `
+ -Name $StorageAccountName `
+ -SkuName 'Standard_LRS' `
+ -Kind StorageV2
- $StorageAccountKeys = Get-AzStorageAccountKey `
- -ResourceGroupName $ResourceGroupName `
- -Name $StorageAccountName
+$StorageAccountKeys = Get-AzStorageAccountKey `
+ -ResourceGroupName $ResourceGroupName `
+ -Name $StorageAccountName
- $StorageAccountKey = $StorageAccountKeys[0].Value
+$StorageAccountKey = $StorageAccountKeys[0].Value
- $StorageContext = New-AzStorageContext `
- -StorageAccountName $StorageAccountName `
- -StorageAccountKey $StorageAccountKey
+$StorageContext = New-AzStorageContext `
+ -StorageAccountName $StorageAccountName `
+ -StorageAccountKey $StorageAccountKey
- New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
- $StartTime = [DateTime]::Now
- $ExpiryTime = $StartTime.AddMonths(6)
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating storage account' `
+ -CurrentOperation 'Uploading cudnn.zip' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
- $SasToken = New-AzStorageAccountSASToken `
- -Service Blob `
- -Permission "racwdlup" `
- -Context $StorageContext `
- -StartTime $StartTime `
- -ExpiryTime $ExpiryTime `
- -ResourceType Service,Container,Object `
- -Protocol HttpsOnly
+New-AzStorageContainer -Name setup -Context $storageContext -Permission blob
- $SasToken = $SasToken.Substring(1) # strip leading ?
+Set-AzStorageBlobContent -File $CudnnPath `
+ -Container 'setup' `
+ -Blob 'cudnn.zip' `
+ -Context $StorageContext
- # Note that we put the storage account into the firewall after creating the above SAS token or we
- # would be denied since the person running this script isn't one of the VMs we're creating here.
- Set-AzStorageAccount `
- -ResourceGroupName $ResourceGroupName `
- -AccountName $StorageAccountName `
- -NetworkRuleSet ( `
- @{bypass="AzureServices"; `
- virtualNetworkRules=( `
- @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
- defaultAction="Deny"})
-}
+$CudnnBlobUrl = "https://$StorageAccountName.blob.core.windows.net/setup/cudnn.zip"
+
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating storage account' `
+ -CurrentOperation 'Creating archives container' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
+
+New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
+
+$StartTime = [DateTime]::Now
+$ExpiryTime = $StartTime.AddMonths(6)
+
+$SasToken = New-AzStorageAccountSASToken `
+ -Service Blob `
+ -Permission "racwdlup" `
+ -Context $StorageContext `
+ -StartTime $StartTime `
+ -ExpiryTime $ExpiryTime `
+ -ResourceType Service,Container,Object `
+ -Protocol HttpsOnly
+
+$SasToken = $SasToken.Substring(1) # strip leading ?
+
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Creating storage account' `
+ -CurrentOperation 'Locking down network' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
+
+# Note that we put the storage account into the firewall after creating the above SAS token or we
+# would be denied since the person running this script isn't one of the VMs we're creating here.
+Set-AzStorageAccount `
+ -ResourceGroupName $ResourceGroupName `
+ -AccountName $StorageAccountName `
+ -NetworkRuleSet ( `
+ @{bypass="AzureServices"; `
+ virtualNetworkRules=( `
+ @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
+ defaultAction="Deny"})
####################################################################################################
Write-Progress `
@@ -245,30 +280,128 @@ New-AzVm ` ####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
- -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' `
+ -Status 'Running provisioning script deploy-psexec.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
-$provisionParameters = @{AdminUserPassword = $AdminPW;}
-if (-Not $Unstable) {
- $provisionParameters['StorageAccountName'] = $StorageAccountName
- $provisionParameters['StorageAccountSasToken'] = $SasToken
+$DeployPsExecResult = Invoke-AzVMRunCommand `
+ -ResourceGroupName $ResourceGroupName `
+ -VMName $ProtoVMName `
+ -CommandId 'RunPowerShellScript' `
+ -ScriptPath "$PSScriptRoot\deploy-psexec.ps1"
+
+Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)"
+
+####################################################################################################
+function Invoke-ScriptWithPrefix {
+ param(
+ [string]$ScriptName,
+ [switch]$AddAdminPw,
+ [switch]$AddCudnnUrl
+ )
+
+ Write-Progress `
+ -Activity $ProgressActivity `
+ -Status "Running provisioning script $ScriptName in VM" `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+ $DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw
+ $UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw
+
+ $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
+ try {
+ $script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw
+ if ($AddAdminPw) {
+ $script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix)
+ }
+
+ if ($AddCudnnUrl) {
+ $script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnBlobUrl'")
+ }
+
+ $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent);
+ Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
+
+ $parameter = $null
+ if ($AddAdminPw) {
+ $parameter = @{AdminUserPassword = $AdminPW;}
+ }
+
+ $InvokeResult = Invoke-AzVMRunCommand `
+ -ResourceGroupName $ResourceGroupName `
+ -VMName $ProtoVMName `
+ -CommandId 'RunPowerShellScript' `
+ -ScriptPath $tempScriptFilename `
+ -Parameter $parameter
+
+ Write-Host "$ScriptName output: $($InvokeResult.value.Message)"
+ } finally {
+ Remove-Item $tempScriptFilename -Force
+ }
}
+Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-wdk.ps1' -AddAdminPw
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -AddCudnnUrl
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Invoke-ScriptWithPrefix -ScriptName 'deploy-pwsh.ps1' -AddAdminPw
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Write-Progress `
+ -Activity $ProgressActivity `
+ -Status 'Running provisioning script deploy-settings.txt (as a .ps1) in VM' `
+ -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
$ProvisionImageResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
- -ScriptPath "$PSScriptRoot\provision-image.txt" `
- -Parameter $provisionParameters
+ -ScriptPath "$PSScriptRoot\deploy-settings.txt"
-Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)"
+Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)"
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
- -Status 'Restarting VM' `
+ -Status 'Deploying SAS token into VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
+try {
+ $script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " `
+ + "-Name PROVISIONED_AZURE_STORAGE_NAME " `
+ + "-Value '$StorageAccountName'`r`n" `
+ + "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " `
+ + "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " `
+ + "-Value '$SasToken'`r`n"
+
+ Write-Host "Script content is:"
+ Write-Host $script
+
+ Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
+ $InvokeResult = Invoke-AzVMRunCommand `
+ -ResourceGroupName $ResourceGroupName `
+ -VMName $ProtoVMName `
+ -CommandId 'RunPowerShellScript' `
+ -ScriptPath $tempScriptFilename
+
+ Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)"
+} finally {
+ Remove-Item $tempScriptFilename -Force
+}
+
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 new file mode 100644 index 000000000..ab1cc1c54 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1
+
+# REPLACE WITH UTILITY-PREFIX.ps1
+
+# REPLACE WITH $CudnnUrl
+
+$CudnnLocalZipPath = "$PSScriptRoot\cudnn-11.2-windows-x64-v8.1.1.33.zip"
+
+$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe'
+
+$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + `
+ 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + `
+ 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + `
+ 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + `
+ 'occupancy_calculator_11.3 thrust_11.3 '
+
+$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.3"
+
+try {
+ Write-Host 'Downloading CUDA...'
+ [string]$installerPath = Get-TempFilePath -Extension 'exe'
+ curl.exe -L -o $installerPath -s -S $CudaUrl
+ Write-Host 'Installing CUDA...'
+ $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $CudaFeatures) -Wait -PassThru
+ $exitCode = $proc.ExitCode
+ if ($exitCode -eq 0) {
+ Write-Host 'Installation successful!'
+ }
+ else {
+ Write-Error "Installation failed! Exited with $exitCode."
+ throw
+ }
+}
+catch {
+ Write-Error "Failed to install CUDA! $($_.Exception.Message)"
+ throw
+}
+
+try {
+ if ([string]::IsNullOrWhiteSpace($CudnnUrl)) {
+ if (-Not (Test-Path $CudnnLocalZipPath)) {
+ throw "CUDNN zip ($CudnnLocalZipPath) was missing, please download from NVidia and place next to this script."
+ }
+
+ $cudnnZipPath = $CudnnLocalZipPath
+ } else {
+ Write-Host 'Downloading CUDNN...'
+ $cudnnZipPath = Get-TempFilePath -Extension 'zip'
+ curl.exe -L -o $cudnnZipPath -s -S $CudnnUrl
+ }
+
+ Write-Host "Installing CUDNN to $destination..."
+ tar.exe -xvf "$cudnnZipPath" --strip 1 --directory "$destination"
+ Write-Host 'Installation successful!'
+}
+catch {
+ Write-Error "Failed to install CUDNN! $($_.Exception.Message)"
+ throw
+}
diff --git a/scripts/azure-pipelines/windows/deploy-mpi.ps1 b/scripts/azure-pipelines/windows/deploy-mpi.ps1 new file mode 100644 index 000000000..ba369f797 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-mpi.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1
+
+# REPLACE WITH UTILITY-PREFIX.ps1
+
+$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe'
+
+<#
+.SYNOPSIS
+Installs MPI
+
+.DESCRIPTION
+Downloads the MPI installer located at $Url, and installs it with the
+correct flags.
+
+.PARAMETER Url
+The URL of the installer.
+#>
+Function InstallMpi {
+ Param(
+ [String]$Url
+ )
+
+ try {
+ Write-Host 'Downloading MPI...'
+ [string]$installerPath = Get-TempFilePath -Extension 'exe'
+ curl.exe -L -o $installerPath -s -S $Url
+ Write-Host 'Installing MPI...'
+ $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru
+ $exitCode = $proc.ExitCode
+ if ($exitCode -eq 0) {
+ Write-Host 'Installation successful!'
+ }
+ else {
+ Write-Error "Installation failed! Exited with $exitCode."
+ throw
+ }
+ }
+ catch {
+ Write-Error "Failed to install MPI! $($_.Exception.Message)"
+ throw
+ }
+}
+
+InstallMpi -Url $MpiUrl
diff --git a/scripts/azure-pipelines/windows/deploy-psexec.ps1 b/scripts/azure-pipelines/windows/deploy-psexec.ps1 new file mode 100644 index 000000000..2a1f6fb14 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-psexec.ps1 @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+$PsExecPath = 'C:\PsExec64.exe'
+Write-Host "Downloading psexec to: $PsExecPath"
+& curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe
diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 new file mode 100644 index 000000000..b766385a3 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1
+
+# REPLACE WITH UTILITY-PREFIX.ps1
+
+$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi'
+InstallMSI -Url $PwshUrl -Name 'PowerShell Core'
diff --git a/scripts/azure-pipelines/windows/deploy-settings.txt b/scripts/azure-pipelines/windows/deploy-settings.txt new file mode 100644 index 000000000..a80ddce39 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-settings.txt @@ -0,0 +1,20 @@ +$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+
+Write-Host 'Disabling pagefile...'
+wmic computersystem set AutomaticManagedPagefile=False
+wmic pagefileset delete
+
+$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue
+if ($null -eq $av) {
+ Write-Host 'AntiVirus not installed, skipping exclusions.'
+} else {
+ Write-Host 'Configuring AntiVirus exclusions...'
+ Add-MpPreference -ExclusionPath C:\agent
+ Add-MPPreference -ExclusionPath D:\
+ Add-MPPreference -ExclusionProcess ninja.exe
+ Add-MPPreference -ExclusionProcess clang-cl.exe
+ Add-MPPreference -ExclusionProcess cl.exe
+ Add-MPPreference -ExclusionProcess link.exe
+ Add-MPPreference -ExclusionProcess python.exe
+}
diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 new file mode 100644 index 000000000..2de2de919 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1
+
+# REPLACE WITH UTILITY-PREFIX.ps1
+
+$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe'
+$Workloads = @(
+ 'Microsoft.VisualStudio.Workload.NativeDesktop',
+ 'Microsoft.VisualStudio.Workload.Universal',
+ 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
+ 'Microsoft.VisualStudio.Component.VC.Tools.ARM',
+ 'Microsoft.VisualStudio.Component.VC.Tools.ARM64',
+ 'Microsoft.VisualStudio.Component.VC.ATL',
+ 'Microsoft.VisualStudio.Component.VC.ATLMFC',
+ 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre',
+ 'Microsoft.VisualStudio.Component.Windows10SDK.18362',
+ 'Microsoft.VisualStudio.Component.Windows10SDK.19041',
+ 'Microsoft.Net.Component.4.8.SDK',
+ 'Microsoft.Component.NetFX.Native',
+ 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
+ 'Microsoft.VisualStudio.Component.VC.Llvm.Clang',
+ 'Microsoft.VisualStudio.Component.VC.v141.x86.x64',
+ 'Microsoft.VisualStudio.Component.VC.140'
+)
+
+<#
+.SYNOPSIS
+Install Visual Studio.
+
+.DESCRIPTION
+InstallVisualStudio takes the $Workloads array, and installs it with the
+installer that's pointed at by $BootstrapperUrl.
+
+.PARAMETER Workloads
+The set of VS workloads to install.
+
+.PARAMETER BootstrapperUrl
+The URL of the Visual Studio installer, i.e. one of vs_*.exe.
+
+.PARAMETER InstallPath
+The path to install Visual Studio at.
+
+.PARAMETER Nickname
+The nickname to give the installation.
+#>
+Function InstallVisualStudio {
+ Param(
+ [String[]]$Workloads,
+ [String]$BootstrapperUrl,
+ [String]$InstallPath = $null,
+ [String]$Nickname = $null
+ )
+
+ try {
+ Write-Host 'Downloading Visual Studio...'
+ [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe'
+ curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl
+ Write-Host 'Installing Visual Studio...'
+ $vsArgs = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache')
+ foreach ($workload in $Workloads) {
+ $vsArgs += '--add'
+ $vsArgs += $workload
+ }
+
+ if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) {
+ $vsArgs += '--installpath'
+ $vsArgs += $InstallPath
+ }
+
+ if (-not ([String]::IsNullOrWhiteSpace($Nickname))) {
+ $vsArgs += '--nickname'
+ $vsArgs += $Nickname
+ }
+
+ $proc = Start-Process -FilePath cmd.exe -ArgumentList $vsArgs -Wait -PassThru
+ PrintMsiExitCodeMessage $proc.ExitCode
+ }
+ catch {
+ Write-Error "Failed to install Visual Studio! $($_.Exception.Message)"
+ throw
+ }
+}
+
+InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable'
diff --git a/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 new file mode 100644 index 000000000..d145f79d7 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1
+
+# REPLACE WITH UTILITY-PREFIX.ps1
+
+$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854'
+
+<#
+.SYNOPSIS
+Installs Windows WDK version 2004
+
+.DESCRIPTION
+Downloads the Windows WDK installer located at $Url, and installs it with the
+correct flags.
+
+.PARAMETER Url
+The URL of the installer.
+#>
+Function InstallWindowsWDK {
+ Param(
+ [String]$Url
+ )
+
+ try {
+ Write-Host 'Downloading Windows WDK...'
+ [string]$installerPath = Get-TempFilePath -Extension 'exe'
+ curl.exe -L -o $installerPath -s -S $Url
+ Write-Host 'Installing Windows WDK...'
+ $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru
+ $exitCode = $proc.ExitCode
+ if ($exitCode -eq 0) {
+ Write-Host 'Installation successful!'
+ }
+ else {
+ Write-Error "Installation failed! Exited with $exitCode."
+ throw
+ }
+ }
+ catch {
+ Write-Error "Failed to install Windows WDK! $($_.Exception.Message)"
+ throw
+ }
+}
+
+InstallWindowsWDK -Url $WindowsWDKUrl
diff --git a/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 new file mode 100644 index 000000000..b4592eabd --- /dev/null +++ b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 @@ -0,0 +1,27 @@ +param(
+ [string]$AdminUserPassword = $null
+)
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+if (-Not [string]::IsNullOrEmpty($AdminUserPassword)) {
+ $PsExecPath = 'C:\PsExec64.exe'
+ $PsExecArgs = @(
+ '-u',
+ 'AdminUser',
+ '-p',
+ $AdminUserPassword,
+ '-accepteula',
+ '-i',
+ '-h',
+ 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe',
+ '-ExecutionPolicy',
+ 'Unrestricted',
+ '-File',
+ $PSCommandPath
+ )
+
+ Write-Host "Executing: $PsExecPath $PsExecArgs"
+ $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru
+ exit $proc.ExitCode
+}
diff --git a/scripts/azure-pipelines/windows/provision-entire-image.ps1 b/scripts/azure-pipelines/windows/provision-entire-image.ps1 new file mode 100644 index 000000000..3bfb5dbd9 --- /dev/null +++ b/scripts/azure-pipelines/windows/provision-entire-image.ps1 @@ -0,0 +1,16 @@ +# This script runs all the scripts we run on Azure machines to deploy prerequisites,
+# and assumes it is being run as an admin user.
+
+. "$PSScriptRoot\utility-prefix.ps1"
+
+. "$PSScriptRoot\deploy-visual-studio.ps1"
+. "$PSScriptRoot\deploy-windows-wdk.ps1"
+. "$PSScriptRoot\deploy-mpi.ps1"
+. "$PSScriptRoot\deploy-cuda.ps1"
+. "$PSScriptRoot\deploy-pwsh.ps1"
+try {
+ Copy-Item "$PSScriptRoot\deploy-settings.txt" "$PSScriptRoot\deploy-settings.ps1"
+ . "$PSScriptRoot\deploy-settings.ps1"
+} finally {
+ Remove-Item "$PSScriptRoot\deploy-settings.ps1"
+}
diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt deleted file mode 100644 index e3e04be28..000000000 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ /dev/null @@ -1,454 +0,0 @@ -# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: MIT
-
-<#
-.SYNOPSIS
-Sets up a machine to be an image for a scale set.
-
-.DESCRIPTION
-provision-image.ps1 runs on an existing, freshly provisioned virtual machine,
-and sets up that virtual machine as a build machine. After this is done,
-(outside of this script), we take that machine and make it an image to be copied
-for setting up new VMs in the scale set.
-
-This script must either be run as admin, or one must pass AdminUserPassword;
-if the script is run with AdminUserPassword, it runs itself again as an
-administrator.
-
-.PARAMETER AdminUserPassword
-The administrator user's password; if this is $null, or not passed, then the
-script assumes it's running on an administrator account.
-
-.PARAMETER StorageAccountName
-The name of the storage account. Stored in the environment variable %PROVISIONED_AZURE_STORAGE_NAME%.
-Used by the CI system to access the global storage.
-
-.PARAMETER StorageAccountSasToken
-The SAS token to access the storage account. Stored in the environment variable
-%PROVISIONED_AZURE_STORAGE_SAS_TOKEN%.
-Used by the CI system to access the global storage.
-#>
-param(
- [string]$AdminUserPassword = $null,
- [string]$StorageAccountName = $null,
- [string]$StorageAccountSasToken = $null
-)
-
-$ErrorActionPreference = 'Stop'
-
-<#
-.SYNOPSIS
-Gets a random file path in the temp directory.
-
-.DESCRIPTION
-Get-TempFilePath takes an extension, and returns a path with a random
-filename component in the temporary directory with that extension.
-
-.PARAMETER Extension
-The extension to use for the path.
-#>
-Function Get-TempFilePath {
- Param(
- [String]$Extension
- )
-
- if ([String]::IsNullOrWhiteSpace($Extension)) {
- throw 'Missing Extension'
- }
-
- $tempPath = [System.IO.Path]::GetTempPath()
- $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension
- return Join-Path $tempPath $tempName
-}
-
-$TranscriptPath = 'C:\provision-image-transcript.txt'
-
-if ([string]::IsNullOrEmpty($AdminUserPassword)) {
- Start-Transcript -Path $TranscriptPath
-} else {
- Write-Host 'AdminUser password supplied; switching to AdminUser.'
- $PsExecPath = Get-TempFilePath -Extension 'exe'
- Write-Host "Downloading psexec to: $PsExecPath"
- & curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe
- $PsExecArgs = @(
- '-u',
- 'AdminUser',
- '-p',
- $AdminUserPassword,
- '-accepteula',
- '-h',
- 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe',
- '-ExecutionPolicy',
- 'Unrestricted',
- '-File',
- $PSCommandPath
- )
-
- if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountName))) {
- $PsExecArgs += '-StorageAccountName'
- $PsExecArgs += $StorageAccountName
- }
-
- if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountSasToken))) {
- $PsExecArgs += '-StorageAccountSasToken'
- $PsExecArgs += $StorageAccountSasToken
- }
-
- Write-Host "Executing: $PsExecPath $PsExecArgs"
-
- $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru
- Write-Host 'Reading transcript...'
- Get-Content -Path $TranscriptPath
- Write-Host 'Cleaning up...'
- Remove-Item $PsExecPath
- exit $proc.ExitCode
-}
-
-$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe'
-$Workloads = @(
- 'Microsoft.VisualStudio.Workload.NativeDesktop',
- 'Microsoft.VisualStudio.Workload.Universal',
- 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
- 'Microsoft.VisualStudio.Component.VC.Tools.ARM',
- 'Microsoft.VisualStudio.Component.VC.Tools.ARM64',
- 'Microsoft.VisualStudio.Component.VC.ATL',
- 'Microsoft.VisualStudio.Component.VC.ATLMFC',
- 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre',
- 'Microsoft.VisualStudio.Component.Windows10SDK.18362',
- 'Microsoft.VisualStudio.Component.Windows10SDK.19041',
- 'Microsoft.Net.Component.4.8.SDK',
- 'Microsoft.Component.NetFX.Native',
- 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
- 'Microsoft.VisualStudio.Component.VC.Llvm.Clang',
- 'Microsoft.VisualStudio.Component.VC.v141.x86.x64',
- 'Microsoft.VisualStudio.Component.VC.140'
-)
-
-$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854'
-
-$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe'
-
-$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe'
-$CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + `
- 'nvdisasm_10.1 nvprof_10.1 visual_profiler_10.1 visual_studio_integration_10.1 cublas_10.1 cublas_dev_10.1 ' + `
- 'cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 ' + `
- 'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + `
- 'occupancy_calculator_10.1 fortran_examples_10.1'
-
-$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi'
-
-$ErrorActionPreference = 'Stop'
-$ProgressPreference = 'SilentlyContinue'
-
-<#
-.SYNOPSIS
-Writes a message to the screen depending on ExitCode.
-
-.DESCRIPTION
-Since msiexec can return either 0 or 3010 successfully, in both cases
-we write that installation succeeded, and which exit code it exited with.
-If msiexec returns anything else, we write an error.
-
-.PARAMETER ExitCode
-The exit code that msiexec returned.
-#>
-Function PrintMsiExitCodeMessage {
- Param(
- $ExitCode
- )
-
- # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED
- if ($ExitCode -eq 0 -or $ExitCode -eq 3010) {
- Write-Host "Installation successful! Exited with $ExitCode."
- }
- else {
- Write-Error "Installation failed! Exited with $ExitCode."
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Install Visual Studio.
-
-.DESCRIPTION
-InstallVisualStudio takes the $Workloads array, and installs it with the
-installer that's pointed at by $BootstrapperUrl.
-
-.PARAMETER Workloads
-The set of VS workloads to install.
-
-.PARAMETER BootstrapperUrl
-The URL of the Visual Studio installer, i.e. one of vs_*.exe.
-
-.PARAMETER InstallPath
-The path to install Visual Studio at.
-
-.PARAMETER Nickname
-The nickname to give the installation.
-#>
-Function InstallVisualStudio {
- Param(
- [String[]]$Workloads,
- [String]$BootstrapperUrl,
- [String]$InstallPath = $null,
- [String]$Nickname = $null
- )
-
- try {
- Write-Host 'Downloading Visual Studio...'
- [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe'
- curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl
- Write-Host 'Installing Visual Studio...'
- $args = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache')
- foreach ($workload in $Workloads) {
- $args += '--add'
- $args += $workload
- }
-
- if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) {
- $args += '--installpath'
- $args += $InstallPath
- }
-
- if (-not ([String]::IsNullOrWhiteSpace($Nickname))) {
- $args += '--nickname'
- $args += $Nickname
- }
-
- $proc = Start-Process -FilePath cmd.exe -ArgumentList $args -Wait -PassThru
- PrintMsiExitCodeMessage $proc.ExitCode
- }
- catch {
- Write-Error "Failed to install Visual Studio! $($_.Exception.Message)"
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Install a .msi file.
-
-.DESCRIPTION
-InstallMSI takes a url where an .msi lives, and installs that .msi to the system.
-
-.PARAMETER Name
-The name of the thing to install.
-
-.PARAMETER Url
-The URL at which the .msi lives.
-#>
-Function InstallMSI {
- Param(
- [String]$Name,
- [String]$Url
- )
-
- try {
- Write-Host "Downloading $Name..."
- [string]$msiPath = Get-TempFilePath -Extension 'msi'
- curl.exe -L -o $msiPath -s -S $Url
- Write-Host "Installing $Name..."
- $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn')
- $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru
- PrintMsiExitCodeMessage $proc.ExitCode
- }
- catch {
- Write-Error "Failed to install $Name! $($_.Exception.Message)"
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Unpacks a zip file to $Dir.
-
-.DESCRIPTION
-InstallZip takes a URL of a zip file, and unpacks the zip file to the directory
-$Dir.
-
-.PARAMETER Name
-The name of the tool being installed.
-
-.PARAMETER Url
-The URL of the zip file to unpack.
-
-.PARAMETER Dir
-The directory to unpack the zip file to.
-#>
-Function InstallZip {
- Param(
- [String]$Name,
- [String]$Url,
- [String]$Dir
- )
-
- try {
- Write-Host "Downloading $Name..."
- [string]$zipPath = Get-TempFilePath -Extension 'zip'
- curl.exe -L -o $zipPath -s -S $Url
- Write-Host "Installing $Name..."
- Expand-Archive -Path $zipPath -DestinationPath $Dir -Force
- }
- catch {
- Write-Error "Failed to install $Name! $($_.Exception.Message)"
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Installs Windows WDK version 2004
-
-.DESCRIPTION
-Downloads the Windows WDK installer located at $Url, and installs it with the
-correct flags.
-
-.PARAMETER Url
-The URL of the installer.
-#>
-Function InstallWindowsWDK {
- Param(
- [String]$Url
- )
-
- try {
- Write-Host 'Downloading Windows WDK...'
- [string]$installerPath = Get-TempFilePath -Extension 'exe'
- curl.exe -L -o $installerPath -s -S $Url
- Write-Host 'Installing Windows WDK...'
- $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru
- $exitCode = $proc.ExitCode
- if ($exitCode -eq 0) {
- Write-Host 'Installation successful!'
- }
- else {
- Write-Error "Installation failed! Exited with $exitCode."
- throw
- }
- }
- catch {
- Write-Error "Failed to install Windows WDK! $($_.Exception.Message)"
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Installs MPI
-
-.DESCRIPTION
-Downloads the MPI installer located at $Url, and installs it with the
-correct flags.
-
-.PARAMETER Url
-The URL of the installer.
-#>
-Function InstallMpi {
- Param(
- [String]$Url
- )
-
- try {
- Write-Host 'Downloading MPI...'
- [string]$installerPath = Get-TempFilePath -Extension 'exe'
- curl.exe -L -o $installerPath -s -S $Url
- Write-Host 'Installing MPI...'
- $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru
- $exitCode = $proc.ExitCode
- if ($exitCode -eq 0) {
- Write-Host 'Installation successful!'
- }
- else {
- Write-Error "Installation failed! Exited with $exitCode."
- throw
- }
- }
- catch {
- Write-Error "Failed to install MPI! $($_.Exception.Message)"
- throw
- }
-}
-
-<#
-.SYNOPSIS
-Installs NVIDIA's CUDA Toolkit.
-
-.DESCRIPTION
-InstallCuda installs the CUDA Toolkit with the features specified as a
-space-separated list of strings in $Features.
-
-.PARAMETER Url
-The URL of the CUDA installer.
-
-.PARAMETER Features
-A space-separated list of features to install.
-#>
-Function InstallCuda {
- Param(
- [String]$Url,
- [String]$Features
- )
-
- try {
- Write-Host 'Downloading CUDA...'
- [string]$installerPath = Get-TempFilePath -Extension 'exe'
- curl.exe -L -o $installerPath -s -S $Url
- Write-Host 'Installing CUDA...'
- $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $Features) -Wait -PassThru
- $exitCode = $proc.ExitCode
- if ($exitCode -eq 0) {
- Write-Host 'Installation successful!'
- }
- else {
- Write-Error "Installation failed! Exited with $exitCode."
- throw
- }
- }
- catch {
- Write-Error "Failed to install CUDA! $($_.Exception.Message)"
- throw
- }
-}
-
-Write-Host "AdminUser password not supplied; assuming already running as AdminUser"
-
-Write-Host 'Disabling pagefile...'
-wmic computersystem set AutomaticManagedPagefile=False
-wmic pagefileset delete
-
-$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue
-if ($null -eq $av) {
- Write-Host 'AntiVirus not installed, skipping exclusions.'
-} else {
- Write-Host 'Configuring AntiVirus exclusions...'
- Add-MpPreference -ExclusionPath C:\agent
- Add-MPPreference -ExclusionPath D:\
- Add-MPPreference -ExclusionProcess ninja.exe
- Add-MPPreference -ExclusionProcess clang-cl.exe
- Add-MPPreference -ExclusionProcess cl.exe
- Add-MPPreference -ExclusionProcess link.exe
- Add-MPPreference -ExclusionProcess python.exe
-}
-
-InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable'
-InstallWindowsWDK -Url $WindowsWDKUrl
-InstallMpi -Url $MpiUrl
-InstallCuda -Url $CudaUrl -Features $CudaFeatures
-InstallMSI -Url $PwshUrl -Name 'PowerShell Core'
-if ([string]::IsNullOrWhiteSpace($StorageAccountName)) {
- Write-Host 'No storage account name configured.'
-} else {
- Write-Host 'Storing storage account name to environment'
- Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `
- -Name PROVISIONED_AZURE_STORAGE_NAME `
- -Value $StorageAccountName
-}
-if ([string]::IsNullOrWhiteSpace($StorageAccountSasToken)) {
- Write-Host 'No storage account key configured.'
-} else {
- Write-Host 'Storing storage account key to environment'
- Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `
- -Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN `
- -Value $StorageAccountSasToken
-}
diff --git a/scripts/azure-pipelines/windows/utility-prefix.ps1 b/scripts/azure-pipelines/windows/utility-prefix.ps1 new file mode 100644 index 000000000..8cd0066db --- /dev/null +++ b/scripts/azure-pipelines/windows/utility-prefix.ps1 @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+
+<#
+.SYNOPSIS
+Gets a random file path in the temp directory.
+
+.DESCRIPTION
+Get-TempFilePath takes an extension, and returns a path with a random
+filename component in the temporary directory with that extension.
+
+.PARAMETER Extension
+The extension to use for the path.
+#>
+Function Get-TempFilePath {
+ Param(
+ [String]$Extension
+ )
+
+ if ([String]::IsNullOrWhiteSpace($Extension)) {
+ throw 'Missing Extension'
+ }
+
+ $tempPath = [System.IO.Path]::GetTempPath()
+ $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension
+ return Join-Path $tempPath $tempName
+}
+
+<#
+.SYNOPSIS
+Writes a message to the screen depending on ExitCode.
+
+.DESCRIPTION
+Since msiexec can return either 0 or 3010 successfully, in both cases
+we write that installation succeeded, and which exit code it exited with.
+If msiexec returns anything else, we write an error.
+
+.PARAMETER ExitCode
+The exit code that msiexec returned.
+#>
+Function PrintMsiExitCodeMessage {
+ Param(
+ $ExitCode
+ )
+
+ # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED
+ if ($ExitCode -eq 0 -or $ExitCode -eq 3010) {
+ Write-Host "Installation successful! Exited with $ExitCode."
+ }
+ else {
+ Write-Error "Installation failed! Exited with $ExitCode."
+ throw
+ }
+}
+
+<#
+.SYNOPSIS
+Install a .msi file.
+
+.DESCRIPTION
+InstallMSI takes a url where an .msi lives, and installs that .msi to the system.
+
+.PARAMETER Name
+The name of the thing to install.
+
+.PARAMETER Url
+The URL at which the .msi lives.
+#>
+Function InstallMSI {
+ Param(
+ [String]$Name,
+ [String]$Url
+ )
+
+ try {
+ Write-Host "Downloading $Name..."
+ [string]$msiPath = Get-TempFilePath -Extension 'msi'
+ curl.exe -L -o $msiPath -s -S $Url
+ Write-Host "Installing $Name..."
+ $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn')
+ $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru
+ PrintMsiExitCodeMessage $proc.ExitCode
+ }
+ catch {
+ Write-Error "Failed to install $Name! $($_.Exception.Message)"
+ throw
+ }
+}
+
+<#
+.SYNOPSIS
+Unpacks a zip file to $Dir.
+
+.DESCRIPTION
+InstallZip takes a URL of a zip file, and unpacks the zip file to the directory
+$Dir.
+
+.PARAMETER Name
+The name of the tool being installed.
+
+.PARAMETER Url
+The URL of the zip file to unpack.
+
+.PARAMETER Dir
+The directory to unpack the zip file to.
+#>
+Function InstallZip {
+ Param(
+ [String]$Name,
+ [String]$Url,
+ [String]$Dir
+ )
+
+ try {
+ Write-Host "Downloading $Name..."
+ [string]$zipPath = Get-TempFilePath -Extension 'zip'
+ curl.exe -L -o $zipPath -s -S $Url
+ Write-Host "Installing $Name..."
+ Expand-Archive -Path $zipPath -DestinationPath $Dir -Force
+ }
+ catch {
+ Write-Error "Failed to install $Name! $($_.Exception.Message)"
+ throw
+ }
+}
diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 01b33f2d1..9ced1bd96 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1784,7 +1784,6 @@ clapack:x64-uwp=skip lapack-reference:arm64-windows=skip lapack-reference:arm-uwp=skip - # failures for x64-windows-static-md ace:x64-windows-static-md=fail activemq-cpp:x64-windows-static-md=fail @@ -1819,3 +1818,15 @@ zyre:x64-windows-static-md=fail usbmuxd:x64-windows-static-md=fail workflow:x64-uwp=fail workflow:arm-uwp=fail + +# wangle triggers an internal compiler error +# https://devdiv.visualstudio.com/DefaultCollection/DevDiv/_workitems/edit/1269468 +wangle:x64-windows=fail +wangle:x64-windows-static=fail +wangle:x64-windows-static-md=fail + +# VS2019 version 16.9.4's project system changes where PDBs are placed in a way that breaks the +# upstream build script of this port. +# See https://developercommunity.visualstudio.com/t/Toolset-169-regression-vcxproj-producin/1356639 +dimcli:x64-windows-static-md=fail +dimcli:x64-windows-static=fail diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml index 50e0476fe..3d48d5294 100644 --- a/scripts/vcpkgTools.xml +++ b/scripts/vcpkgTools.xml @@ -8,25 +8,25 @@ <archiveName>python-3.9.2-embed-win32.zip</archiveName> </tool> <tool name="cmake" os="windows"> - <version>3.19.2</version> - <exeRelativePath>cmake-3.19.2-win32-x86\bin\cmake.exe</exeRelativePath> - <url>https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-win32-x86.zip</url> - <sha512>1ed3706d8dd5c5244ceb4c22443f006fcb9ffa76cb6d9ebdbbb8f11a50deb3f92a9a4080ff3fd125f79c0143f553ea0c02a6a2d27521438705c2fea4dd97d239</sha512> - <archiveName>cmake-3.19.2-win32-x86.zip</archiveName> + <version>3.20.1</version> + <exeRelativePath>cmake-3.20.1-windows-i386\bin\cmake.exe</exeRelativePath> + <url>https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-windows-i386.zip</url> + <sha512>bc0be4e32bbffe102b2cb970591118bf2132d6afd6c97469cf46a4f772627fa81a8a57e4069dc32a93c0ffcce176e562d9d008e6780f053876e5d5eb97051554</sha512> + <archiveName>cmake-3.20.1-windows-i386.zip</archiveName> </tool> <tool name="cmake" os="osx"> - <version>3.19.2</version> - <exeRelativePath>cmake-3.19.2-macos-universal/CMake.app/Contents/bin/cmake</exeRelativePath> - <url>https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-macos-universal.tar.gz</url> - <sha512>031196439a26d6ff2a1f42830ca26b5159752dc2c2c889a4e304327c858157b68088905b10727090918c3605fe7fe429aa19d46e68ed30c17d4d89b3d209aae8</sha512> - <archiveName>cmake-3.19.2-macos-universal.tar.gz</archiveName> + <version>3.20.1</version> + <exeRelativePath>cmake-3.20.1-macos-universal/CMake.app/Contents/bin/cmake</exeRelativePath> + <url>https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-macos-universal.tar.gz</url> + <sha512>fd17d09a0639833a2aa5f30045cfcbbb30e0cfa5092d2f22f0bc502ea361a458d26be96437fbfb6b5e68b73954050c0256653af7e2e20bb253a878e206992135</sha512> + <archiveName>cmake-3.20.1-macos-universal.tar.gz</archiveName> </tool> <tool name="cmake" os="linux"> - <version>3.19.2</version> - <exeRelativePath>cmake-3.19.2-Linux-x86_64/bin/cmake</exeRelativePath> - <url>https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-Linux-x86_64.tar.gz</url> - <sha512>67ea7ee428e6c93039646d6d0cbe84c0be337a2b878487635fee71ec2c8992a9078a660e6b793ef11ce13530def44a16b716af96e10a97675d014a26caa0aed4</sha512> - <archiveName>cmake-3.19.2-Linux-x86_64.tar.gz</archiveName> + <version>3.20.1</version> + <exeRelativePath>cmake-3.20.1-linux-x86_64/bin/cmake</exeRelativePath> + <url>https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-Linux-x86_64.tar.gz</url> + <sha512>e14b6cbc03abfbae074c36e34f90eea94af02e727e75ed597f748203e2ce7483ffe693bd173cb02a1de7ac55254a891da5505c43b92528adc918eff9e8b024e6</sha512> + <archiveName>cmake-3.20.1-linux-x86_64.tar.gz</archiveName> </tool> <tool name="cmake" os="freebsd"> <version>3.12.4</version> @@ -134,10 +134,10 @@ <archiveName>ninja-freebsd-1.8.2.zip</archiveName> </tool> <tool name="powershell-core" os="windows"> - <version>7.1.0</version> + <version>7.1.3</version> <exeRelativePath>pwsh.exe</exeRelativePath> - <url>https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x86.zip</url> - <sha512>ea6ed619c784af65b09a5e98cdfc0d41333ba7373b7409e488ec70aaf1e0c4e5568e846d2b1748d72c3e3a9c1a4827f0ba29e9f6e93e79b4f49312ca40af6d2a</sha512> - <archiveName>PowerShell-7.1.0-win-x86.zip</archiveName> + <url>https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x86.zip</url> + <sha512>8c2ce510b5c641aad2da6adefc92d47e09bc842d47db3b5d15e14859555a74fe13ad52eaeabf1b2954ca9af737e628b567731c8a3db9bbf0e4aad05279bc1fd8</sha512> + <archiveName>PowerShell-7.1.3-win-x86.zip</archiveName> </tool> </tools> diff --git a/versions/baseline.json b/versions/baseline.json index 09f969c6a..52e92cec1 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -1206,7 +1206,7 @@ }, "chromaprint": { "baseline": "1.5.0", - "port-version": 0 + "port-version": 1 }, "chromium-base": { "baseline": "86.0.4199.1", @@ -1526,7 +1526,7 @@ }, "cudnn": { "baseline": "7.6.5", - "port-version": 1 + "port-version": 2 }, "cunit": { "baseline": "2.1.3-6", @@ -1574,7 +1574,7 @@ }, "dartsim": { "baseline": "6.9.4", - "port-version": 0 + "port-version": 1 }, "dataframe": { "baseline": "1.12.0", @@ -3009,7 +3009,7 @@ "port-version": 0 }, "libdatachannel": { - "baseline": "0.12.0", + "baseline": "0.12.2", "port-version": 0 }, "libdatrie": { @@ -3146,7 +3146,7 @@ }, "libhv": { "baseline": "1.0.0", - "port-version": 1 + "port-version": 2 }, "libhydrogen": { "baseline": "2019-08-11", @@ -3558,7 +3558,7 @@ }, "libudis86": { "baseline": "2018-01-28-56ff6c87", - "port-version": 0 + "port-version": 1 }, "libudns": { "baseline": "0.4-2", @@ -3606,7 +3606,7 @@ }, "libvpx": { "baseline": "1.9.0", - "port-version": 8 + "port-version": 9 }, "libwandio": { "baseline": "4.2.1", @@ -5922,7 +5922,7 @@ }, "tensorflow-common": { "baseline": "2.4.1", - "port-version": 0 + "port-version": 1 }, "termcolor": { "baseline": "2.0.0", @@ -6198,7 +6198,7 @@ }, "upb": { "baseline": "2020-12-19", - "port-version": 0 + "port-version": 1 }, "urdfdom": { "baseline": "1.0.4", diff --git a/versions/c-/chromaprint.json b/versions/c-/chromaprint.json index 5fc772975..a97cedf9c 100644 --- a/versions/c-/chromaprint.json +++ b/versions/c-/chromaprint.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "d841d451ed68f08043a27b2ec0beb24d3a276c81", + "version": "1.5.0", + "port-version": 1 + }, + { "git-tree": "1f3fe321540c3d87ad63fe85efa5b48bd9c55ac8", "version-string": "1.5.0", "port-version": 0 diff --git a/versions/c-/cudnn.json b/versions/c-/cudnn.json index 4930853d4..cc7e05d0c 100644 --- a/versions/c-/cudnn.json +++ b/versions/c-/cudnn.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "5581791a9ad35390ed30629985b3863cf8b85bba", + "version-string": "7.6.5", + "port-version": 2 + }, + { "git-tree": "495cc586b4bfdb28c3c339d3f471b9fbfc8bc481", "version-string": "7.6.5", "port-version": 1 diff --git a/versions/d-/dartsim.json b/versions/d-/dartsim.json index 4fd122f2f..f5b4f03e2 100644 --- a/versions/d-/dartsim.json +++ b/versions/d-/dartsim.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "104efce0932ef1a1e1e01358a442cfff637b285e", + "version": "6.9.4", + "port-version": 1 + }, + { "git-tree": "e9a9e2df385efc1d41aeea022f88f7e6a07f81ba", "version-string": "6.9.4", "port-version": 0 diff --git a/versions/l-/libdatachannel.json b/versions/l-/libdatachannel.json index b85861e67..2b289cd5d 100644 --- a/versions/l-/libdatachannel.json +++ b/versions/l-/libdatachannel.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "dd3107dd99419236f97e299ce4b61c2379a08f55", + "version-semver": "0.12.2", + "port-version": 0 + }, + { "git-tree": "1c0a9c87c0695f1ecc03f342ba87d21ae152f7dd", "version-semver": "0.12.0", "port-version": 0 diff --git a/versions/l-/libhv.json b/versions/l-/libhv.json index 8d373a261..9fae80665 100644 --- a/versions/l-/libhv.json +++ b/versions/l-/libhv.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "3a460f29d525c6c1ce8154593238960415a24d80", + "version": "1.0.0", + "port-version": 2 + }, + { "git-tree": "82104ab053e797fdcaa93d64b11fe05d37e5e14c", "version": "1.0.0", "port-version": 1 diff --git a/versions/l-/libudis86.json b/versions/l-/libudis86.json index 62fefa67d..0e098416d 100644 --- a/versions/l-/libudis86.json +++ b/versions/l-/libudis86.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "e8ab6af2cfbbb52901433f92f0c028bc766ce3ae", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1 + }, + { "git-tree": "55fbd94b278b0b3d753a2db345afa5f20b95c350", "version-string": "2018-01-28-56ff6c87", "port-version": 0 diff --git a/versions/l-/libvpx.json b/versions/l-/libvpx.json index 0d90bed5f..65efd9346 100644 --- a/versions/l-/libvpx.json +++ b/versions/l-/libvpx.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "dd62b59825999b4a4136fb9e92b393b5600596d1", + "version-semver": "1.9.0", + "port-version": 9 + }, + { "git-tree": "b3c25e1e8a48e31e14fb824fd7b8b618266093b9", "version-semver": "1.9.0", "port-version": 8 diff --git a/versions/t-/tensorflow-common.json b/versions/t-/tensorflow-common.json index a42b859be..fc8ec6df0 100644 --- a/versions/t-/tensorflow-common.json +++ b/versions/t-/tensorflow-common.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "c2a11af14488e643cf0a07f676d08319a0f70f08", + "version-semver": "2.4.1", + "port-version": 1 + }, + { "git-tree": "566f036d2265f6c960014ceb526ed0d143f65c60", "version-semver": "2.4.1", "port-version": 0 diff --git a/versions/u-/upb.json b/versions/u-/upb.json index bbe7a8b07..6d0406e44 100644 --- a/versions/u-/upb.json +++ b/versions/u-/upb.json @@ -1,6 +1,11 @@ { "versions": [ { + "git-tree": "df35bcef11c276d5cc7280edeca76eeff3dad223", + "version-date": "2020-12-19", + "port-version": 1 + }, + { "git-tree": "8c7a75bb8754caa918eddfd0e6f9457834e81f64", "version-date": "2020-12-19", "port-version": 0 |
