Merge branch 'master' of https://github.com/microsoft/vcpkg into dev/philc/4914

5 files changed, 284 insertions, 96 deletions

diff --git a/scripts/cmake/vcpkg_common_functions.cmake b/scripts/cmake/vcpkg_common_functions.cmake
index 595ebb00e..d66fc5eff 100644
--- a/scripts/cmake/vcpkg_common_functions.cmake
+++ b/scripts/cmake/vcpkg_common_functions.cmake
@@ -4,6 +4,7 @@ include(vcpkg_check_linkage)
 include(vcpkg_clean_msbuild)
 include(vcpkg_download_distfile)
 include(vcpkg_extract_source_archive)
+include(vcpkg_extract_source_archive_ex)
 include(vcpkg_execute_required_process)
 include(vcpkg_execute_required_process_repeat)
 include(vcpkg_find_acquire_program)
@@ -27,4 +28,5 @@ include(vcpkg_get_program_files_32_bit)
 include(vcpkg_get_program_files_platform_bitness)
 include(vcpkg_get_windows_sdk)
 include(vcpkg_replace_string)
+include(vcpkg_from_git)
 include(vcpkg_test_cmake)
diff --git a/scripts/cmake/vcpkg_extract_source_archive.cmake b/scripts/cmake/vcpkg_extract_source_archive.cmake
index da0ac611a..a55419b19 100644
--- a/scripts/cmake/vcpkg_extract_source_archive.cmake
+++ b/scripts/cmake/vcpkg_extract_source_archive.cmake
@@ -1,6 +1,6 @@
 ## # vcpkg_extract_source_archive
 ##
-## Extract an archive into the source directory.
+## Extract an archive into the source directory. Deprecated in favor of [`vcpkg_extract_source_archive_ex`](vcpkg_extract_source_archive_ex.md).
 ##
 ## ## Usage
 ## ```cmake
@@ -48,82 +48,3 @@ function(vcpkg_extract_source_archive ARCHIVE)
         file(WRITE ${WORKING_DIRECTORY}/${ARCHIVE_FILENAME}.extracted)
     endif()
 endfunction()
-
-function(vcpkg_extract_source_archive_ex)
-    cmake_parse_arguments(_vesae "NO_REMOVE_ONE_LEVEL" "OUT_SOURCE_PATH;ARCHIVE;REF;WORKING_DIRECTORY" "PATCHES" ${ARGN})
-
-    if(NOT _vesae_ARCHIVE)
-        message(FATAL_ERROR "Must specify ARCHIVE parameter to vcpkg_extract_source_archive_ex()")
-    endif()
-
-    if(NOT DEFINED _vesae_OUT_SOURCE_PATH)
-        message(FATAL_ERROR "Must specify OUT_SOURCE_PATH parameter to vcpkg_extract_source_archive_ex()")
-    endif()
-
-    if(NOT DEFINED _vesae_WORKING_DIRECTORY)
-        set(_vesae_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/src)
-    endif()
-
-    if(NOT DEFINED _vesae_REF)
-        get_filename_component(_vesae_REF ${_vesae_ARCHIVE} NAME_WE)
-    endif()
-
-    string(REPLACE "/" "-" SANITIZED_REF "${_vesae_REF}")
-
-    # Take the last 10 chars of the REF
-    set(REF_MAX_LENGTH 10)
-    string(LENGTH ${SANITIZED_REF} REF_LENGTH)
-    math(EXPR FROM_REF ${REF_LENGTH}-${REF_MAX_LENGTH})
-    if(FROM_REF LESS 0)
-        set(FROM_REF 0)
-    endif()
-    string(SUBSTRING ${SANITIZED_REF} ${FROM_REF} ${REF_LENGTH} SHORTENED_SANITIZED_REF)
-
-    # Hash the archive hash along with the patches. Take the first 10 chars of the hash
-    file(SHA512 ${_vesae_ARCHIVE} PATCHSET_HASH)
-    foreach(PATCH IN LISTS _vesae_PATCHES)
-        get_filename_component(ABSOLUTE_PATCH "${PATCH}" ABSOLUTE BASE_DIR "${CURRENT_PORT_DIR}")
-        file(SHA512 ${ABSOLUTE_PATCH} CURRENT_HASH)
-        string(APPEND PATCHSET_HASH ${CURRENT_HASH})
-    endforeach()
-
-    string(SHA512 PATCHSET_HASH ${PATCHSET_HASH})
-    string(SUBSTRING ${PATCHSET_HASH} 0 10 PATCHSET_HASH)
-    set(SOURCE_PATH "${_vesae_WORKING_DIRECTORY}/${SHORTENED_SANITIZED_REF}-${PATCHSET_HASH}")
-
-    if(NOT EXISTS ${SOURCE_PATH})
-        set(TEMP_DIR "${_vesae_WORKING_DIRECTORY}/TEMP")
-        file(REMOVE_RECURSE ${TEMP_DIR})
-        vcpkg_extract_source_archive("${_vesae_ARCHIVE}" "${TEMP_DIR}")
-
-        if(_vesae_NO_REMOVE_ONE_LEVEL)
-            set(TEMP_SOURCE_PATH ${TEMP_DIR})
-        else()
-            file(GLOB _ARCHIVE_FILES "${TEMP_DIR}/*")
-            list(LENGTH _ARCHIVE_FILES _NUM_ARCHIVE_FILES)
-            set(TEMP_SOURCE_PATH)
-            foreach(dir IN LISTS _ARCHIVE_FILES)
-                if (IS_DIRECTORY ${dir})
-                    set(TEMP_SOURCE_PATH "${dir}")
-                    break()
-                endif()
-            endforeach()
-
-            if(NOT _NUM_ARCHIVE_FILES EQUAL 2 OR NOT TEMP_SOURCE_PATH)
-                message(FATAL_ERROR "Could not unwrap top level directory from archive. Pass NO_REMOVE_ONE_LEVEL to disable this.")
-            endif()
-        endif()
-
-        vcpkg_apply_patches(
-            SOURCE_PATH ${TEMP_SOURCE_PATH}
-            PATCHES ${_vesae_PATCHES}
-        )
-
-        file(RENAME ${TEMP_SOURCE_PATH} ${SOURCE_PATH})
-        file(REMOVE_RECURSE ${TEMP_DIR})
-    endif()
-
-    set(${_vesae_OUT_SOURCE_PATH} "${SOURCE_PATH}" PARENT_SCOPE)
-    message(STATUS "Using source at ${SOURCE_PATH}")
-    return()
-endfunction()
diff --git a/scripts/cmake/vcpkg_extract_source_archive_ex.cmake b/scripts/cmake/vcpkg_extract_source_archive_ex.cmake
new file mode 100644
index 000000000..a70a5e4a3
--- /dev/null
+++ b/scripts/cmake/vcpkg_extract_source_archive_ex.cmake
@@ -0,0 +1,130 @@
+## # vcpkg_extract_source_archive_ex

+##

+## Extract an archive into the source directory. Replaces [`vcpkg_extract_source_archive`](vcpkg_extract_source_archive.md).

+##

+## ## Usage

+## ```cmake

+## vcpkg_extract_source_archive_ex(

+##     OUT_SOURCE_PATH <SOURCE_PATH>

+##     ARCHIVE <${ARCHIVE}>

+##     [REF <1.0.0>]

+##     [NO_REMOVE_ONE_LEVEL]

+##     [WORKING_DIRECTORY <${CURRENT_BUILDTREES_DIR}/src>]

+##     [PATCHES <a.patch>...]

+## )

+## ```

+## ## Parameters

+## ### OUT_SOURCE_PATH

+## Specifies the out-variable that will contain the extracted location.

+##

+## This should be set to `SOURCE_PATH` by convention.

+##

+## ### ARCHIVE

+## The full path to the archive to be extracted.

+##

+## This is usually obtained from calling [`vcpkg_download_distfile`](vcpkg_download_distfile.md).

+##

+## ### REF

+## A friendly name that will be used instead of the filename of the archive.

+##

+## By convention, this is set to the version number or tag fetched

+##

+## ### WORKING_DIRECTORY

+## If specified, the archive will be extracted into the working directory instead of `${CURRENT_BUILDTREES_DIR}/src/`.

+##

+## Note that the archive will still be extracted into a subfolder underneath that directory (`${WORKING_DIRECTORY}/${REF}-${HASH}/`).

+##

+## ### PATCHES

+## A list of patches to be applied to the extracted sources.

+##

+## Relative paths are based on the port directory.

+##

+## ### NO_REMOVE_ONE_LEVEL

+## Specifies that the default removal of the top level folder should not occur.

+##

+## ## Examples

+##

+## * [bzip2](https://github.com/Microsoft/vcpkg/blob/master/ports/bzip2/portfile.cmake)

+## * [sqlite3](https://github.com/Microsoft/vcpkg/blob/master/ports/sqlite3/portfile.cmake)

+## * [cairo](https://github.com/Microsoft/vcpkg/blob/master/ports/cairo/portfile.cmake)

+include(vcpkg_apply_patches)

+include(vcpkg_extract_source_archive)

+

+function(vcpkg_extract_source_archive_ex)

+    cmake_parse_arguments(_vesae "NO_REMOVE_ONE_LEVEL" "OUT_SOURCE_PATH;ARCHIVE;REF;WORKING_DIRECTORY" "PATCHES" ${ARGN})

+

+    if(NOT _vesae_ARCHIVE)

+        message(FATAL_ERROR "Must specify ARCHIVE parameter to vcpkg_extract_source_archive_ex()")

+    endif()

+

+    if(NOT DEFINED _vesae_OUT_SOURCE_PATH)

+        message(FATAL_ERROR "Must specify OUT_SOURCE_PATH parameter to vcpkg_extract_source_archive_ex()")

+    endif()

+

+    if(NOT DEFINED _vesae_WORKING_DIRECTORY)

+        set(_vesae_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/src)

+    endif()

+

+    if(NOT DEFINED _vesae_REF)

+        get_filename_component(_vesae_REF ${_vesae_ARCHIVE} NAME_WE)

+    endif()

+

+    string(REPLACE "/" "-" SANITIZED_REF "${_vesae_REF}")

+

+    # Take the last 10 chars of the REF

+    set(REF_MAX_LENGTH 10)

+    string(LENGTH ${SANITIZED_REF} REF_LENGTH)

+    math(EXPR FROM_REF ${REF_LENGTH}-${REF_MAX_LENGTH})

+    if(FROM_REF LESS 0)

+        set(FROM_REF 0)

+    endif()

+    string(SUBSTRING ${SANITIZED_REF} ${FROM_REF} ${REF_LENGTH} SHORTENED_SANITIZED_REF)

+

+    # Hash the archive hash along with the patches. Take the first 10 chars of the hash

+    file(SHA512 ${_vesae_ARCHIVE} PATCHSET_HASH)

+    foreach(PATCH IN LISTS _vesae_PATCHES)

+        get_filename_component(ABSOLUTE_PATCH "${PATCH}" ABSOLUTE BASE_DIR "${CURRENT_PORT_DIR}")

+        file(SHA512 ${ABSOLUTE_PATCH} CURRENT_HASH)

+        string(APPEND PATCHSET_HASH ${CURRENT_HASH})

+    endforeach()

+

+    string(SHA512 PATCHSET_HASH ${PATCHSET_HASH})

+    string(SUBSTRING ${PATCHSET_HASH} 0 10 PATCHSET_HASH)

+    set(SOURCE_PATH "${_vesae_WORKING_DIRECTORY}/${SHORTENED_SANITIZED_REF}-${PATCHSET_HASH}")

+

+    if(NOT EXISTS ${SOURCE_PATH})

+        set(TEMP_DIR "${_vesae_WORKING_DIRECTORY}/TEMP")

+        file(REMOVE_RECURSE ${TEMP_DIR})

+        vcpkg_extract_source_archive("${_vesae_ARCHIVE}" "${TEMP_DIR}")

+

+        if(_vesae_NO_REMOVE_ONE_LEVEL)

+            set(TEMP_SOURCE_PATH ${TEMP_DIR})

+        else()

+            file(GLOB _ARCHIVE_FILES "${TEMP_DIR}/*")

+            list(LENGTH _ARCHIVE_FILES _NUM_ARCHIVE_FILES)

+            set(TEMP_SOURCE_PATH)

+            foreach(dir IN LISTS _ARCHIVE_FILES)

+                if (IS_DIRECTORY ${dir})

+                    set(TEMP_SOURCE_PATH "${dir}")

+                    break()

+                endif()

+            endforeach()

+

+            if(NOT _NUM_ARCHIVE_FILES EQUAL 2 OR NOT TEMP_SOURCE_PATH)

+                message(FATAL_ERROR "Could not unwrap top level directory from archive. Pass NO_REMOVE_ONE_LEVEL to disable this.")

+            endif()

+        endif()

+

+        vcpkg_apply_patches(

+            SOURCE_PATH ${TEMP_SOURCE_PATH}

+            PATCHES ${_vesae_PATCHES}

+        )

+

+        file(RENAME ${TEMP_SOURCE_PATH} ${SOURCE_PATH})

+        file(REMOVE_RECURSE ${TEMP_DIR})

+    endif()

+

+    set(${_vesae_OUT_SOURCE_PATH} "${SOURCE_PATH}" PARENT_SCOPE)

+    message(STATUS "Using source at ${SOURCE_PATH}")

+    return()

+endfunction()

diff --git a/scripts/cmake/vcpkg_find_acquire_program.cmake b/scripts/cmake/vcpkg_find_acquire_program.cmake
index 4cb123fd6..ebe46b335 100644
--- a/scripts/cmake/vcpkg_find_acquire_program.cmake
+++ b/scripts/cmake/vcpkg_find_acquire_program.cmake
@@ -77,12 +77,18 @@ function(vcpkg_find_acquire_program VAR)
     set(NOEXTRACT ON)
     set(HASH c1945669d983b632a10c5ff31e86d6ecbff143c3d8b2c433c0d3d18f84356d2b351f71ac05fd44e5403651b00c31db0d14615d7f9a6ecce5750438d37105c55b)
   elseif(VAR MATCHES "PYTHON3")
-    set(PROGNAME python)
-    set(SUBDIR "python3")
-    set(PATHS ${DOWNLOADS}/tools/python/${SUBDIR})
-    set(URL "https://www.python.org/ftp/python/3.5.4/python-3.5.4-embed-win32.zip")
-    set(ARCHIVE "python-3.5.4-embed-win32.zip")
-    set(HASH b5240fdc95088c2d7f65d2dd598650f8dd106b49589d94156bd4a078b108c6cabbe7a38ef73e2b2cf00e8312a93d2e587eac2c54ce85540d3c7a26cc60013156)
+    if(CMAKE_HOST_WIN32)
+      set(PROGNAME python)
+      set(SUBDIR "python3")
+      set(PATHS ${DOWNLOADS}/tools/python/${SUBDIR})
+      set(URL "https://www.python.org/ftp/python/3.5.4/python-3.5.4-embed-win32.zip")
+      set(ARCHIVE "python-3.5.4-embed-win32.zip")
+      set(HASH b5240fdc95088c2d7f65d2dd598650f8dd106b49589d94156bd4a078b108c6cabbe7a38ef73e2b2cf00e8312a93d2e587eac2c54ce85540d3c7a26cc60013156)
+    else()
+      set(PROGNAME python3)
+      set(BREW_PACKAGE_NAME "python")
+      set(APT_PACKAGE_NAME "python3")
+    endif()
   elseif(VAR MATCHES "PYTHON2")
     set(PROGNAME python)
     set(SUBDIR "python2")
@@ -140,12 +146,13 @@ function(vcpkg_find_acquire_program VAR)
     set(ARCHIVE "meson-0.47.1.zip")
     set(HASH 0f6462835583a51707bee82d852018cfcb7444c0dad95b2ba08814e500a5cfe3f731dc6c1fb73c765d1120ee2a2d6600e15d8d393bab1993e84bd4354b2e6855)
   elseif(VAR MATCHES "FLEX")
-  if(CMAKE_HOST_WIN32)
-    set(PROGNAME win_flex)
-      set(PATHS ${DOWNLOADS}/tools/win_flex)
-      set(URL "https://sourceforge.net/projects/winflexbison/files/win_flex_bison-2.5.9.zip/download")
-      set(ARCHIVE "win_flex_bison-2.5.9.zip")
-      set(HASH 9580f0e46893670a011645947c1becda69909a41a38bb4197fe33bd1ab7719da6b80e1be316f269e1a4759286870d49a9b07ef83afc4bac33232bd348e0bc814)
+    if(CMAKE_HOST_WIN32)
+      set(PROGNAME win_flex)
+      set(SUBDIR win_flex-2.5.16)
+      set(PATHS ${DOWNLOADS}/tools/win_flex/${SUBDIR})
+      set(URL "https://sourceforge.net/projects/winflexbison/files/winflexbison-2.5.16.zip/download")
+      set(ARCHIVE "win_flex_bison-2.5.16.zip")
+      set(HASH 0a14154bff5d998feb23903c46961528f8ccb4464375d5384db8c4a7d230c0c599da9b68e7a32f3217a0a0735742242eaf3769cb4f03e00931af8640250e9123)
     else()
       set(PROGNAME flex)
       set(APT_PACKAGE_NAME flex)
@@ -154,10 +161,11 @@ function(vcpkg_find_acquire_program VAR)
   elseif(VAR MATCHES "BISON")
     if(CMAKE_HOST_WIN32)
       set(PROGNAME win_bison)
-      set(PATHS ${DOWNLOADS}/tools/win_bison)
-      set(URL "https://sourceforge.net/projects/winflexbison/files/win_flex_bison-2.5.9.zip/download")
-      set(ARCHIVE "win_flex_bison-2.5.9.zip")
-      set(HASH 9580f0e46893670a011645947c1becda69909a41a38bb4197fe33bd1ab7719da6b80e1be316f269e1a4759286870d49a9b07ef83afc4bac33232bd348e0bc814)
+      set(SUBDIR win_bison-2.5.16)
+      set(PATHS ${DOWNLOADS}/tools/win_bison/${SUBDIR})
+      set(URL "https://sourceforge.net/projects/winflexbison/files/winflexbison-2.5.16.zip/download")
+      set(ARCHIVE "win_flex_bison-2.5.16.zip")
+      set(HASH 0a14154bff5d998feb23903c46961528f8ccb4464375d5384db8c4a7d230c0c599da9b68e7a32f3217a0a0735742242eaf3769cb4f03e00931af8640250e9123)
     else()
       set(PROGNAME bison)
       set(APT_PACKAGE_NAME bison)
diff --git a/scripts/cmake/vcpkg_from_git.cmake b/scripts/cmake/vcpkg_from_git.cmake
new file mode 100644
index 000000000..2fc66b279
--- /dev/null
+++ b/scripts/cmake/vcpkg_from_git.cmake
@@ -0,0 +1,127 @@
+## # vcpkg_from_git
+##
+## Download and extract a project from git
+##
+## ## Usage:
+## ```cmake
+## vcpkg_from_git(
+##     OUT_SOURCE_PATH <SOURCE_PATH>
+##     URL <https://android.googlesource.com/platform/external/fdlibm>
+##     REF <59f7335e4d...>
+##     SHA512 <abcdef123...>
+##     [PATCHES <patch1.patch> <patch2.patch>...]
+## )
+## ```
+##
+## ## Parameters:
+## ### OUT_SOURCE_PATH
+## Specifies the out-variable that will contain the extracted location.
+##
+## This should be set to `SOURCE_PATH` by convention.
+##
+## ### URL
+## The url of the git repository.
+##
+## ### SHA512
+## The SHA512 hash that should match the archive form of the commit.
+##
+## This is most easily determined by first setting it to `0`, then trying to build the port. The error message will contain the full hash, which can be copied back into the portfile.
+##
+## ### REF
+## A stable git commit-ish (ideally a tag or commit) that will not change contents. **This should not be a branch.**
+##
+## For repositories without official releases, this can be set to the full commit id of the current latest master.
+##
+## ### PATCHES
+## A list of patches to be applied to the extracted sources.
+##
+## Relative paths are based on the port directory.
+##
+## ## Notes:
+## `OUT_SOURCE_PATH`, `REF`, `SHA512`, and `URL` must be specified.
+##
+## ## Examples:
+##
+## * [fdlibm](https://github.com/Microsoft/vcpkg/blob/master/ports/fdlibm/portfile.cmake)
+
+function(vcpkg_from_git)
+  set(oneValueArgs OUT_SOURCE_PATH URL REF SHA512)
+  set(multipleValuesArgs PATCHES)
+  cmake_parse_arguments(_vdud "" "${oneValueArgs}" "${multipleValuesArgs}" ${ARGN})
+
+  if(NOT DEFINED _vdud_OUT_SOURCE_PATH)
+    message(FATAL_ERROR "OUT_SOURCE_PATH must be specified.")
+  endif()
+
+  if(NOT DEFINED _vdud_URL)
+    message(FATAL_ERROR "The git url must be specified")
+  endif()
+
+  if(NOT DEFINED _vdud_REF)
+    message(FATAL_ERROR "The git ref must be specified.")
+  endif()
+
+  if(NOT DEFINED _vdud_SHA512)
+    message(FATAL_ERROR "vcpkg_from_git requires a SHA512 argument. If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.")
+  endif()
+
+  # using .tar.gz instead of .zip because the hash of the latter is affected by timezone.
+  string(REPLACE "/" "-" SANITIZED_REF "${_vdud_REF}")
+  set(TEMP_ARCHIVE "${DOWNLOADS}/temp/${PORT}-${SANITIZED_REF}.tar.gz")
+  set(ARCHIVE "${DOWNLOADS}/${PORT}-${SANITIZED_REF}.tar.gz")
+  set(TEMP_SOURCE_PATH "${CURRENT_BUILDTREES_DIR}/src/${SANITIZED_REF}")
+
+  function(test_hash FILE_PATH FILE_KIND CUSTOM_ERROR_ADVICE)
+    file(SHA512 ${FILE_PATH} FILE_HASH)
+    if(NOT FILE_HASH STREQUAL _vdud_SHA512)
+        message(FATAL_ERROR
+            "\nFile does not have expected hash:\n"
+            "        File path: [ ${FILE_PATH} ]\n"
+            "    Expected hash: [ ${_vdud_SHA512} ]\n"
+            "      Actual hash: [ ${FILE_HASH} ]\n"
+            "${CUSTOM_ERROR_ADVICE}\n")
+    endif()
+  endfunction()
+
+  if(NOT EXISTS "${ARCHIVE}")
+    if(_VCPKG_NO_DOWNLOADS)
+        message(FATAL_ERROR "Downloads are disabled, but '${ARCHIVE}' does not exist.")
+    endif()
+    message(STATUS "Fetching ${_vdud_URL}...")
+    find_program(GIT NAMES git git.cmd)
+    # Note: git init is safe to run multiple times
+    vcpkg_execute_required_process(
+      COMMAND ${GIT} init git-tmp
+      WORKING_DIRECTORY ${DOWNLOADS}
+      LOGNAME git-init
+    )
+    vcpkg_execute_required_process(
+      COMMAND ${GIT} fetch ${_vdud_URL} ${_vdud_REF} --depth 1 -n
+      WORKING_DIRECTORY ${DOWNLOADS}/git-tmp
+      LOGNAME git-fetch
+    )
+    file(MAKE_DIRECTORY "${DOWNLOADS}/temp")
+    vcpkg_execute_required_process(
+      COMMAND ${GIT} archive FETCH_HEAD -o "${TEMP_ARCHIVE}"
+      WORKING_DIRECTORY ${DOWNLOADS}/git-tmp
+      LOGNAME git-archive
+    )
+    test_hash("${TEMP_ARCHIVE}" "downloaded repo" "")
+    get_filename_component(downloaded_file_dir "${ARCHIVE}" DIRECTORY)
+    file(MAKE_DIRECTORY "${downloaded_file_dir}")
+    file(RENAME "${TEMP_ARCHIVE}" "${ARCHIVE}")
+  else()
+    message(STATUS "Using cached ${ARCHIVE}")
+    test_hash("${ARCHIVE}" "cached file" "Please delete the file and retry if this file should be downloaded again.")
+  endif()
+
+  vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE "${ARCHIVE}"
+    REF "${SANITIZED_REF}"
+    PATCHES ${_vdud_PATCHES}
+    NO_REMOVE_ONE_LEVEL
+  )
+
+  set(${_vdud_OUT_SOURCE_PATH} "${SOURCE_PATH}" PARENT_SCOPE)
+endfunction()