[nccl] Remove NCCL auto-download due to licensing issues (#17431)

* [nccl] Only download if compatible with found CUDA version

* add-version

* Remove auto-download

* Remove windows branch

* Ran x-add-version --overwrite-version

* Add hardcoded find hints in /usr

* Fix version.

Co-authored-by: Billy Robert ONeal III <bion@microsoft.com>

5 files changed, 70 insertions, 53 deletions

diff --git a/ports/nccl/FindNCCL.cmake b/ports/nccl/FindNCCL.cmake
index 72ebd9ad7..771cd65cd 100644
--- a/ports/nccl/FindNCCL.cmake
+++ b/ports/nccl/FindNCCL.cmake
@@ -14,7 +14,7 @@
 
 set(NCCL_INCLUDE_DIR $ENV{NCCL_INCLUDE_DIR} CACHE PATH "Folder contains NVIDIA NCCL headers")
 set(NCCL_LIB_DIR $ENV{NCCL_LIB_DIR} CACHE PATH "Folder contains NVIDIA NCCL libraries")
-set(NCCL_VERSION $ENV{NCCL_VERSION} CACHE STRING "Version of NCCL to build with")
+set(_NCCL_VERSION $ENV{NCCL_VERSION} CACHE STRING "Version of NCCL to build with")
 
 list(APPEND NCCL_ROOT $ENV{NCCL_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR})
 # Compatible layer for CMake <3.12. NCCL_ROOT will be accounted in for searching paths and libraries for CMake >=3.12.
@@ -22,29 +22,76 @@ list(APPEND CMAKE_PREFIX_PATH ${NCCL_ROOT})
 
 find_path(NCCL_INCLUDE_DIRS
   NAMES nccl.h
-  HINTS ${NCCL_INCLUDE_DIR})
+  HINTS
+  ${NCCL_INCLUDE_DIR}
+  $ENV{CUDNN_ROOT_DIR}
+  $ENV{CUDA_PATH}
+  $ENV{CUDNN_ROOT_DIR}
+  $ENV{CUDA_TOOLKIT_ROOT_DIR}
+  $ENV{NCCL}
+  /usr/include
+  PATH_SUFFIXES
+  include
+)
 
 if (USE_STATIC_NCCL)
   MESSAGE(STATUS "USE_STATIC_NCCL is set. Linking with static NCCL library.")
   SET(NCCL_LIBNAME "nccl_static")
-  if (NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  if (_NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a.${_NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
   endif()
 else()
   SET(NCCL_LIBNAME "nccl")
-  if (NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  if (_NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${_NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  endif()
+endif()
+
+# Read version from header
+if(EXISTS "${NCCL_INCLUDE_DIRS}/nccl.h")
+  file(READ ${NCCL_INCLUDE_DIRS}/nccl.h NCCL_HEADER_CONTENTS)
+endif()
+if(NCCL_HEADER_CONTENTS)
+  string(REGEX MATCH "define NCCL_MAJOR * +([0-9]+)"
+               _NCCL_VERSION_MAJOR "${NCCL_HEADER_CONTENTS}")
+  string(REGEX REPLACE "define NCCL_MAJOR * +([0-9]+)" "\\1"
+               _NCCL_VERSION_MAJOR "${_NCCL_VERSION_MAJOR}")
+  string(REGEX MATCH "define NCCL_MINOR * +([0-9]+)"
+               _NCCL_VERSION_MINOR "${NCCL_HEADER_CONTENTS}")
+  string(REGEX REPLACE "define NCCL_MINOR * +([0-9]+)" "\\1"
+    _NCCL_VERSION_MINOR "${_NCCL_VERSION_MINOR}")
+  string(REGEX MATCH "define NCCL_PATCH * +([0-9]+)"
+    _NCCL_VERSION_PATCH "${NCCL_HEADER_CONTENTS}")
+  string(REGEX REPLACE "define NCCL_PATCH * +([0-9]+)" "\\1"
+    _NCCL_VERSION_PATCH "${_NCCL_VERSION_PATCH}")
+  if(NOT _NCCL_VERSION_MAJOR)
+    set(_NCCL_VERSION "?")
+  else()
+    set(_NCCL_VERSION "${_NCCL_VERSION_MAJOR}.${_NCCL_VERSION_MINOR}.${_NCCL_VERSION_PATCH}")
   endif()
 endif()
 
 find_library(NCCL_LIBRARIES
   NAMES ${NCCL_LIBNAME}
-  HINTS ${NCCL_LIB_DIR})
+  HINTS
+  ${NCCL_LIB_DIR}
+  ${CUDA_TOOLKIT_ROOT}
+  $ENV{CUDA_PATH}
+  $ENV{CUDNN_ROOT_DIR}
+  $ENV{CUDA_TOOLKIT_ROOT_DIR}
+  $ENV{NCCL}
+  /usr/lib/x86_64-linux-gnu/
+  PATH_SUFFIXES
+  lib
+  lib64
+)
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+find_package_handle_standard_args(NCCL
+  REQUIRED_VARS NCCL_INCLUDE_DIRS NCCL_LIBRARIES
+  VERSION_VAR   _NCCL_VERSION)
 
 if(NCCL_FOUND)  # obtaining NCCL version and some sanity checks
-  message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})")
-  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+  message(STATUS "Found NCCL ${_NCCL_VERSION} (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})")
+  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES NCCL_VERSION)
 endif()
diff --git a/ports/nccl/portfile.cmake b/ports/nccl/portfile.cmake
index ecdacc235..e6c184daa 100644
--- a/ports/nccl/portfile.cmake
+++ b/ports/nccl/portfile.cmake
@@ -1,55 +1,19 @@
 vcpkg_fail_port_install(ON_TARGET "Windows" "OSX" ON_ARCH "x86" "arm")
 
-# note: this port must be kept in sync with CUDA port: every time one is upgraded, the other must be too
-set(NCCL_VERSION "2.4.6.1")
-set(NCCL_FULL_VERSION "${NCCL_VERSION}-cuda10.1_0")
-string(REPLACE "." ";" VERSION_LIST ${NCCL_VERSION})
-list(GET VERSION_LIST 0 NCCL_VERSION_MAJOR)
-list(GET VERSION_LIST 1 NCCL_VERSION_MINOR)
-list(GET VERSION_LIST 2 NCCL_VERSION_PATCH)
-
-set(NCCL_DOWNLOAD_LINK "https://anaconda.org/nvidia/nccl/${NCCL_VERSION}/download/linux-64/nccl-${NCCL_FULL_VERSION}.tar.bz2")
-
-# Try to find NCCL if it exists; only download if it doesn't exist
+# Find NCCL. We can use FindNCCL directly since it doesn't call any functions
+# that are disallowed in CMake script mode
+set(MINIMUM_NCCL_VERSION "2.4.6.1")
 set(NCCL_PREV_MODULE_PATH ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
-find_package(NCCL ${NCCL_VERSION})
+find_package(NCCL ${MINIMUM_NCCL_VERSION})
 set(CMAKE_MODULE_PATH ${NCCL_PREV_MODULE_PATH})
 
 # Download or return
 if(NCCL_FOUND)
-  message(STATUS "Using NCCL located on system.")
+  message(STATUS "Using NCCL ${_NCCL_VERSION} located on system.")
   set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
 else()
-  message(STATUS "NCCL not found on system. Downloading...")
-  vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT)
-
-  set(SHA512_NCCL "0fe69ad559f70aab97c78906296e2b909b4a9c042a228a2770252b3d03016c7c39acce3c0e0bd0ba651abd63471743dcffdfec307c486989c6e5745634aabde1")
-  set(NCCL_OS "linux")
-
-  vcpkg_download_distfile(ARCHIVE
-      URLS ${NCCL_DOWNLOAD_LINK}
-      FILENAME "nccl-${NCCL_FULL_VERSION}-${NCCL_OS}.tar.bz2"
-      SHA512 ${SHA512_NCCL}
-  )
-
-  vcpkg_extract_source_archive_ex(
-      OUT_SOURCE_PATH SOURCE_PATH
-      ARCHIVE ${ARCHIVE}
-      NO_REMOVE_ONE_LEVEL
-  )
-
-  file(INSTALL "${SOURCE_PATH}/include/nccl.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
-  file(INSTALL "${SOURCE_PATH}/include/nccl_net.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
-
-  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
-  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
-  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}.${NCCL_VERSION_MINOR}.${NCCL_VERSION_PATCH}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
-
-  file(INSTALL "${SOURCE_PATH}/info/licenses/LICENSE.txt" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
-  file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindNCCL.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
-  file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
-
+  message(FATAL_ERROR "Please install NCCL using your system package manager (the same way you installed CUDA). For example: apt install libnccl2 libnccl-dev.")
 endif()
 
 file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
diff --git a/ports/nccl/vcpkg.json b/ports/nccl/vcpkg.json
index 7f4416480..479c4df48 100644
--- a/ports/nccl/vcpkg.json
+++ b/ports/nccl/vcpkg.json
@@ -1,6 +1,7 @@
 {
   "name": "nccl",
   "version-string": "2.4.6",
+  "port-version": 1,
   "description": "Optimized primitives for collective multi-GPU communication.",
   "supports": "linux & x64",
   "dependencies": [
diff --git a/versions/baseline.json b/versions/baseline.json
index 1783b2f6b..0b1d076ae 100644
--- a/versions/baseline.json
+++ b/versions/baseline.json
@@ -4210,7 +4210,7 @@
     },
     "nccl": {
       "baseline": "2.4.6",
-      "port-version": 0
+      "port-version": 1
     },
     "ncurses": {
       "baseline": "6.2",
diff --git a/versions/n-/nccl.json b/versions/n-/nccl.json
index f9451a050..67df82234 100644
--- a/versions/n-/nccl.json
+++ b/versions/n-/nccl.json
@@ -1,6 +1,11 @@
 {
   "versions": [
     {
+      "git-tree": "c9b172ef032bf322539dab71b19f5dc27b679fd0",
+      "version-string": "2.4.6",
+      "port-version": 1
+    },
+    {
       "git-tree": "cb9af30276eda49db198401cdefa5a15fcc9e8ce",
       "version-string": "2.4.6",
       "port-version": 0