[nccl] New Port (#14683)

* [nccl] New Port

* Update ports/nccl/portfile.cmake

* Add usage to vcpkg-cmake-wrapper

* Move usage to usage file

* Update ports/nccl/portfile.cmake

* Don't copy debug libraries because they aren't provided

Co-authored-by: Jack·Boos·Yu <47264268+JackBoosY@users.noreply.github.com>

6 files changed, 126 insertions, 1 deletions

diff --git a/ports/cuda/portfile.cmake b/ports/cuda/portfile.cmake
index 6e5a497da..ecdce4cfb 100644
--- a/ports/cuda/portfile.cmake
+++ b/ports/cuda/portfile.cmake
@@ -2,7 +2,7 @@
 # Other packages can depend on this package to declare a dependency on CUDA.
 # If this package is installed, we assume that CUDA is properly installed.
 
-#note: this port must be kept in sync with CUDNN port: every time one is upgraded, the other must be too
+#note: this port must be kept in sync with CUDNN and NCCL ports: every time one is upgraded, the other must be too
 
 include(${CMAKE_CURRENT_LIST_DIR}/vcpkg_find_cuda.cmake)
 
diff --git a/ports/nccl/FindNCCL.cmake b/ports/nccl/FindNCCL.cmake
new file mode 100644
index 000000000..72ebd9ad7
--- /dev/null
+++ b/ports/nccl/FindNCCL.cmake
@@ -0,0 +1,50 @@
+# Find the nccl libraries
+#
+# The following variables are optionally searched for defaults
+#  NCCL_ROOT: Base directory where all NCCL components are found
+#  NCCL_INCLUDE_DIR: Directory where NCCL header is found
+#  NCCL_LIB_DIR: Directory where NCCL library is found
+#
+# The following are set after configuration is done:
+#  NCCL_FOUND
+#  NCCL_INCLUDE_DIRS
+#  NCCL_LIBRARIES
+#
+# Adapted from https://github.com/pytorch/pytorch/blob/master/cmake/Modules/FindNCCL.cmake
+
+set(NCCL_INCLUDE_DIR $ENV{NCCL_INCLUDE_DIR} CACHE PATH "Folder contains NVIDIA NCCL headers")
+set(NCCL_LIB_DIR $ENV{NCCL_LIB_DIR} CACHE PATH "Folder contains NVIDIA NCCL libraries")
+set(NCCL_VERSION $ENV{NCCL_VERSION} CACHE STRING "Version of NCCL to build with")
+
+list(APPEND NCCL_ROOT $ENV{NCCL_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR})
+# Compatible layer for CMake <3.12. NCCL_ROOT will be accounted in for searching paths and libraries for CMake >=3.12.
+list(APPEND CMAKE_PREFIX_PATH ${NCCL_ROOT})
+
+find_path(NCCL_INCLUDE_DIRS
+  NAMES nccl.h
+  HINTS ${NCCL_INCLUDE_DIR})
+
+if (USE_STATIC_NCCL)
+  MESSAGE(STATUS "USE_STATIC_NCCL is set. Linking with static NCCL library.")
+  SET(NCCL_LIBNAME "nccl_static")
+  if (NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  endif()
+else()
+  SET(NCCL_LIBNAME "nccl")
+  if (NCCL_VERSION)  # Prefer the versioned library if a specific NCCL version is specified
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  endif()
+endif()
+
+find_library(NCCL_LIBRARIES
+  NAMES ${NCCL_LIBNAME}
+  HINTS ${NCCL_LIB_DIR})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+
+if(NCCL_FOUND)  # obtaining NCCL version and some sanity checks
+  message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})")
+  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+endif()
diff --git a/ports/nccl/portfile.cmake b/ports/nccl/portfile.cmake
new file mode 100644
index 000000000..ecdacc235
--- /dev/null
+++ b/ports/nccl/portfile.cmake
@@ -0,0 +1,55 @@
+vcpkg_fail_port_install(ON_TARGET "Windows" "OSX" ON_ARCH "x86" "arm")
+
+# note: this port must be kept in sync with CUDA port: every time one is upgraded, the other must be too
+set(NCCL_VERSION "2.4.6.1")
+set(NCCL_FULL_VERSION "${NCCL_VERSION}-cuda10.1_0")
+string(REPLACE "." ";" VERSION_LIST ${NCCL_VERSION})
+list(GET VERSION_LIST 0 NCCL_VERSION_MAJOR)
+list(GET VERSION_LIST 1 NCCL_VERSION_MINOR)
+list(GET VERSION_LIST 2 NCCL_VERSION_PATCH)
+
+set(NCCL_DOWNLOAD_LINK "https://anaconda.org/nvidia/nccl/${NCCL_VERSION}/download/linux-64/nccl-${NCCL_FULL_VERSION}.tar.bz2")
+
+# Try to find NCCL if it exists; only download if it doesn't exist
+set(NCCL_PREV_MODULE_PATH ${CMAKE_MODULE_PATH})
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
+find_package(NCCL ${NCCL_VERSION})
+set(CMAKE_MODULE_PATH ${NCCL_PREV_MODULE_PATH})
+
+# Download or return
+if(NCCL_FOUND)
+  message(STATUS "Using NCCL located on system.")
+  set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+else()
+  message(STATUS "NCCL not found on system. Downloading...")
+  vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT)
+
+  set(SHA512_NCCL "0fe69ad559f70aab97c78906296e2b909b4a9c042a228a2770252b3d03016c7c39acce3c0e0bd0ba651abd63471743dcffdfec307c486989c6e5745634aabde1")
+  set(NCCL_OS "linux")
+
+  vcpkg_download_distfile(ARCHIVE
+      URLS ${NCCL_DOWNLOAD_LINK}
+      FILENAME "nccl-${NCCL_FULL_VERSION}-${NCCL_OS}.tar.bz2"
+      SHA512 ${SHA512_NCCL}
+  )
+
+  vcpkg_extract_source_archive_ex(
+      OUT_SOURCE_PATH SOURCE_PATH
+      ARCHIVE ${ARCHIVE}
+      NO_REMOVE_ONE_LEVEL
+  )
+
+  file(INSTALL "${SOURCE_PATH}/include/nccl.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
+  file(INSTALL "${SOURCE_PATH}/include/nccl_net.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
+
+  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+  file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}.${NCCL_VERSION_MINOR}.${NCCL_VERSION_PATCH}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+
+  file(INSTALL "${SOURCE_PATH}/info/licenses/LICENSE.txt" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
+  file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindNCCL.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
+  file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
+
+endif()
+
+file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
diff --git a/ports/nccl/usage b/ports/nccl/usage
new file mode 100644
index 000000000..62ebd7bab
--- /dev/null
+++ b/ports/nccl/usage
@@ -0,0 +1,5 @@
+The package nccl provides CMake variables:
+
+    find_package(NCCL REQUIRED)
+    target_link_libraries(main PRIVATE ${NCCL_LIBRARIES})
+    target_include_directories(main PRIVATE ${NCCL_INCLUDE_DIRS})
diff --git a/ports/nccl/vcpkg-cmake-wrapper.cmake b/ports/nccl/vcpkg-cmake-wrapper.cmake
new file mode 100644
index 000000000..27893ec4f
--- /dev/null
+++ b/ports/nccl/vcpkg-cmake-wrapper.cmake
@@ -0,0 +1,6 @@
+set(NCCL_PREV_MODULE_PATH ${CMAKE_MODULE_PATH})
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
+
+_find_package(${ARGS})
+
+set(CMAKE_MODULE_PATH ${NCCL_PREV_MODULE_PATH})
diff --git a/ports/nccl/vcpkg.json b/ports/nccl/vcpkg.json
new file mode 100644
index 000000000..7f4416480
--- /dev/null
+++ b/ports/nccl/vcpkg.json
@@ -0,0 +1,9 @@
+{
+  "name": "nccl",
+  "version-string": "2.4.6",
+  "description": "Optimized primitives for collective multi-GPU communication.",
+  "supports": "linux & x64",
+  "dependencies": [
+    "cuda"
+  ]
+}