Initial conan.io support

2 files changed, 468 insertions, 16 deletions

diff --git a/nimterop/build.nim b/nimterop/build.nim
index e0d630a..d5ce59c 100644
--- a/nimterop/build.nim
+++ b/nimterop/build.nim
@@ -206,6 +206,14 @@ proc rmDir*(dir: string) =
   ## Remove a directory or pattern at compile time
   rmFile(dir, dir = true)
 
+proc cleanDir*(dir: string) =
+  ## Remove all contents of a directory at compile time
+  for kind, path in walkDir(dir):
+    if kind == pcDir:
+      rmDir(path)
+    else:
+      rmFile(path)
+
 proc getProjectCacheDir*(name: string, forceClean = true): string =
   ## Get a cache directory where all nimterop artifacts can be stored
   ##
@@ -281,7 +289,7 @@ proc extractTar*(tarfile, outdir: string) =
   if name.len != 0:
     rmFile(outdir / name)
 
-proc downloadUrl*(url, outdir: string) =
+proc downloadUrl*(url, outdir: string, quiet = false) =
   ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory
   ##
   ## If an archive file, it is automatically extracted after download.
@@ -291,7 +299,8 @@ proc downloadUrl*(url, outdir: string) =
     archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"]
 
   if not (ext in archives and fileExists(outdir/file)):
-    echo "# Downloading " & file
+    if not quiet:
+      echo "# Downloading " & file
     mkDir(outdir)
     var cmd = findExe("curl")
     if cmd.len != 0:
@@ -304,7 +313,7 @@ proc downloadUrl*(url, outdir: string) =
         cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#"
       else:
         doAssert false, "No download tool available - curl, wget"
-    discard execAction(cmd % [url, (outdir/file).sanitizePath], retry = 1)
+    discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1)
 
     if ext == ".zip":
       extractZip(file, outdir)
@@ -732,6 +741,36 @@ proc getGccLibPaths*(mode: string): seq[string] =
   when defined(osx):
     result.add "/usr/lib"
 
+proc getGccInfo*(): tuple[arch, os, compiler, version: string] =
+  let
+    (outp, _) = execAction(&"{getCompiler()} -v")
+  for line in outp.splitLines():
+    if line.startsWith("Target: "):
+      result.arch = line.split(' ')[1].split('-')[0]
+      result.os =
+        if "linux" in line:
+          "linux"
+        elif "android" in line:
+          "android"
+        elif "darwin" in line:
+          "macos"
+        elif "w64" in line or "mingw" in line:
+          "windows"
+        else:
+          "unknown"
+    elif " version " in line:
+      result.version = line.split(" version ")[1].split(' ')[0]
+  if "clang" in outp:
+    if result.os == "macos":
+      result.compiler = "apple-clang"
+    else:
+      result.compiler = "clang"
+  else:
+    result.compiler = "gcc"
+
+# Conan support
+include conan
+
 proc getStdPath(header, mode: string): string =
   for inc in getGccPaths(mode):
     result = findFile(header, inc, recurse = false, first = true)
@@ -784,6 +823,28 @@ proc getDlPath(header, url, outdir, version: string): string =
 
   result = findFile(header, outdir)
 
+proc getConanPath(header, uri, outdir, version: string, shared: bool): string =
+  var
+    uri = uri
+
+  if "$#" in uri or "$1" in uri:
+    doAssert version.len != 0, "Need version for Conan uri"
+    uri = uri % version
+  else:
+    doAssert version.len == 0, "Conan uri does not contain version"
+
+  let
+    pkg = newConanPackageFromUri(uri, shared)
+  downloadConan(pkg, outdir)
+
+  result = findFile(header, outdir)
+
+proc getConanLPath(outdir: string): string =
+  let
+    pkg = loadConanInfo(outdir)
+  
+  result = pkg.getConanLibs(outdir).join(" ")
+
 proc getLocalPath(header, outdir: string): string =
   if outdir.len != 0:
     result = findFile(header, outdir)
@@ -937,7 +998,9 @@ macro isDefined*(def: untyped): untyped =
       false
   )
 
-macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "",
+macro getHeader*(
+  header: static[string], giturl: static[string] = "", dlurl: static[string] = "",
+  conanuri: static[string] = "", outdir: static[string] = "",
   conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "",
   altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped =
   ## Get the path to a header file for wrapping with
@@ -950,16 +1013,18 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
   ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux
   ## `-d:xxxGit` - clone source from a git repo specified in `giturl`
   ## `-d:xxxDL` - download source from `dlurl` and extract if required
+  ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri`
+  ##   typically formatted as `name/version[@user/channel][:bhash]`
   ##
   ## This allows a single wrapper to be used in different ways depending on the user's needs.
   ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is.
   ##
-  ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git` or `DL`.
-  ## This allows using a system installed library if available before falling back to manual
-  ## building.
+  ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or
+  ## `Conan`. This allows using a system installed library if available before falling back
+  ## to manual building.
   ##
   ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag
-  ## name for Git whereas for DL, it replaces `$1` in the URL defined.
+  ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined.
   ##
   ## All defines can also be set in code using `setDefines()`.
   ##
@@ -1006,6 +1071,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
     stdStr = name & "Std"
     gitStr = name & "Git"
     dlStr = name & "DL"
+    conanStr = name & "Conan"
 
     staticStr = name & "Static"
     verStr = name & "SetVer"
@@ -1014,6 +1080,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
     nameStd = newIdentNode(stdStr)
     nameGit = newIdentNode(gitStr)
     nameDL = newIdentNode(dlStr)
+    nameConan = newIdentNode(conanStr)
 
     nameStatic = newIdentNode(staticStr)
 
@@ -1031,6 +1098,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
     stdVal = gDefines.hasKey(stdStr)
     gitVal = gDefines.hasKey(gitStr)
     dlVal = gDefines.hasKey(dlStr)
+    conanVal = gDefines.hasKey(conanStr)
     staticVal = gDefines.hasKey(staticStr)
     verVal =
       if gDefines.hasKey(verStr):
@@ -1052,14 +1120,17 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
       `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1
       `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1
       `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1
+      `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1
       `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1
 
     # Search for header in outdir (after retrieving code) depending on -d:xxx mode
-    proc getPath(header, giturl, dlurl, outdir, version: string): string =
+    proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string =
       when `nameGit`:
         getGitPath(header, giturl, outdir, version)
       elif `nameDL`:
         getDlPath(header, dlurl, outdir, version)
+      elif `nameConan`:
+        getConanPath(header, conanuri, outdir, version, shared)
       else:
         getLocalPath(header, outdir)
 
@@ -1077,23 +1148,30 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
       stdLPath =
         when `nameStd`: getStdLibPath(`lname`, `mode`) else: ""
 
+      useStd = stdPath.len != 0 and stdLPath.len != 0
+
       # Look elsewhere if requested while prioritizing standard paths
       prePath =
-        when stdPath.len != 0 and stdLPath.len != 0:
+        when useStd:
           stdPath
         else:
-          getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`)
+          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
 
-    # Run preBuild hook before building library if not standard
-    when (prePath != stdPath or prePath.len == 0) and declared(`preBuild`):
+    # Run preBuild hook before building library if not Std or Conan
+    when not (useStd or `nameConan`) and declared(`preBuild`):
       static:
         `preBuild`(`outdir`, prePath)
 
     const
       # Library binary path - build if not standard
       `lpath`* =
-        when stdPath.len != 0 and stdLPath.len != 0:
+        when useStd:
           stdLPath
+        elif `nameConan`:
+          when `nameStatic`:
+            getConanLPath(`outdir`)
+          else:
+            findFile(`lname`, `outdir`, regex = true)
         else:
           buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`)
 
@@ -1102,10 +1180,11 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
         if prePath.len != 0:
           prePath
         else:
-          getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`)
+          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
 
     static:
-      doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name`
+      doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " &
+        "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name`
       doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found"
       echo "# Including library " & `lpath`
 
diff --git a/nimterop/conan.nim b/nimterop/conan.nim
new file mode 100644
index 0000000..0707ecc
--- /dev/null
+++ b/nimterop/conan.nim
@@ -0,0 +1,373 @@
+import json, marshal, os, strformat, strutils, tables
+
+type
+  ConanPackage* = ref object
+    ## ConanPackage type that stores conan uri and recipes/builds/revisions
+    name*: string
+    version*: string
+    user*: string
+    channel*: string
+    recipes*: OrderedTableRef[string, seq[ConanBuild]]
+
+    bhash*: string
+    shared*: bool
+    headers*: seq[string]
+    libs*: seq[string]
+    requires*: seq[ConanPackage]
+
+  ConanBuild* = ref object
+    ## Build type that stores build specific info and revisions
+    bhash*: string
+    settings*: TableRef[string, string]
+    options*: TableRef[string, string]
+    requires*: seq[string]
+    recipe_hash*: string
+    revisions*: seq[string]
+
+const
+  # Conan API urls
+  baseUrl = "https://conan.bintray.com/v2/conans"
+  searchUrl = baseUrl & "/search?q=$query"
+  pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query"
+  cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions"
+  dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file"
+
+  # Bintray download sub-URL for explicit `user/channel` (not _/_)
+  dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file"
+
+  # Strings
+  conanInfo = "conaninfo.json"
+  conanPackage = "conan_package.tgz"
+  conanManifest = "conanmanifest.txt"
+
+# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim
+# https://bintray.com/conan/conan-center
+# https://bintray.com/bincrafters/public-conan/
+
+# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared
+
+var
+  # Bintray download URL for explicit `user/channel`
+  baseAltUrl {.compiletime.} = {
+    "bincrafters": "https://bintray.com/bincrafters/public-conan",
+    "conan": "https://bintray.com/conan-community/conan"
+  }.toTable()
+
+template fixOutDir() {.dirty.} =
+  let
+    outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
+
+proc addAltBaseUrl*(name, url: string) =
+  # Add an alternate base URL for a custom conan repo on bintray
+  baseAltUrl[name] = url
+
+proc jsonGet(url: string): JsonNode =
+  # Make HTTP call and return content as JSON
+  let
+    temp = getTempDir()
+    file = temp / url.extractFilename()
+
+  downloadUrl(url, temp, quiet = true)
+  result = readFile(file).parseJson()
+  rmFile(file)
+
+proc `==`(pkg1, pkg2: ConanPackage): bool =
+  (not pkg1.isNil and not pkg2.isNil and
+    pkg1.name == pkg2.name and
+    pkg1.version == pkg2.version and
+    pkg1.user == pkg2.user and
+    pkg1.channel == pkg2.channel and
+
+    pkg1.bhash == pkg2.bhash and
+    pkg1.shared == pkg2.shared)
+
+proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shared = true): ConanPackage =
+  ## Create a new ConanPackage with specified name and version
+  result = new(ConanPackage)
+  result.name = name
+  result.version = version
+  result.user = user
+  result.channel = channel
+  result.recipes = newOrderedTable[string, seq[ConanBuild]](2)
+
+  result.bhash = bhash
+  result.shared = shared
+
+proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage =
+  ## Create a new ConanPackage from a conan uri
+  ##
+  ## name/version[@user/channel][:bhash]
+  var
+    name, version, user, channel, bhash: string
+
+    spl = uri.split(":")
+
+  if spl.len > 1:
+    bhash = spl[1]
+
+  spl = spl[0].split('/')
+
+  name = spl[0]
+  user = "_"
+  channel = "_"
+
+  if spl.len > 2:
+    channel = spl[2]
+  if spl.len > 1:
+    spl = spl[1].split('@')
+
+    version = spl[0]
+    if spl.len > 1:
+      user = spl[1]
+
+  result = newConanPackage(name, version, user, channel, bhash, shared)
+
+proc getUriFromConanPackage*(pkg: ConanPackage): string =
+  ## Convert a ConanPackage to a conan uri
+  result = pkg.name
+  if pkg.version.len != 0:
+    result &= "/" & pkg.version
+  if pkg.user.len != 0:
+    result &= "@" & pkg.user
+  if pkg.channel.len != 0:
+    result &= "/" & pkg.channel
+  if pkg.bhash.len != 0:
+    result &= ":" & pkg.bhash
+
+proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage =
+  ## Search for package by `name` and optional `version`, `user` and `channel`
+  var
+    query = name
+  if version.len != 0:
+    query &= "/" & version
+    if user.len != 0:
+      query &= "@" & user
+      if channel.len != 0:
+        query &= "/" & channel
+
+  let
+    j1 = jsonGet(searchUrl % ["query", query])
+    res = j1.getOrDefault("results").getElems()
+
+  if res.len != 0:
+    # Return last entry - latest
+    result = newConanPackageFromUri(res[^1].getStr())
+
+proc searchConan*(pkg: ConanPackage): ConanPackage =
+  ## Search for latest package based on incomplete package info
+  result = searchConan(pkg.name, pkg.version, pkg.user, pkg.channel)
+
+proc getConanBuilds*(pkg: ConanPackage, filter = "") =
+  ## Get all builds for a package based on the C compiler's target OS/arch info
+  ##
+  ## `filter` can be used to tweak search terms
+  ##    e.g. build_type=Debug&compiler=clang
+  let
+    (arch, os, compiler, _) = getGccInfo()
+
+    query =
+      if pkg.bhash.len == 0:
+        block:
+          var
+            query = &"?q=arch={arch}&os={os.capitalizeAscii()}"
+          if "build_type" notin filter:
+            query &= "&build_type=Release"
+          if "shared=" notin filter:
+            query &= &"&options.shared={($pkg.shared).capitalizeAscii()}"
+          if filter.len != 0:
+            query &= &"&{filter}"
+          if "compiler=" notin filter and os != "windows":
+            query &= &"&compiler={compiler}"
+          query.replace("&", "%20and%20")
+      else: ""
+
+    url = pkgUrl % [
+      "name", pkg.name,
+      "version", pkg.version,
+      "user", pkg.user,
+      "channel", pkg.channel,
+      "query", query
+    ]
+
+    j1 = jsonGet(url)
+
+  if not j1.isNil:
+    for bhash, bdata in j1.getFields():
+      if pkg.bhash.len == 0 or pkg.bhash == bhash:
+        let
+          bld = new(ConanBuild)
+          settings = bdata.getOrDefault("settings")
+          options = bdata.getOrDefault("options")
+          requires = bdata.getOrDefault("requires")
+        bld.bhash = bhash
+        if not settings.isNil:
+          bld.settings = newTable[string, string](8)
+          for key, value in settings.getFields():
+            bld.settings[key] = value.getStr()
+        if not options.isNil:
+          bld.options = newTable[string, string](8)
+          for key, value in options.getFields():
+            bld.options[key] = value.getStr()
+        bld.requires = requires.to(seq[string])
+        bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr()
+
+        if pkg.recipes.hasKey(bld.recipe_hash):
+          pkg.recipes[bld.recipe_hash].add bld
+        else:
+          pkg.recipes[bld.recipe_hash] = @[bld]
+
+        # Only need first or matching build
+        break
+
+proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) =
+  ## Get all revisions of a build
+  let
+    url = cfgUrl % [
+      "name", pkg.name,
+      "version", pkg.version,
+      "user", pkg.user,
+      "channel", pkg.channel,
+      "recipe", bld.recipe_hash,
+      "build", bld.bhash
+    ]
+
+    j1 = jsonGet(url)
+
+  if not j1.isNil:
+    let
+      revs = j1.getOrDefault("revisions")
+    for i in revs:
+      bld.revisions.add i.getOrDefault("revision").getStr()
+
+proc loadConanInfo*(outdir: string): ConanPackage =
+  ## Load cached package info from `outdir/conaninfo.txt`
+  fixOutDir()
+  let
+    file = outdir / conanInfo
+
+  if fileExists(file):
+    result = to[ConanPackage](readFile(file))
+
+proc saveConanInfo*(pkg: ConanPackage, outdir: string) =
+  ## Save downloaded package info to `outdir/conaninfo.txt`
+  fixOutDir()
+  let
+    file = outdir / conanInfo
+
+  writeFile(file, $$pkg)
+
+proc parseConanManifest(pkg: ConanPackage, outdir: string) =
+  let
+    file = outdir / conanManifest
+  
+  if fileExists(file):
+    let
+      data = readFile(file)
+    for line in data.splitLines():
+      let
+        line = line.split(':')[0]
+      if line.startsWith("include/"):
+        pkg.headers.add line
+      elif line.startsWith("lib/"):
+        pkg.libs.add line
+      elif line.startsWith("bin/") and line.endsWith("dll"):
+        pkg.libs.add line
+
+proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") =
+  ## Download specific `revision` of `bld` to `outdir`
+  ##
+  ## If omitted, the latest revision (first) is downloaded
+  fixOutDir()
+  let
+    revision =
+      if revision.len != 0:
+        revision
+      else:
+        bld.revisions[0]
+
+    url =
+      if pkg.user == "_":
+        dlUrl % [
+          "name", pkg.name,
+          "version", pkg.version,
+          "user", pkg.user,
+          "channel", pkg.channel,
+          "recipe", bld.recipe_hash,
+          "build", bld.bhash,
+          "revision", revision,
+          "file", conanPackage
+        ]
+      else:
+        baseAltUrl[pkg.user] & dlAltUrl % [
+          "name", pkg.name,
+          "version", pkg.version,
+          "user", pkg.user,
+          "channel", pkg.channel,
+          "build", bld.bhash,
+          "file", conanPackage
+        ]
+
+  downloadUrl(url, outdir, quiet = true)
+  downloadUrl(url.replace(conanPackage, conanManifest), outdir, quiet = true)
+
+  pkg.parseConanManifest(outdir)
+
+  rmFile(outdir / url.extractFilename())
+  rmFile(outdir / conanManifest)
+
+proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string)
+proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
+  ## Download latest recipe/build/revision of `pkg` to `outdir`
+  fixOutDir()
+  let
+    pkg =
+      if pkg.version.len == 0:
+        searchConan(pkg)
+      else:
+        pkg
+
+    cpkg = loadConanInfo(outdir)
+
+  if cpkg == pkg:
+    return
+  elif clean:
+    cleanDir(outdir)
+
+  pkg.getConanBuilds()
+
+  for recipe, builds in pkg.recipes:
+    for build in builds:
+      if pkg.bhash.len == 0 or pkg.bhash == build.bhash:
+        pkg.getConanRevisions(build)
+        pkg.dlConanBuild(build, outdir)
+        pkg.dlConanRequires(build, outdir)
+        break
+    break
+
+  if clean:
+    pkg.saveConanInfo(outdir)
+
+proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
+  ## Download all required dependencies of this `bld`
+  ##
+  ## This is not required for shared libs since conan builds them
+  ## with all dependencies statically linked in
+  fixOutDir()
+  if bld.options["shared"] == "False":
+    for req in bld.requires:
+      let
+        rpkg = newConanPackageFromUri(req)
+
+      downloadConan(rpkg, outdir, clean = false)
+      pkg.requires.add rpkg
+
+proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] =
+  ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps
+  ## in descending order
+  ## 
+  ## `outdir` is prefixed to each entry
+  for lib in pkg.libs:
+    result.add outdir / lib
+
+  for cpkg in pkg.requires:
+    result.add cpkg.getConanLibs(outdir)
\ No newline at end of file