OS specific sections, execAction, nowildcard, multi git repos, branch/tag/commit

8 files changed, 188 insertions, 133 deletions

diff --git a/README.md b/README.md
index 0ee6c9f..153fe98 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,8 @@ In all sections below, environment variables are supported via Nim's string inte
     "${output}/library/include"
     "${MY_INCLUDE_PATH}/include"
 
+Append -win, -lin and -mac/osx for OS specific sections and tasks. E.g. n.global-win, n.post-lin, download-win, execute-lin-mac.unique1. -unix can be used as a shortcut for -lin-mac.
+
 _[n.global]_
 
 ```output``` = name of the Nimble project once installed, also location to place generated .nim files
@@ -119,17 +121,21 @@ List of all directories or files to exclude from all parsing. If an entry here m
 
 _[n.prepare]_
 
-The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -mac/osx can be used for OS specific tasks. E.g. download-win, execute-lin,mac.unique1
+The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1.
 
 ```download``` = url to download to the output directory. ZIP files are automatically extracted. Files are not redownloaded if already present but re-extracted
 
 ```extract``` = ZIP file to extract in case they are local and don't need to be downloaded. Path is relative to output directory.
 
-```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets to HEAD if already present
+```gitcheckout``` = branch, commit or tag of repository to checkout in following Git command, resets after each use. Use "-b name" for branches
+
+```gitoutput`` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory]
+
+```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets if already present
 
 ```gitremote``` = url of Git repository to partially checkout. Use with gitsparse to pull only files and dirs of interest
 
-```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets to HEAD if already present
+```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets if already present
 
 ```execute``` = command to run during preparation
 
@@ -139,7 +145,9 @@ _[n.post]_
 
 This section is the same as the prepare section, but for performing actions after the project has been processed.
 
-```reset``` = whether or not to perform a git reset on all files after processing [default: false]
+```gitoutput`` = output directory for Git reset [default: `n.global:output` directory]
+
+```reset``` = perform a Git reset on all files after processing [default: false]
 
 ```execute``` = command to run after processing
 
@@ -155,7 +163,7 @@ This section allows selection of multiple sourcefiles without requiring a detail
 
 _[sourcefile]_
 
-The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win
+The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win.
 
 ```recurse``` = find #include files and process them [default: false]
 
@@ -173,6 +181,8 @@ The following keys apply to library source code and help with generating the .ni
 
 ```noprocess``` = do not process this source file with c2nim [default: false] - this is useful if a file only needs to be manipulated
 
+```nowildcard``` = ignore any wildcard definitions for this sourcefile
+
 ```reset``` = reset the file back to original state after all processing [default: false]
 
 Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir
diff --git a/appveyor.yml b/appveyor.yml
index a7337a1..531866f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -10,7 +10,7 @@ matrix:
 environment:
   matrix:
     - NIM_VERSION: 0.18.0
-    - NIM_VERSION: 0.19.0
+    # - NIM_VERSION: 0.19.0
 
 for:
 -
diff --git a/nimgen.nimble b/nimgen.nimble
index cb85eec..c99b7a6 100644
--- a/nimgen.nimble
+++ b/nimgen.nimble
@@ -1,6 +1,6 @@
 # Package
 
-version       = "0.4.0"
+version       = "0.5.0"
 author        = "genotrance"
 description   = "c2nim helper to simplify and automate the wrapping of C libraries"
 license       = "MIT"
@@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"]
 
 # Dependencies
 
-requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex <= 0.7.4"
+requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.10.0"
 
 task test, "Test nimgen":
     exec "nim c -r tests/rununittests.nim"
diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim
index 30f23f0..0464c6a 100644
--- a/src/nimgen/c2nim.nim
+++ b/src/nimgen/c2nim.nim
@@ -2,8 +2,13 @@ import os, ospaths, regex, strutils
 
 import external, file, fileops, gencore, globals
 
-template relativePath(path: untyped): untyped =
-  path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")])
+proc relativePath(path: string): string =
+  if gOutput.len() == 0:
+    result = path
+  else:
+    # multiReplace() bug - #9557
+    result = path.replace(gOutput, "")
+  return result.multiReplace([("\\", "/"), ("//", "/")])
 
 proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) =
   var file = search(fl)
diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim
index 651ccf3..a34ea69 100644
--- a/src/nimgen/external.nim
+++ b/src/nimgen/external.nim
@@ -15,6 +15,16 @@ proc execProc*(cmd: string): string =
     echo result
     quit(1)
 
+proc execAction*(cmd: string): string =
+  var ccmd = ""
+  when defined(Windows):
+    ccmd = "cmd /c " & cmd.replace("/", "\\")
+  when defined(Linux) or defined(MacOSX):
+    ccmd = "bash -c '" & cmd & "'"
+
+  echo "Running '" & ccmd[0..min(64, len(ccmd))] & "'"
+  return execProc(ccmd)
+
 proc extractZip*(zipfile: string) =
   var cmd = "unzip -o $#"
   if defined(Windows):
@@ -44,13 +54,16 @@ proc downloadUrl*(url: string) =
   if ext == ".zip":
     extractZip(file)
 
+template setGitDir() =
+  setCurrentDir(gGitOutput)
+  defer: setCurrentDir(gProjectDir)
+
 proc gitReset*() =
-  echo "Resetting Git repo"
+  echo "Resetting " & gGitOutput
 
-  setCurrentDir(gOutput)
-  defer: setCurrentDir(gProjectDir)
+  setGitDir()
 
-  let cmd = "git reset --hard HEAD"
+  let cmd = "git reset --hard"
   while execCmdEx(cmd)[0].contains("Permission denied"):
     sleep(1000)
     echo "  Retrying ..."
@@ -58,45 +71,51 @@ proc gitReset*() =
 proc gitCheckout*(file: string) =
   echo "Resetting " & file
 
-  setCurrentDir(gOutput)
-  defer: setCurrentDir(gProjectDir)
+  setGitDir()
 
-  let cmd = "git checkout $#" % file.replace(gOutput & "/", "")
+  let cmd = "git checkout $#" % file.replace(gGitOutput & "/", "")
   while execCmdEx(cmd)[0].contains("Permission denied"):
     sleep(500)
     echo "  Retrying ..."
 
+proc gitPull() =
+  if gGitCheckout.len() != 0:
+    echo "Checking out " & gGitCheckout
+    #discard execProc("git fetch --depth=1 origin " & gGitCheckout)
+    discard execProc("git pull --tags origin master")
+    discard execProc("git checkout " & gGitCheckout)
+    gGitCheckout = ""
+  else:
+    echo "Pulling repository"
+    discard execProc("git pull --depth=1 origin master")
+
 proc gitRemotePull*(url: string, pull=true) =
-  if dirExists(gOutput/".git"):
+  if dirExists(gGitOutput/".git"):
     if pull:
       gitReset()
     return
 
-  setCurrentDir(gOutput)
-  defer: setCurrentDir(gProjectDir)
+  setGitDir()
 
-  echo "Setting up Git repo"
+  echo "Setting up Git repo: " & url
   discard execProc("git init .")
   discard execProc("git remote add origin " & url)
 
   if pull:
-    echo "Checking out artifacts"
-    discard execProc("git pull --depth=1 origin master")
+    gitPull()
 
 proc gitSparseCheckout*(plist: string) =
   let sparsefile = ".git/info/sparse-checkout"
-  if fileExists(gOutput/sparsefile):
+  if fileExists(gGitOutput/sparsefile):
     gitReset()
     return
 
-  setCurrentDir(gOutput)
-  defer: setCurrentDir(gProjectDir)
+  setGitDir()
 
   discard execProc("git config core.sparsecheckout true")
   writeFile(sparsefile, plist)
 
-  echo "Checking out artifacts"
-  discard execProc("git pull --depth=1 origin master")
+  gitPull()
 
 proc runPreprocess*(file, ppflags, flags: string, inline: bool): string =
   var
diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim
index edf39ea..769c914 100644
--- a/src/nimgen/file.nim
+++ b/src/nimgen/file.nim
@@ -16,8 +16,7 @@ proc getNimout*(file: string, rename=true): string =
   if gRenames.hasKey(file):
     result = gRenames[file]
 
-  if not dirExists(parentDir(result)):
-    createDir(parentDir(result))
+  createDir(parentDir(result))
 
 proc exclude*(file: string): bool =
   for excl in gExcludes:
diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim
index a83460f..9892b99 100644
--- a/src/nimgen/globals.nim
+++ b/src/nimgen/globals.nim
@@ -7,21 +7,27 @@ const
   defaultCppCompiler* = "g++"
 
 var
-  gDoneRecursive*: seq[string] = @[]
-  gDoneInline*: seq[string] = @[]
-
-  gProjectDir* = ""
+  # Config
   gConfig*: Config
-  gFilter* = ""
-  gQuotes* = true
-  gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler)
-  gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler)
-  gOutput* = ""
-  gIncludes*: seq[string] = @[]
   gExcludes*: seq[string] = @[]
+  gIncludes*: seq[string] = @[]
   gRenames* = initTable[string, string]()
   gWildcards* = newConfig()
+
+  # n.global
+  gOutput* = "."
+  gQuotes* = true
+  gFilter* = ""
+  gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler)
+  gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler)
+
+  # State tracking
+  gGitCheckout* = ""
+  gGitOutput* = ""
+  gProjectDir* = ""
   gCompile*: seq[string] = @[]
+  gDoneInline*: seq[string] = @[]
+  gDoneRecursive*: seq[string] = @[]
 
 type
   c2nimConfigObj* = object
diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim
index 5f04b04..f2310ff 100644
--- a/src/nimgen/runcfg.nim
+++ b/src/nimgen/runcfg.nim
@@ -6,18 +6,19 @@ proc `[]`*(table: OrderedTableRef[string, string], key: string): string =
   ## Gets table values with env vars inserted
   tables.`[]`(table, key).addEnv
 
-proc getKey(ukey: string): tuple[key: string, val: bool] =
-  var kv = ukey.replace(re"\..*", "").split("-", 1)
+proc getKey(ukey: string, section = false): tuple[key: string, val: bool] =
+  var kv = if not section: ukey.replace(re"\..*", "").split("-", 1) else: ukey.split("-", 1)
   if kv.len() == 1:
     kv.add("")
 
   if kv[1] == "":
     return (kv[0], true)
 
-  for ostyp in kv[1].split(","):
+  for ostyp in kv[1].split("-"):
     if (ostyp == "win" and defined(Windows)) or
        (ostyp == "lin" and defined(Linux)) or
-       ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)):
+       ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)) or
+       (ostyp == "unix" and (defined(Linux) or defined(MacOSX))):
       return (kv[0], true)
 
   return (kv[0], false)
@@ -26,6 +27,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str
   var
     cfg = cfgin
     sfile = search(file)
+    nowildcard = false
 
   if sfile in gDoneRecursive:
     return
@@ -34,13 +36,20 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str
     echo "Processing " & sfile
     gDoneRecursive.add(sfile)
 
-  for pattern in gWildcards.keys():
-    var m: RegexMatch
-    let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?")
-    if file.find(toPattern(pat), m):
-      echo "  Appending keys for wildcard " & pattern
-      for key in gWildcards[pattern].keys():
-        cfg[key & "." & pattern] = gWildcards[pattern][key]
+  for act in cfg.keys():
+    let (action, val) = getKey(act)
+    if val == true and action == "nowildcard" and cfg[act] == "true":
+      nowildcard = true
+      break
+
+  if not nowildcard:
+    for pattern in gWildcards.keys():
+      var m: RegexMatch
+      let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?")
+      if file.find(toPattern(pat), m):
+        echo "  Appending keys for wildcard " & pattern
+        for key in gWildcards[pattern].keys():
+          cfg[key & "." & pattern] = gWildcards[pattern][key]
 
   var
     srch = ""
@@ -200,92 +209,99 @@ proc runCfg*(cfg: string) =
   gProjectDir = parentDir(cfg.expandFilename()).sanitizePath
 
   gConfig = loadConfig(cfg)
+  gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler).quoteShell
+  gCCompiler = getEnv(cCompilerEnv, defaultCCompiler).quoteShell
+  gGitOutput = gOutput
 
-  if gConfig.hasKey("n.global"):
-    if gConfig["n.global"].hasKey("output"):
-      setOutputDir(gConfig["n.global"]["output"])
-
-    if gConfig["n.global"].hasKey("cpp_compiler"):
-      gCppCompiler = gConfig["n.global"]["cpp_compiler"]
-    else:
-      # Reset on a per project basis
-      gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler)
-
-    if gConfig["n.global"].hasKey("c_compiler"):
-      gCCompiler = gConfig["n.global"]["c_compiler"]
-    else:
-      # Reset on a per project basis
-      gCCompiler = getEnv(cCompilerEnv, defaultCCompiler)
-
-    gCppCompiler = gCppCompiler.quoteShell
-    gCCompiler = gCCompiler.quoteShell
-
-    if gConfig["n.global"].hasKey("filter"):
-      gFilter = gConfig["n.global"]["filter"]
-    if gConfig["n.global"].hasKey("quotes"):
-      if gConfig["n.global"]["quotes"] == "false":
-        gQuotes = false
-
-  if gConfig.hasKey("n.include"):
-    for inc in gConfig["n.include"].keys():
-      gIncludes.add(inc.addEnv().sanitizePath)
-
-  if gConfig.hasKey("n.exclude"):
-    for excl in gConfig["n.exclude"].keys():
-      gExcludes.add(excl.addEnv().sanitizePath)
-
-  if gConfig.hasKey("n.prepare"):
-    for prep in gConfig["n.prepare"].keys():
-      let (key, val) = getKey(prep)
-      if val == true:
-        let prepVal = gConfig["n.prepare"][prep]
-        if key == "download":
-          downloadUrl(prepVal)
-        elif key == "extract":
-          extractZip(prepVal)
-        elif key == "git":
-          gitRemotePull(prepVal)
-        elif key == "gitremote":
-          gitRemotePull(prepVal, false)
-        elif key == "gitsparse":
-          gitSparseCheckout(prepVal)
-        elif key == "execute":
-          discard execProc(prepVal)
-        elif key == "copy":
-          doCopy(prepVal)
-
-  if gConfig.hasKey("n.wildcard"):
-    var wildcard = ""
-    for wild in gConfig["n.wildcard"].keys():
-      let (key, val) = getKey(wild)
-      if val == true:
-        if key == "wildcard":
-          wildcard = gConfig["n.wildcard"][wild]
-        else:
-          gWildcards.setSectionKey(wildcard, wild,
-                                   gConfig["n.wildcard"][wild])
-
-  for file in gConfig.keys():
-    if file in @["n.global", "n.include", "n.exclude",
-                 "n.prepare", "n.wildcard", "n.post"]:
+  for section in gConfig.keys():
+    let (sname, sval) = getKey(section, true)
+    if not sval:
       continue
 
-    if file == "n.sourcefile":
-      for pattern in gConfig["n.sourcefile"].keys():
-        for file in walkFiles(pattern.addEnv):
-          runFile(file)
-    else:
-      runFile(file, gConfig[file])
+    case sname:
+      of "n.global":
+        for glob in gConfig[section].keys():
+          let (key, val) = getKey(glob)
+          if val == true:
+            let globVal = gConfig[section][glob]
+            case key:
+              of "output":
+                setOutputDir(globVal)
+              of "cpp_compiler":
+                gCppCompiler = globVal.quoteShell
+              of "c_compiler":
+                gCCompiler = globVal.quoteShell
+              of "filter":
+                gFilter = globVal
+              of "quotes":
+                if globVal == "false":
+                  gQuotes = false
+
+      of "n.include":
+        for inc in gConfig[section].keys():
+          gIncludes.add(inc.addEnv().sanitizePath)
+
+      of "n.exclude":
+        for excl in gConfig[section].keys():
+          gExcludes.add(excl.addEnv().sanitizePath)
+
+      of "n.prepare":
+        for prep in gConfig[section].keys():
+          let (key, val) = getKey(prep)
+          if val == true:
+            let prepVal = gConfig[section][prep]
+            case key:
+              of "download":
+                downloadUrl(prepVal)
+              of "extract":
+                extractZip(prepVal)
+              of "gitcheckout":
+                gGitCheckout = prepVal
+              of "gitoutput":
+                gGitOutput = gOutput/prepVal
+                createDir(gGitOutput)
+              of "git":
+                gitRemotePull(prepVal)
+              of "gitremote":
+                gitRemotePull(prepVal, false)
+              of "gitsparse":
+                gitSparseCheckout(prepVal)
+              of "execute":
+                discard execAction(prepVal)
+              of "copy":
+                doCopy(prepVal)
+
+      of "n.wildcard":
+        var wildcard = ""
+        for wild in gConfig[section].keys():
+          let (key, val) = getKey(wild)
+          if val == true:
+            if key == "wildcard":
+              wildcard = gConfig[section][wild]
+            else:
+              gWildcards.setSectionKey(wildcard, wild,
+                                      gConfig[section][wild])
+
+      of "n.sourcefile":
+        for pattern in gConfig[section].keys():
+          for file in walkFiles(pattern.addEnv):
+            runFile(file)
+
+      of "n.post":
+        for post in gConfig[section].keys():
+          let (key, val) = getKey(post)
+          if val == true:
+            let postVal = gConfig[section][post]
+            case key:
+              of "gitoutput":
+                gGitOutput = gOutput/postVal
+              of "reset":
+                gitReset()
+              of "execute":
+                discard execAction(postVal)
 
-  if gConfig.hasKey("n.post"):
-    for post in gConfig["n.post"].keys():
-      let (key, val) = getKey(post)
-      if val == true:
-        let postVal = gConfig["n.post"][post]
-        if key == "reset":
-          gitReset()
-        elif key == "execute":
-          discard execProc(postVal)
+      else:
+        runFile(section, gConfig[section])
 
 let gHelp = """
 Nimgen is a helper for c2nim to simplify and automate the wrapping of C libraries