Merge branch 'toast-1'

7 files changed, 208 insertions, 83 deletions

diff --git a/README.md b/README.md
index fcab924..ca16293 100644
--- a/README.md
+++ b/README.md
@@ -93,23 +93,28 @@ The `toast` binary can also be used directly on the CLI:
 toast -h
 Usage:
   main [optional-params] C/C++ source/header
-  Options(opt-arg sep :|=|spc):
+Options(opt-arg sep :|=|spc):
   -h, --help                           print this cligen-erated help
-  --help-syntax                        advanced: prepend, multi-val,..
-  -p, --preprocess     bool     false  run preprocessor on header
-  -a, --past           bool     false  print AST output
-  -n, --pnim           bool     false  print Nim output
-  -r, --recurse        bool     false  process #include files
-  -c, --nocomments     bool     false  exclude top-level comments from output
+  --help-syntax                        advanced: prepend,plurals,..
+  -k, --check          bool     false  check generated wrapper with compiler
+  -d, --debug          bool     false  enable debug output
   -D=, --defines=      strings  {}     definitions to pass to preprocessor
-  -I=, --includeDirs=  strings  {}     include directory to pass to preprocessor
   -l=, --dynlib=       string   ""     Import symbols from library in specified Nim string
-  -O=, --symOverride=  strings  {}     skip generating specified symbols
-  --nim=               string   "nim"  use a particular Nim executable (default: $PATH/nim)
-  --pluginSourcePath=  string   ""     Nim file to build and load as a plugin
-  -d, --debug          bool     false  enable debug output
+  -I=, --includeDirs=  strings  {}     include directory to pass to preprocessor
   -m=, --mode=         string   "cpp"  language parser: c or cpp
+  --nim=               string   "nim"  use a particular Nim executable (default: $PATH/nim)
+  -c, --nocomments     bool     false  exclude top-level comments from output
+  -o=, --output=       string   ""     file to output content - default stdout
+  -a, --past           bool     false  print AST output
   -g, --pgrammar       bool     false  print grammar
+  --pluginSourcePath=  string   ""     Nim file to build and load as a plugin
+  -n, --pnim           bool     false  print Nim output
+  -E=, --prefix=       strings  {}     Strip prefix from identifiers
+  -p, --preprocess     bool     false  run preprocessor on header
+  -r, --recurse        bool     false  process #include files
+  -s, --stub           bool     false  stub out undefined type references as objects
+  -F=, --suffix=       strings  {}     Strip suffix from identifiers
+  -O=, --symOverride=  strings  {}     skip generating specified symbols
 ```
 
 __Implementation Details__
diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim
index 65a5401..c13ce1f 100644
--- a/nimterop/cimport.nim
+++ b/nimterop/cimport.nim
@@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1
 
 include "."/globals
 
-import "."/[build, paths, types]
+import "."/[build, compat, paths, types]
 export types
 
 proc interpPath(dir: string): string=
@@ -116,12 +116,7 @@ proc getNimCheckError(output: string): tuple[tmpFile, errors: string] =
   doAssert fileExists(result.tmpFile), "Failed to write to cache dir: " & result.tmpFile
 
   let
-    nim =
-      when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
-        getCurrentCompilerExe()
-      else:
-        "nim"
-    (check, _) = gorgeEx(&"{nim} check {result.tmpFile.sanitizePath}")
+    (check, _) = gorgeEx(&"{getCurrentCompilerExe()} check {result.tmpFile.sanitizePath}")
 
   result.errors = "\n\n" & check
 
@@ -140,7 +135,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
     cmd.add " --recurse"
 
   if flags.nBl:
-    cmd.add flags
+    cmd.add " " & flags
 
   for i in gStateCT.defines:
     cmd.add &" --defines+={i.quoteShell}"
@@ -157,8 +152,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
     if gStateCT.symOverride.nBl:
       cmd.add &" --symOverride={gStateCT.symOverride.join(\",\")}"
 
-    when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
-      cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}"
+    cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}"
 
     if gStateCT.pluginSourcePath.nBl:
       cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}"
@@ -300,13 +294,18 @@ macro cPlugin*(body): untyped =
   ##
   ##     proc onSymbol(sym: var Symbol) {.exportc, dynlib.}
   ##
-  ## `onSymbol()` can be used to handle symbol name modifications required due to invalid
-  ## characters like leading/trailing `_` or rename symbols that would clash due to Nim's style
-  ## insensitivity. It can also be used to remove prefixes and suffixes like `SDL_`. The symbol
-  ## name and type is provided to the callback and the name can be modified.
+  ## `onSymbol()` can be used to handle symbol name modifications required due
+  ## to invalid characters in identifiers or to rename symbols that would clash
+  ## due to Nim's style insensitivity. The symbol name and type is provided to
+  ## the callback and the name can be modified.
+  ##
+  ## While `cPlugin` can easily remove leading/trailing `_` or prefixes and
+  ## suffixes like `SDL_`, passing `--prefix` or `--suffix` flags to `cImport`
+  ## in the `flags` parameter is much easier. However, these flags will only be
+  ## considered when no `cPlugin` is specified.
   ##
-  ## Returning a blank name will result in the symbol being skipped. This will fail for `nskParam`
-  ## and `nskField` since the generated Nim code will be wrong.
+  ## Returning a blank name will result in the symbol being skipped. This will
+  ## fail for `nskParam` and `nskField` since the generated Nim code will be wrong.
   ##
   ## Symbol types can be any of the following:
   ## - `nskConst` for constants
@@ -316,10 +315,12 @@ macro cPlugin*(body): untyped =
   ## - `nskEnumField` for enum (field) names, though they are in the global namespace as `nskConst`
   ## - `nskProc` - for proc names
   ##
-  ## `nimterop/plugins` is implicitly imported to provide access to standard plugin facilities.
+  ## `nimterop/plugins` is implicitly imported to provide access to standard
+  ## plugin facilities.
   ##
   ## `cPlugin() <cimport.html#cPlugin.m>`_  only affects calls to
-  ## `cImport() <cimport.html#cImport.m%2C%2Cstring%2Cstring%2Cstring>`_ that follow it.
+  ## `cImport() <cimport.html#cImport.m%2C%2Cstring%2Cstring%2Cstring>`_ that
+  ## follow it.
   runnableExamples:
     cPlugin:
       import strutils
@@ -572,7 +573,9 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st
   ## `mode` is purely for forward compatibility when toast adds C++ support. It can
   ## be ignored for the foreseeable future.
   ##
-  ## `flags` can be used to pass any other command line arguments to `toast`.
+  ## `flags` can be used to pass any other command line arguments to `toast`. A
+  ## good example would be `--prefix` and `--suffix` which strip leading and
+  ## trailing strings from identifiers, `_` being quite common.
   ##
   ## `cImport()` consumes and resets preceding `cOverride()` calls. `cPlugin()`
   ## is retained for the next `cImport()` call unless a new `cPlugin()` call is
diff --git a/nimterop/compat.nim b/nimterop/compat.nim
index 1115252..7e2bc98 100644
--- a/nimterop/compat.nim
+++ b/nimterop/compat.nim
@@ -30,3 +30,5 @@ else:
     if not base.endsWith DirSep: base.add DirSep
     doAssert file.startsWith base
     result = file[base.len .. ^1]
+
+  proc getCurrentCompilerExe*(): string = "nim"
diff --git a/nimterop/getters.nim b/nimterop/getters.nim
index 53e04f8..1c5f8aa 100644
--- a/nimterop/getters.nim
+++ b/nimterop/getters.nim
@@ -112,6 +112,7 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent="
 
   if name notin nimState.gState.symOverride or parent.nBl:
     if nimState.gState.onSymbol != nil:
+      # Use onSymbol from plugin provided by user
       var
         sym = Symbol(name: name, parent: parent, kind: kind)
       nimState.gState.onSymbol(sym)
@@ -120,11 +121,23 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent="
     else:
       result = name
 
+      # Strip out --prefix from CLI if specified
+      for str in nimState.gState.prefix:
+        if result.startsWith(str):
+          result = result[str.len .. ^1]
+
+      # Strip out --suffix from CLI if specified
+      for str in nimState.gState.suffix:
+        if result.endsWith(str):
+          result = result[0 .. ^(str.len+1)]
+
     checkIdentifier(result, $kind, parent, name)
 
     if result in gReserved or (result == "object" and kind != nskType):
+      # Enclose in backticks since Nim reserved word
       result = &"`{result}`"
   else:
+    # Skip identifier since in symOverride
     result = ""
 
 proc getUniqueIdentifier*(nimState: NimState, prefix = ""): string =
diff --git a/nimterop/globals.nim b/nimterop/globals.nim
index de29385..7c512b1 100644
--- a/nimterop/globals.nim
+++ b/nimterop/globals.nim
@@ -53,7 +53,7 @@ type
   AstTable {.used.} = TableRef[string, seq[ref Ast]]
 
   State = ref object
-    compile*, defines*, headers*, includeDirs*, searchDirs*, symOverride*: seq[string]
+    compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string]
 
     nocache*, nocomments*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool
 
diff --git a/nimterop/toast.nim b/nimterop/toast.nim
index 9da8283..5923e56 100644
--- a/nimterop/toast.nim
+++ b/nimterop/toast.nim
@@ -1,8 +1,8 @@
-import os, strformat, strutils
+import os, osproc, strformat, strutils, times
 
 import "."/treesitter/[api, c, cpp]
 
-import "."/[ast, globals, getters, grammar]
+import "."/[ast, compat, globals, getters, grammar]
 
 proc printLisp(gState: State, root: TSNode) =
   var
@@ -54,7 +54,7 @@ proc printLisp(gState: State, root: TSNode) =
       break
 
 proc process(gState: State, path: string, astTable: AstTable) =
-  doAssert existsFile(path), "Invalid path " & path
+  doAssert existsFile(path), &"Invalid path {path}"
 
   var
     parser = tsParserNew()
@@ -81,7 +81,7 @@ proc process(gState: State, path: string, astTable: AstTable) =
   elif gState.mode == "cpp":
     doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
   else:
-    doAssert false, "Invalid parser " & gState.mode
+    doAssert false, &"Invalid parser {gState.mode}"
 
   var
     tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32)
@@ -97,84 +97,191 @@ proc process(gState: State, path: string, astTable: AstTable) =
   elif gState.preprocess:
     echo gState.code
 
+# CLI processing with default values
 proc main(
-    preprocess = false,
-    past = false,
-    pnim = false,
-    recurse = false,
-    nocomments = false,
+    check = false,
+    debug = false,
     defines: seq[string] = @[],
-    includeDirs: seq[string] = @[],
     dynlib: string = "",
-    symOverride: seq[string] = @[],
-    nim: string = "nim",
-    pluginSourcePath: string = "",
-    debug = false,
+    includeDirs: seq[string] = @[],
     mode = modeDefault,
+    nim: string = "nim",
+    nocomments = false,
+    output = "",
+    past = false,
     pgrammar = false,
+    pluginSourcePath: string = "",
+    pnim = false,
+    prefix: seq[string] = @[],
+    preprocess = false,
+    recurse = false,
+    stub = false,
+    suffix: seq[string] = @[],
+    symOverride: seq[string] = @[],
     source: seq[string]
   ) =
 
+  # Setup global state with arguments
   var gState = State(
-    preprocess: preprocess,
-    past: past,
-    pnim: pnim,
-    recurse: recurse,
-    nocomments: nocomments,
+    debug: debug,
     defines: defines,
-    includeDirs: includeDirs,
     dynlib: dynlib,
-    symOverride: symOverride,
+    includeDirs: includeDirs,
+    mode: mode,
     nim: nim,
+    nocomments: nocomments,
+    past: past,
     pluginSourcePath: pluginSourcePath,
-    debug: debug,
-    mode: mode,
-    pretty: true
+    pnim: pnim,
+    prefix: prefix,
+    preprocess: preprocess,
+    pretty: true,
+    recurse: recurse,
+    suffix: suffix,
+    symOverride: symOverride
   )
 
+  # Split some arguments with ,
   gState.symOverride = gState.symOverride.getSplitComma()
+  gState.prefix = gState.prefix.getSplitComma()
+  gState.suffix = gState.suffix.getSplitComma()
 
   if pluginSourcePath.nBl:
     gState.loadPlugin(pluginSourcePath)
 
+  # Backup stdout
+  var
+    outputFile = output
+    outputHandle: File
+    stdoutBackup = stdout
+    check = check or stub
+
+  # Fix output file extention
+  if outputFile.len != 0:
+    if outputFile.splitFile().ext != ".nim":
+      outputFile = outputFile & ".nim"
+
+  # Check needs a file
+  if check and outputFile.len == 0:
+    outputFile = getTempDir() / "toast_" & ($getTime().toUnix()).addFileExt("nim")
+    when defined(windows):
+      # https://github.com/nim-lang/Nim/issues/12939
+      echo &"Cannot print wrapper with check on Windows, review {outputFile}\n"
+
+  # Redirect output to file
+  if outputFile.len != 0:
+    when defined(windows):
+      doAssert stdout.reopen(outputFile, fmWrite), &"Failed to write to {outputFile}"
+    else:
+      doAssert outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}"
+      stdout = outputHandle
+
+  # Process grammar into AST
   let
     astTable = parseGrammar()
+
   if pgrammar:
+    # Print AST of grammar
     astTable.printGrammar()
   elif source.nBl:
+    # Print source after preprocess or Nim output
     if gState.pnim:
       printNimHeader()
     for src in source:
       gState.process(src.expandSymlinkAbs(), astTable)
 
+  when not defined(windows):
+    # Restore stdout
+    stdout = stdoutBackup
+
+  # Check Nim output
+  if gState.pnim and check:
+    # Run nim check on generated wrapper
+    var
+      (check, err) = execCmdEx(&"{getCurrentCompilerExe()} check {outputFile}")
+    if err != 0:
+      # Failed check so try stubbing
+      if stub:
+        # Close output file to prepend stubs
+        when not defined(windows):
+          outputHandle.close()
+        else:
+          stdout.close()
+
+        # Find undeclared identifiers in error
+        var
+          data = ""
+          stubData = ""
+        for line in check.splitLines:
+          if "undeclared identifier" in line:
+            try:
+              # Add stub of object type
+              stubData &= "  " & line.split("'")[1] & " = object\n"
+            except:
+              discard
+
+        # Include in wrapper file
+        data = outputFile.readFile()
+        let
+          idx = data.find("\ntype\n")
+        if idx != -1:
+          # In first existing type block
+          data = data[0 ..< idx+6] & stubData & data[idx+6 .. ^1]
+        else:
+          # At the top if none already
+          data = "type\n" & stubData & data
+        outputFile.writeFile(data)
+
+        # Rerun nim check on stubbed wrapper
+        (check, err) = execCmdEx(&"{getCurrentCompilerExe()} check {outputFile}")
+        doAssert err == 0, "# Nim check with stub failed:\n\n" & check
+      else:
+        doAssert err == 0, "# Nim check failed:\n\n" & check
+
+  when not defined(windows):
+    # Print wrapper if temporarily redirected to file
+    if check and output.len == 0:
+      stdout.write outputFile.readFile()
+
 when isMainModule:
+  # Setup cligen command line help and short flags
   import cligen
   dispatch(main, help = {
-    "preprocess": "run preprocessor on header",
-    "past": "print AST output",
-    "pnim": "print Nim output",
-    "recurse": "process #include files",
-    "nocomments": "exclude top-level comments from output",
+    "check": "check generated wrapper with compiler",
+    "debug": "enable debug output",
     "defines": "definitions to pass to preprocessor",
-    "includeDirs": "include directory to pass to preprocessor",
     "dynlib": "Import symbols from library in specified Nim string",
-    "symOverride": "skip generating specified symbols",
-    "nim": "use a particular Nim executable (default: $PATH/nim)",
-    "pluginSourcePath": "Nim file to build and load as a plugin",
-    "debug": "enable debug output",
+    "includeDirs": "include directory to pass to preprocessor",
     "mode": "language parser: c or cpp",
+    "nim": "use a particular Nim executable (default: $PATH/nim)",
+    "nocomments": "exclude top-level comments from output",
+    "output": "file to output content - default stdout",
+    "past": "print AST output",
     "pgrammar": "print grammar",
-    "source" : "C/C++ source/header"
+    "pluginSourcePath": "Nim file to build and load as a plugin",
+    "pnim": "print Nim output",
+    "preprocess": "run preprocessor on header",
+    "recurse": "process #include files",
+    "source" : "C/C++ source/header",
+    "prefix": "Strip prefix from identifiers",
+    "stub": "stub out undefined type references as objects",
+    "suffix": "Strip suffix from identifiers",
+    "symOverride": "skip generating specified symbols"
   }, short = {
-    "preprocess": 'p',
+    "check": 'k',
+    "debug": 'd',
+    "defines": 'D',
+    "dynlib": 'l',
+    "includeDirs": 'I',
+    "nocomments": 'c',
+    "output": 'o',
     "past": 'a',
+    "pgrammar": 'g',
     "pnim": 'n',
+    "prefix": 'E',
+    "preprocess": 'p',
     "recurse": 'r',
-    "nocomments": 'c',
-    "defines": 'D',
-    "includeDirs": 'I',
-    "dynlib": 'l',
-    "symOverride": 'O',
-    "debug": 'd',
-    "pgrammar": 'g'
+    "stub": 's',
+    "suffix": 'F',
+    "symOverride": 'O'
   })
diff --git a/tests/lzma.nim b/tests/lzma.nim
index 785e0bb..4f6c41a 100644
--- a/tests/lzma.nim
+++ b/tests/lzma.nim
@@ -4,6 +4,7 @@ import nimterop/[build, cimport]
 
 const
   baseDir = getProjectCacheDir("nimterop" / "tests" / "liblzma")
+  flags = "--prefix=___,__,_ --suffix=__,_"
 
 static:
   cDebug()
@@ -21,12 +22,6 @@ getHeader(
   conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo"
 )
 
-cPlugin:
-  import strutils
-
-  proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
-    sym.name = sym.name.strip(chars = {'_'})
-
 cOverride:
   type
     lzma_internal = object
@@ -39,8 +34,8 @@ cOverride:
     lzma_index_iter = object
 
 when not lzmaStatic:
-  cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath")
+  cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath", flags = flags)
 else:
-  cImport(lzmaPath, recurse = true)
+  cImport(lzmaPath, recurse = true, flags = flags)
 
 echo "liblzma version = " & $lzma_version_string()