Include comments in output - #61, clean up output

7 files changed, 69 insertions, 49 deletions

diff --git a/nimterop/ast.nim b/nimterop/ast.nim
index c85d5dd..eeaa13e 100644
--- a/nimterop/ast.nim
+++ b/nimterop/ast.nim
@@ -162,10 +162,10 @@ proc printNim*(fullpath: string, root: TSNode, astTable: AstTable) =
   root.searchAst(astTable, nimState)
 
   if nimState.enumStr.nBl:
-    echo nimState.enumStr
+    echo &"{nimState.enumStr}\n"
 
   if nimState.constStr.nBl:
-    echo &"const {nimState.constStr}\n"
+    echo &"const{nimState.constStr}\n"
 
   echo &"""
 {{.pragma: {nimState.impHeader}, importc, header: {nimState.currentHeader}.}}
@@ -173,7 +173,7 @@ proc printNim*(fullpath: string, root: TSNode, astTable: AstTable) =
 """
 
   if nimState.typeStr.nBl:
-    echo &"type {nimState.typeStr}\n"
+    echo &"type{nimState.typeStr}\n"
 
   if nimState.procStr.nBl:
     echo &"{nimState.procStr}\n"
diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim
index 1f82b6f..5fade3a 100644
--- a/nimterop/cimport.nim
+++ b/nimterop/cimport.nim
@@ -138,7 +138,7 @@ proc getToast(fullpath: string, recurse: bool = false): string =
     cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.quoteShell}"
 
   cmd.add &" {fullpath.quoteShell}"
-  echo "# " & cmd
+
   # see https://github.com/nimterop/nimterop/issues/69
   (result, ret) = gorgeEx(cmd, cache=getCacheValue(fullpath))
   doAssert ret == 0, getToastError(result)
@@ -195,7 +195,7 @@ macro cOverride*(body): untyped =
   result = body
 
   if gStateCT.debug:
-    echo "Overriding " & gStateCT.symOverride.join(" ")
+    echo "# Overriding " & gStateCT.symOverride.join(" ")
 
 proc cSkipSymbol*(skips: seq[string]) {.compileTime.} =
   ## Similar to `cOverride() <cimport.html#cOverride.m,>`_, this macro allows
@@ -320,7 +320,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped =
       {.passC: `str`.}
 
     if gStateCT.debug:
-      echo result.repr
+      echo result.repr & "\n"
 
 proc cAddSearchDir*(dir: string) {.compileTime.} =
   ## Add directory ``dir`` to the search path used in calls to
@@ -483,13 +483,13 @@ macro cImport*(filename: static string, recurse: static bool = false): untyped =
   let
     output = getToast(fullpath, recurse)
 
+  if gStateCT.debug:
+    echo output
+
   try:
     let body = parseStmt(output)
 
     result.add body
-
-    if gStateCT.debug:
-      echo result.repr
   except:
     let
       (tmpFile, errors) = getNimCheckError(output)
diff --git a/nimterop/getters.nim b/nimterop/getters.nim
index c9aecf8..93beb80 100644
--- a/nimterop/getters.nim
+++ b/nimterop/getters.nim
@@ -204,7 +204,7 @@ proc removeStatic(content: string): string =
 proc getPreprocessor*(fullpath: string, mode = "cpp"): string =
   var
     mmode = if mode == "cpp": "c++" else: mode
-    cmd = &"gcc -E -dD -x{mmode} -w "
+    cmd = &"gcc -E -CC -dD -x{mmode} -w "
 
     rdata: seq[string] = @[]
     start = false
diff --git a/nimterop/git.nim b/nimterop/git.nim
index c652d16..3a0f493 100644
--- a/nimterop/git.nim
+++ b/nimterop/git.nim
@@ -54,7 +54,7 @@ proc extractZip*(zipfile, outdir: string) =
           "'System.IO.Compression.FileSystem'; " &
           "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\""
 
-  echo "Extracting " & zipfile
+  echo "# Extracting " & zipfile
   discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}")
 
 proc downloadUrl*(url, outdir: string) =
@@ -63,7 +63,7 @@ proc downloadUrl*(url, outdir: string) =
     ext = file.splitFile().ext.toLowerAscii()
 
   if not (ext == ".zip" and fileExists(outdir/file)):
-    echo "Downloading " & file
+    echo "# Downloading " & file
     mkDir(outdir)
     var cmd = if defined(Windows):
       "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#"
@@ -75,20 +75,20 @@ proc downloadUrl*(url, outdir: string) =
       extractZip(file, outdir)
 
 proc gitReset*(outdir: string) =
-  echo "Resetting " & outdir
+  echo "# Resetting " & outdir
 
   let cmd = &"cd {outdir.quoteShell} && git reset --hard"
   while execAction(cmd).contains("Permission denied"):
     sleep(1000)
-    echo "  Retrying ..."
+    echo "#   Retrying ..."
 
 proc gitCheckout*(file, outdir: string) =
-  echo "Resetting " & file
+  echo "# Resetting " & file
   let file2 = file.relativePath outdir
   let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}"
   while execAction(cmd).contains("Permission denied"):
     sleep(500)
-    echo "  Retrying ..."
+    echo "#   Retrying ..."
 
 proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
   if dirExists(outdir/".git"):
@@ -100,7 +100,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
 
   mkDir(outdir)
 
-  echo "Setting up Git repo: " & url
+  echo "# Setting up Git repo: " & url
   discard execAction(&"cd {outdirQ} && git init .")
   discard execAction(&"cd {outdirQ} && git remote add origin {url}")
 
@@ -112,27 +112,27 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
     writeFile(sparsefile, plist)
 
   if checkout.len != 0:
-    echo "Checking out " & checkout
+    echo "# Checking out " & checkout
     discard execAction(&"cd {outdirQ} && git pull --tags origin master")
     discard execAction(&"cd {outdirQ} && git checkout {checkout}")
   else:
-    echo "Pulling repository"
+    echo "# Pulling repository"
     discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master")
 
 proc configure*(path, check: string) =
   if (path / check).fileExists():
     return
 
-  echo "Configuring " & path
+  echo "# Configuring " & path
 
   if not fileExists(path / "configure"):
     if fileExists(path / "autogen.sh"):
-      echo "  Running autogen.sh"
+      echo "#   Running autogen.sh"
 
       discard execAction(&"cd {path.quoteShell} && bash autogen.sh")
 
   if fileExists(path / "configure"):
-    echo "  Running configure"
+    echo "#   Running configure"
 
     discard execAction(&"cd {path.quoteShell} && bash configure")
 
diff --git a/nimterop/globals.nim b/nimterop/globals.nim
index 61054b9..2d8d651 100644
--- a/nimterop/globals.nim
+++ b/nimterop/globals.nim
@@ -64,7 +64,7 @@ type
   NimState {.used.} = ref object
     identifiers*: TableRef[string, string]
 
-    constStr*, debugStr*, enumStr*, procStr*, typeStr*: string
+    constStr*, debugStr*, enumStr*, procStr*, typeStr*, commentStr*: string
 
     debug*: bool
 
diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim
index 11c5f55..4918d05 100644
--- a/nimterop/grammar.nim
+++ b/nimterop/grammar.nim
@@ -23,6 +23,11 @@ proc genPragma(nimState: NimState, pragmas: varargs[string]): string =
 
   result = result.replace(nimState.impHeader & ", cdecl", nimState.impHeader & "C")
 
+proc getComments(nimState: NimState): string =
+  if nimState.commentStr.len != 0:
+    result = "\n" & nimState.commentStr
+    nimState.commentStr = ""
+
 proc initGrammar(): Grammar =
   # #define X Y
   result.add(("""
@@ -43,7 +48,7 @@ proc initGrammar(): Grammar =
           name = nimState.data[0].val.getIdentifier(nskConst)
 
         if name.nBl and nimState.identifiers.addNewIdentifer(name):
-          nimState.constStr &= &"\n  {name}* = {val}"
+          nimState.constStr &= &"{nimState.getComments()}\n  {name}* = {val}"
   ))
 
   let
@@ -191,9 +196,9 @@ proc initGrammar(): Grammar =
             pout = pout[0 .. ^3]
 
           if tptr.len != 0 or typ != "object":
-            nimState.typeStr &= &"\n  {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}"
           else:
-            nimState.typeStr &= &"\n  {nname}*{pragma} = proc({pout}) {{.cdecl.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = proc({pout}) {{.cdecl.}}"
         else:
           if i < nimState.data.len and nimState.data[i].name in ["identifier", "number_literal"]:
             var
@@ -201,12 +206,12 @@ proc initGrammar(): Grammar =
             if nimState.data[i].name == "identifier":
               flen = flen.getIdentifier(nskConst, nname)
 
-            nimState.typeStr &= &"\n  {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]"
           else:
             if nname == typ:
-              nimState.typeStr &= &"\n  {nname}*{pragma} = object"
+              nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = object"
             else:
-              nimState.typeStr &= &"\n  {nname}*{pragma} = {getPtrType(tptr&typ)}"
+              nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = {getPtrType(tptr&typ)}"
   ))
 
   proc pDupTypeCommon(nname: string, fend: int, nimState: NimState, isEnum=false) =
@@ -225,13 +230,13 @@ proc initGrammar(): Grammar =
     if ndname.nBl and ndname != nname:
       if isEnum:
         if nimState.identifiers.addNewIdentifer(ndname):
-          nimState.enumStr &= &"\ntype {ndname}* = {dptr}{nname}"
+          nimState.enumStr &= &"{nimState.getComments()}\ntype {ndname}* = {dptr}{nname}"
       else:
         if nimState.identifiers.addNewIdentifer(ndname):
           let
             pragma = nimState.genPragma(nimState.genImportc(dname, ndname), "bycopy")
           nimState.typeStr &=
-            &"\n  {ndname}*{pragma} = {dptr}{nname}"
+            &"{nimState.getComments()}\n  {ndname}*{pragma} = {dptr}{nname}"
 
   proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, nimState: NimState) =
     if nimState.debug:
@@ -266,11 +271,11 @@ proc initGrammar(): Grammar =
 
     if nname.nBl and nimState.identifiers.addNewIdentifer(nname):
       if nimState.data.len == 1:
-        nimState.typeStr &= &"\n  {nname}* {{.bycopy.}} = object{union}"
+        nimState.typeStr &= &"{nimState.getComments()}\n  {nname}* {{.bycopy.}} = object{union}"
       else:
         let
           pragma = nimState.genPragma(nimState.genImportC(prefix & name, nname), "bycopy")
-        nimState.typeStr &= &"\n  {nname}*{pragma} = object{union}"
+        nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = object{union}"
 
       var
         i = fstart
@@ -302,7 +307,7 @@ proc initGrammar(): Grammar =
         if i+1 < nimState.data.len-fend and nimState.data[i+1].name in gEnumVals:
           let
             flen = nimState.data[i+1].val.getNimExpression()
-          nimState.typeStr &= &"\n    {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]"
+          nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]"
           i += 2
         elif i+1 < nimState.data.len-fend and nimState.data[i+1].name == "function_declarator":
           var
@@ -323,15 +328,15 @@ proc initGrammar(): Grammar =
           if pout.len != 0 and pout[^2 .. ^1] == ", ":
             pout = pout[0 .. ^3]
           if fptr.len != 0 or ftyp != "object":
-            nimState.typeStr &= &"\n    {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}"
           else:
-            nimState.typeStr &= &"\n    {fname}*: proc({pout}) {{.cdecl.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: proc({pout}) {{.cdecl.}}"
             i += 1
         else:
           if ftyp == "object":
-            nimState.typeStr &= &"\n    {fname}*: pointer"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: pointer"
           else:
-            nimState.typeStr &= &"\n    {fname}*: {getPtrType(fptr&ftyp)}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: {getPtrType(fptr&ftyp)}"
           i += 1
 
       if node.tsNodeType() == "type_definition" and
@@ -439,7 +444,7 @@ proc initGrammar(): Grammar =
         name.getIdentifier(nskType)
 
     if nname.nBl and nimState.identifiers.addNewIdentifer(nname):
-      nimState.enumStr &= &"\ndefineEnum({nname})"
+      nimState.enumStr &= &"{nimState.getComments()}\ndefineEnum({nname})"
 
       var
         i = fstart
@@ -455,7 +460,7 @@ proc initGrammar(): Grammar =
         if i+1 < nimState.data.len-fend and
           nimState.data[i+1].name in gEnumVals:
           if fname.nBl and nimState.identifiers.addNewIdentifer(fname):
-            nimState.constStr &= &"\n  {fname}* = ({nimState.data[i+1].val.getNimExpression()}).{nname}"
+            nimState.constStr &= &"{nimState.getComments()}\n  {fname}* = ({nimState.data[i+1].val.getNimExpression()}).{nname}"
           try:
             count = nimState.data[i+1].val.parseInt() + 1
           except:
@@ -463,7 +468,7 @@ proc initGrammar(): Grammar =
           i += 2
         else:
           if fname.nBl and nimState.identifiers.addNewIdentifer(fname):
-            nimState.constStr &= &"\n  {fname}* = {count}.{nname}"
+            nimState.constStr &= &"{nimState.getComments()}\n  {fname}* = {count}.{nname}"
           i += 1
           count += 1
 
@@ -584,9 +589,25 @@ proc initGrammar(): Grammar =
             pragma = nimState.genPragma(nimState.genImportC(fname, fnname), "cdecl")
 
           if fptr.len != 0 or ftyp != "object":
-            nimState.procStr &= &"\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}"
+            nimState.procStr &= &"{nimState.getComments()}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}"
           else:
-            nimState.procStr &= &"\nproc {fnname}*({pout}){pragma}"
+            nimState.procStr &= &"{nimState.getComments()}\nproc {fnname}*({pout}){pragma}"
+  ))
+
+  # // comment
+  result.add((&"""
+   (comment
+   )
+  """,
+    proc (ast: ref Ast, node: TSNode, nimState: NimState) =
+      let
+        cmt = $node.getNodeVal()
+
+      for line in cmt.splitLines():
+        let
+          line = line.multiReplace([("//", ""), ("/*", ""), ("*/", "")])
+
+        nimState.commentStr &= &"\n#{line.strip(leading=false)}"
   ))
 
 proc initRegex(ast: ref Ast) =
diff --git a/nimterop/lisp.nim b/nimterop/lisp.nim
index 119de36..8a256be 100644
--- a/nimterop/lisp.nim
+++ b/nimterop/lisp.nim
@@ -30,12 +30,11 @@ proc readFromTokens(): ref Ast =
   if gTokens[idx] == "(":
     if gTokens.len - idx < 2:
       doAssert false, "Corrupt AST " & $(gTokensLen: gTokens.len, idx: idx)
-    if gTokens[idx+1] != "comment":
-      result = new(Ast)
-      (result.name, result.kind, result.recursive) = gTokens[idx+1].getNameKind()
-      result.children = @[]
-      if result.recursive:
-        result.children.add(result)
+    result = new(Ast)
+    (result.name, result.kind, result.recursive) = gTokens[idx+1].getNameKind()
+    result.children = @[]
+    if result.recursive:
+      result.children.add(result)
     idx += 2
     while gTokens[idx] != ")":
       var res = readFromTokens()