ast2 proc pragmas, fix typedef struct X importc, convention selection, ast2 bitfield support

8 files changed, 183 insertions, 77 deletions

diff --git a/README.md b/README.md
index b1fee44..18f74ce 100644
--- a/README.md
+++ b/README.md
@@ -86,7 +86,6 @@ Check out [template.nim](https://github.com/nimterop/nimterop/blob/master/nimter
 
 Refer to the ```tests``` directory for examples on how the library can be used.
 
-
 The `toast` binary can also be used directly on the CLI:
 
 ```
@@ -94,29 +93,30 @@ The `toast` binary can also be used directly on the CLI:
 Usage:
   main [optional-params] C/C++ source/header
 Options:
-  -h, --help                            print this cligen-erated help
-  --help-syntax                         advanced: prepend,plurals,..
-  -k, --check          bool      false  check generated wrapper with compiler
-  -d, --debug          bool      false  enable debug output
-  -D=, --defines=      strings   {}     definitions to pass to preprocessor
-  -l=, --dynlib=       string    ""     import symbols from library in specified Nim string
-  -f=, --feature=      Features  {}     flags to enable experimental features
-  -H, --includeHeader  bool      false  add {.header.} pragma to wrapper
-  -I=, --includeDirs=  strings   {}     include directory to pass to preprocessor
-  -m=, --mode=         string    "cpp"  language parser: c or cpp
-  --nim=               string    "nim"  use a particular Nim executable (default: $PATH/nim)
-  -c, --nocomments     bool      false  exclude top-level comments from output
-  -o=, --output=       string    ""     file to output content - default stdout
-  -a, --past           bool      false  print AST output
-  -g, --pgrammar       bool      false  print grammar
-  --pluginSourcePath=  string    ""     nim file to build and load as a plugin
-  -n, --pnim           bool      false  print Nim output
-  -E=, --prefix=       strings   {}     strip prefix from identifiers
-  -p, --preprocess     bool      false  run preprocessor on header
-  -r, --recurse        bool      false  process #include files
-  -s, --stub           bool      false  stub out undefined type references as objects
-  -F=, --suffix=       strings   {}     strip suffix from identifiers
-  -O=, --symOverride=  strings   {}     skip generating specified symbols
+  -h, --help                              print this cligen-erated help
+  --help-syntax                           advanced: prepend,plurals,..
+  -k, --check          bool      false    check generated wrapper with compiler
+  -C=, --convention=   string    "cdecl"  calling convention for wrapped procs - default: cdecl
+  -d, --debug          bool      false    enable debug output
+  -D=, --defines=      strings   {}       definitions to pass to preprocessor
+  -l=, --dynlib=       string    ""       import symbols from library in specified Nim string
+  -f=, --feature=      Features  {}       flags to enable experimental features
+  -H, --includeHeader  bool      false    add {.header.} pragma to wrapper
+  -I=, --includeDirs=  strings   {}       include directory to pass to preprocessor
+  -m=, --mode=         string    "cpp"    language parser: c or cpp
+  --nim=               string    "nim"    use a particular Nim executable - default: $PATH/nim
+  -c, --nocomments     bool      false    exclude top-level comments from output
+  -o=, --output=       string    ""       file to output content - default: stdout
+  -a, --past           bool      false    print AST output
+  -g, --pgrammar       bool      false    print grammar
+  --pluginSourcePath=  string    ""       nim file to build and load as a plugin
+  -n, --pnim           bool      false    print Nim output
+  -E=, --prefix=       strings   {}       strip prefix from identifiers
+  -p, --preprocess     bool      false    run preprocessor on header
+  -r, --recurse        bool      false    process #include files
+  -s, --stub           bool      false    stub out undefined type references as objects
+  -F=, --suffix=       strings   {}       strip suffix from identifiers
+  -O=, --symOverride=  strings   {}       skip generating specified symbols
 ```
 
 __Implementation Details__
diff --git a/nimterop.nimble b/nimterop.nimble
index 8008b27..b20f6a7 100644
--- a/nimterop.nimble
+++ b/nimterop.nimble
@@ -18,9 +18,9 @@ proc execCmd(cmd: string) =
   echo "execCmd:" & cmd
   exec cmd
 
-proc execTest(test: string) =
-  execCmd "nim c -f -r " & test
-  execCmd "nim cpp -r " & test
+proc execTest(test: string, flags = "") =
+  execCmd "nim c -f " & flags & " -r " & test
+  execCmd "nim cpp " & flags & " -r " & test
 
 task buildToast, "build toast":
   execCmd("nim c -f nimterop/toast.nim")
@@ -35,11 +35,15 @@ task test, "Test":
   buildToastTask()
 
   execTest "tests/tast2.nim"
-  execCmd "nim c -f -d:HEADER -r tests/tast2.nim"
+  execTest "tests/tast2.nim", "-d:HEADER"
 
   execTest "tests/tnimterop_c.nim"
+  execTest "tests/tnimterop_c.nim", "-d:AST2"
+  execTest "tests/tnimterop_c.nim", "-d:HEADER -d:AST2"
+
   execCmd "nim cpp -f -r tests/tnimterop_cpp.nim"
   execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h"
+  execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h"
   execTest "tests/tpcre.nim"
 
   # Platform specific tests
diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim
index 7eb22a1..4c001f9 100644
--- a/nimterop/ast2.nim
+++ b/nimterop/ast2.nim
@@ -286,6 +286,24 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "",
       #  ),
       #  nkEmpty()
       # )
+      #
+      # type name* {.bycopy, importc: "abc".} =
+      #
+      # nkTypeDef(
+      #  nkPragmaExpr(
+      #   nkPostfix(
+      #    nkIdent("*"),
+      #    nkIdent(name)
+      #   ),
+      #   nkPragma(
+      #    nkIdent("bycopy"),
+      #    nkExprColonExpr(
+      #     nkIdent("importc"),
+      #     nkStrLit("abc")
+      #    )
+      #   )
+      #  )
+      # )
       var
         pragmas =
           if nimState.includeHeader:
@@ -329,6 +347,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "",
       #  nkIdent("*"),
       #  nkIdent(name)
       # )
+      #
+      # No pragmas here since proc pragmas are elsewhere in the AST
       result = ident
 
     nimState.identifierNodes[name] = result
@@ -465,8 +485,17 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn
           (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name)
           pident = nimState.getIdent(pname, pinfo, exported)
 
+          # Bitfield support - typedef struct { int field: 1; };
+          prident =
+            if node.len > start+1 and node[start+2].getName() == "bitfield_clause":
+              nimState.newPragmaExpr(node, pident, "bitsize",
+                newIntNode(nkIntLit, parseInt(nimState.getNodeVal(node[start+2].getAtom()))))
+            else:
+              pident
+
           count = node[start+1].getPtrCount()
-        result.add pident
+
+        result.add prident
         if count > 0:
           result.add nimState.newPtrTree(count, tident)
         else:
@@ -527,11 +556,11 @@ proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNo
   #    ..
   #   )
   #  ),
-  #  nkEmpty()
+  #  nkPragma(...)
   # )
   result = newNode(nkProcTy)
   result.add nimState.newFormalParams(name, node, rtyp)
-  result.add newNode(nkEmpty)
+  result.add nimState.newPragma(node, nimState.gState.convention)
 
 proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode =
   # Create nkRecList tree for specified object
@@ -602,6 +631,37 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname
     #   nkEmpty()
     #  )
     # )
+    #
+    # type
+    #   X* {.bycopy.} = object
+    #     field1*: cint
+    #
+    # nkTypeDef(
+    #  nkPragmaExpr(
+    #   nkPostfix(
+    #    nkIdent("*"),
+    #    nkIdent("X")
+    #   ),
+    #   nkPragma(
+    #    nkIdent("bycopy")
+    #   )
+    #  ),
+    #  nkEmpty(),
+    #  nkObjectTy(
+    #   nkEmpty(),
+    #   nkEmpty(),
+    #   nkRecList(
+    #    nkIdentDefs(
+    #     nkPostfix(
+    #      nkIdent("*"),
+    #      nkIdent("field1")
+    #     ),
+    #     nkIdent("cint"),
+    #     nkEmpty()
+    #    )
+    #   )
+    #  )
+    # )
     let
       name = typeDef.getIdentName()
       obj = newNode(nkObjectTy)
@@ -893,7 +953,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) =
     #     ),
     #     ...
     #    ),
-    #    nkEmpty()
+    #    nkPragma(...)
     #   )
     #  )
     # )
@@ -1069,7 +1129,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) =
 
             # First add struct as object
             decho("addType(): case 6")
-            nimState.addTypeObject(node[0], istype = true, union = union)
+            nimState.addTypeObject(node[0], union = union)
 
             if node.len > 1 and nimState.getNodeVal(node[1]) != "":
               # Add any additional names
@@ -1193,6 +1253,7 @@ proc addProc(nimState: NimState, node: TSNode) =
       let
         # Only need the ident tree, not nkTypeDef parent
         name = ident.getIdentName()
+        origname = nimState.getNodeVal(node[i].getAtom())
 
         # node[i] could have nested pointers
         tcount = node[i].getPtrCount()
@@ -1249,7 +1310,27 @@ proc addProc(nimState: NimState, node: TSNode) =
 
       # Proc with return type and params
       procDef.add nimState.newFormalParams(name, plist, retType)
-      procDef.add newNode(nkEmpty) # Pragmas
+
+      # Pragmas
+      let
+        prident =
+          if name != origname:
+            # Explicit {.importc: "origname".}
+            nimState.newPragma(node[i], "importc", newStrNode(nkStrLit, origname))
+          else:
+            # {.impnameC.} shortcut
+            nimState.newPragma(node[i], nimState.impShort & "C")
+
+      # Need {.convention.} and {.header.} if applicable
+      if name != origname:
+        if nimState.includeHeader():
+          # {.impnameHC.} shortcut
+          nimState.addPragma(node[i], prident, nimState.impShort & "HC")
+        else:
+          # {.convention.}
+          nimState.addPragma(node[i], prident, nimState.gState.convention)
+
+      procDef.add prident
       procDef.add newNode(nkEmpty)
       procDef.add newNode(nkEmpty)
 
@@ -1324,37 +1405,46 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) =
   # Create shortcut pragmas to reduce clutter
   var
     hdrPragma: PNode
+    hdrConvPragma: PNode
     impPragma = newNode(nkPragma)
-    impCPragma = newNode(nkPragma)
+    impConvPragma = newNode(nkPragma)
 
-  # {.importc.}
+  # {.pragma: impname, importc.}
   nimState.addPragma(root, impPragma, "pragma", nimState.getIdent(nimState.impShort))
   nimState.addPragma(root, impPragma, "importc")
 
   if nimState.includeHeader():
+    # Path to header const
     nimState.constSection.add nimState.newConstDef(
       root, fname = nimState.currentHeader, fval = '"' & fullpath & '"')
 
-    # {.header: "xxx".}
+    # {.pragma: impnameH, header: "xxx".} for types when name != origname
     hdrPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "H"))
     nimState.addPragma(root, hdrPragma, "header", nimState.getIdent(nimState.currentHeader))
 
+    # Add {.impnameH.} to {.impname.}
     nimState.addPragma(root, impPragma, nimState.impShort & "H")
 
-  # {.importc.} + {.cdecl.} for procs
-  nimState.addPragma(root, impCPragma, "pragma", nimState.getIdent(nimState.impShort & "C"))
-  nimState.addPragma(root, impCPragma, nimState.impShort)
-  nimState.addPragma(root, impCPragma, "cdecl")
+    # {.pragma: impnameHC, impnameH, convention.} for procs when name != origname
+    hdrConvPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "HC"))
+    nimState.addPragma(root, hdrConvPragma, nimState.impShort & "H")
+    nimState.addPragma(root, hdrConvPragma, nimState.gState.convention)
+
+  # {.pragma: impnameC, impname, convention.} for procs
+  nimState.addPragma(root, impConvPragma, "pragma", nimState.getIdent(nimState.impShort & "C"))
+  nimState.addPragma(root, impConvPragma, nimState.impShort)
+  nimState.addPragma(root, impConvPragma, nimState.gState.convention)
 
   if nimState.gState.dynlib.nBl:
     # {.dynlib.} for DLLs
-    nimState.addPragma(root, impCPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib))
+    nimState.addPragma(root, impConvPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib))
 
   # Add all pragma shortcuts to output
   if not hdrPragma.isNil:
     nimState.pragmaSection.add hdrPragma
+    nimState.pragmaSection.add hdrConvPragma
   nimState.pragmaSection.add impPragma
-  nimState.pragmaSection.add impCPragma
+  nimState.pragmaSection.add impConvPragma
 
 proc printNimHeader*(gState: State) =
   # Top level output with context info
diff --git a/nimterop/globals.nim b/nimterop/globals.nim
index 68f18b0..0e4827f 100644
--- a/nimterop/globals.nim
+++ b/nimterop/globals.nim
@@ -59,7 +59,7 @@ type
 
     debug*, includeHeader*, nocache*, nocomments*, past*, preprocess*, pnim*, recurse*: bool
 
-    code*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string
+    code*, convention*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string
 
     feature*: seq[Feature]
 
diff --git a/nimterop/toast.nim b/nimterop/toast.nim
index 323b4ec..a82a4ea 100644
--- a/nimterop/toast.nim
+++ b/nimterop/toast.nim
@@ -54,6 +54,7 @@ proc process(gState: State, path: string, astTable: AstTable) =
 # CLI processing with default values
 proc main(
     check = false,
+    convention = "cdecl",
     debug = false,
     defines: seq[string] = @[],
     dynlib: string = "",
@@ -79,6 +80,7 @@ proc main(
 
   # Setup global state with arguments
   var gState = State(
+    convention: convention,
     debug: debug,
     defines: defines,
     dynlib: dynlib,
@@ -189,6 +191,7 @@ when isMainModule:
   import cligen
   dispatch(main, help = {
     "check": "check generated wrapper with compiler",
+    "convention": "calling convention for wrapped procs - default: cdecl",
     "debug": "enable debug output",
     "defines": "definitions to pass to preprocessor",
     "dynlib": "import symbols from library in specified Nim string",
@@ -196,9 +199,9 @@ when isMainModule:
     "includeHeader": "add {.header.} pragma to wrapper",
     "includeDirs": "include directory to pass to preprocessor",
     "mode": "language parser: c or cpp",
-    "nim": "use a particular Nim executable (default: $PATH/nim)",
+    "nim": "use a particular Nim executable - default: $PATH/nim",
     "nocomments": "exclude top-level comments from output",
-    "output": "file to output content - default stdout",
+    "output": "file to output content - default: stdout",
     "past": "print AST output",
     "pgrammar": "print grammar",
     "pluginSourcePath": "nim file to build and load as a plugin",
@@ -212,6 +215,7 @@ when isMainModule:
     "symOverride": "skip generating specified symbols"
   }, short = {
     "check": 'k',
+    "convention": 'C',
     "debug": 'd',
     "defines": 'D',
     "dynlib": 'l',
diff --git a/tests/include/tast2.h b/tests/include/tast2.h
index db539c7..1518a85 100644
--- a/tests/include/tast2.h
+++ b/tests/include/tast2.h
@@ -1,3 +1,7 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define A 1
 #define B 1.0
 #define C 0x10
@@ -26,7 +30,7 @@ struct A4 {
 typedef char *A9p[3]; //, A9[4];
 typedef char *A10[3][6];
 typedef char *(*A11)[3];
-typedef struct A1 *A111[12];
+typedef struct A0 *A111[12];
 
 typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int));
 typedef int A13(int, int, void (*func)(void));
@@ -212,4 +216,8 @@ typedef enum VSPresetFormat {
 
 //struct A2 test_proc1(struct A0 a);
 
+#endif
+
+#ifdef __cplusplus
+}
 #endif
\ No newline at end of file
diff --git a/tests/tast2.nim b/tests/tast2.nim
index d77ae39..d98907f 100644
--- a/tests/tast2.nim
+++ b/tests/tast2.nim
@@ -11,7 +11,7 @@ const
 when defined(HEADER):
   cDefine("HEADER")
   const
-    flags = " -H -d"
+    flags = " -H"
     pHeader = @["header:" & path.replace("\\", "/")]
     pHeaderImp = @["importc"] & pHeader
 else:
@@ -183,21 +183,15 @@ a11 = addr a9p
 assert A111 is array[12, ptr A1]
 checkPragmas(A111, pHeaderImp)
 var a111: A111
-a111[11] = addr a1
+a111[11] = addr a0
 
-assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint
-checkPragmas(A12, pHeaderImp)
-when not defined(HEADER):
-  # Unclear why this fails
-  # request for member ‘ClE_0’ in something not a structure or union
-  var a12: A12
+assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint {.cdecl.}): ptr ptr cint {.cdecl.}
+checkPragmas(A12, pHeaderImp & "cdecl")
+var a12: A12
 
-assert A13 is proc(a1: cint, a2: cint, `func`: proc()): cint
-checkPragmas(A13, pHeaderImp)
-when not defined(HEADER):
-  # Unclear why this fails
-  # request for member ‘ClE_0’ in something not a structure or union
-  var a13: A13
+assert A13 is proc(a1: cint, a2: cint, `func`: proc() {.cdecl.}): cint {.cdecl.}
+checkPragmas(A13, pHeaderImp & "cdecl")
+var a13: A13
 
 assert A14 is object
 testFields(A14, "a1:cchar")
@@ -216,20 +210,16 @@ a15.a2[0] = addr a15i
 
 assert A16 is object
 testFields(A16, "f1:cchar")
-checkPragmas(A16, pHeaderImpBy)
-when not defined(HEADER):
-  # Similar to A2
-  var a16: A16
-  a16.f1 = 's'
+checkPragmas(A16, pHeaderBy, istype = false)
+var a16: A16
+a16.f1 = 's'
 
 assert A17 is object
 testFields(A17, "a1|a2:cstring|array[1, ptr cint]")
-checkPragmas(A17, pHeaderImpBy)
-when not defined(HEADER):
-  # Similar to A2
-  var a17: A17
-  a17.a1 = "hello".cstring
-  a17.a2[0] = addr a15i
+checkPragmas(A17, pHeaderBy, istype = false)
+var a17: A17
+a17.a1 = "hello".cstring
+a17.a2[0] = addr a15i
 
 assert A18 is A17
 checkPragmas(A18, pHeaderImp)
@@ -254,7 +244,7 @@ a19p = addr a19
 
 assert A20 is object
 testFields(A20, "a1:cchar")
-checkPragmas(A20, pHeaderImpBy)
+checkPragmas(A20, pHeaderBy, istype = false)
 var a20: A20
 a20.a1 = 'a'
 
@@ -270,7 +260,7 @@ a21p = addr a20
 
 assert A22 is object
 testFields(A22, "f1|f2:ptr ptr cint|array[123 + 132, ptr cint]")
-checkPragmas(A22, pHeaderImpBy)
+checkPragmas(A22, pHeaderBy, istype = false)
 var a22: A22
 a22.f1 = addr a15.a2[0]
 
@@ -282,7 +272,7 @@ u1.f1 = 5
 
 assert U2 is object
 assert sizeof(U2) == 256 * sizeof(cint)
-checkPragmas(U2, pHeaderImpBy & @["union"])
+checkPragmas(U2, pHeaderBy & @["union"], istype = false)
 var u2: U2
 u2.f1 = addr a15.a2[0]
 
diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim
index 732efbf..afd73e2 100644
--- a/tests/tnimterop_c.nim
+++ b/tests/tnimterop_c.nim
@@ -41,7 +41,17 @@ cOverride:
   proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.}
   proc weirdfunc2(mango: ptr ptr cchar): int {.importc.}
 
-cImport(cSearchPath("test.h"))
+# includeHeader
+const header =
+  when defined(HEADER): " -H"
+  else: ""
+
+# Test AST2
+const mode =
+  when defined(AST2): " -f:ast2"
+  else: ""
+
+cImport(cSearchPath("test.h"), flags = header & mode)
 
 check TEST_INT == 512
 check TEST_FLOAT == 5.12
@@ -65,7 +75,7 @@ var
   ct: CUSTTYPE
   cct: CCUSTTYPE
 
-  s0: STRUCT0
+  s0: ptr STRUCT0
   s1: STRUCT1
   s2: STRUCT2
   s3: STRUCT3