diff options
| -rw-r--r-- | nimterop.nimble | 2 | ||||
| -rw-r--r-- | nimterop/ast.nim | 12 | ||||
| -rw-r--r-- | nimterop/ast2.nim | 895 | ||||
| -rw-r--r-- | nimterop/compat.nim | 34 | ||||
| -rw-r--r-- | nimterop/getters.nim | 301 | ||||
| -rw-r--r-- | nimterop/globals.nim | 17 | ||||
| -rw-r--r-- | nimterop/grammar.nim | 24 | ||||
| -rw-r--r-- | nimterop/toast.nim | 62 | ||||
| -rw-r--r-- | tests/include/tast2.h | 42 | ||||
| -rw-r--r-- | tests/tast2.nim | 71 |
10 files changed, 1320 insertions, 140 deletions
diff --git a/nimterop.nimble b/nimterop.nimble index 24bbb4f..c33e78b 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -36,6 +36,8 @@ task docs, "Generate docs": task test, "Test": buildToastTask() + execTest "tests/tast2.nim" + execTest "tests/tnimterop_c.nim" execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 8ffb850..a61e65a 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -1,4 +1,4 @@ -import hashes, macros, os, sets, strformat, strutils, tables, times +import hashes, macros, os, sets, strformat, strutils, tables import regex @@ -166,16 +166,6 @@ proc searchAst(root: TSNode, astTable: AstTable, nimState: NimState) = if node == root: break -proc printNimHeader*(gState: State) = - gecho """# Generated at $1 -# Command line: -# $2 $3 - -{.hint[ConvFromXtoItselfNotNeeded]: off.} - -import nimterop/types -""" % [$now(), getAppFilename(), commandLineParams().join(" ")] - proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = var nimState = new(NimState) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim new file mode 100644 index 0000000..697262f --- /dev/null +++ b/nimterop/ast2.nim @@ -0,0 +1,895 @@ +import macros, os, sets, strutils, tables, times + +import regex + +import compiler/[ast, idents, options, renderer] + +import "."/treesitter/api + +import "."/[globals, getters] + +# Move to getters after ast2 becomes default + +proc getPtrType*(str: string): string = + result = case str: + of "cchar": + "cstring" + of "object": + "pointer" + else: + str + +proc getLit*(str: string): PNode = + # Used to convert #define literals into const + let + str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() + + if str.contains(re"^[\-]?[\d]+$"): # decimal + result = newIntNode(nkIntLit, parseInt(str)) + + elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float + result = newFloatNode(nkFloatLit, parseFloat(str)) + + # TODO - hex becomes int on render + elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + result = newIntNode(nkIntLit, parseHexInt(str)) + + elif str.contains(re"^'[[:ascii:]]'$"): # char + result = newNode(nkCharLit) + result.intVal = str[1].int64 + + elif str.contains(re"""^"[[:ascii:]]+"$"""): # char * + result = newStrNode(nkStrLit, str[1 .. ^2]) + + else: + result = newNode(nkNilLit) + +proc addConst(nimState: NimState, node: TSNode) = + # #define X Y + # + # (preproc_def + # (identifier) + # (preproc_arg) + # ) + decho("addConst()") + nimState.printDebug(node) + + if node[0].getName() == "identifier" and + node[1].getName() == "preproc_arg": + let + constDef = newNode(nkConstDef) + + # node[0] = identifier = const name + (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) + # TODO - check blank and override + ident = nimState.getIdent(name, info) + + # node[1] = preproc_arg = value + val = nimState.getNodeVal(node[1]).getLit() + + # If supported literal + if val.kind != nkNilLit: + # const X* = Y + # + # nkConstDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkXLit(Y) + # ) + constDef.add ident + constDef.add newNode(nkEmpty) + constDef.add val + + # nkConstSection.add + nimState.constSection.add constDef + + nimState.printDebug(constDef) + +proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = + # Create nkTypeDef PNode with first ident + # + # If `override`, use it instead of node.getAtom() for name + let + (name, info) = nimState.getNameInfo(node.getAtom(), nskType) + # TODO - check blank and override + ident = + if override.len != 0: + nimState.getIdent(override, info) + else: + nimState.getIdent(name, info) + + # type name* = + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkEmpty() + # ) + result = newNode(nkTypeDef) + result.add ident + result.add newNode(nkEmpty) + +proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = + # Create nkPtrTy tree depending on count + # + # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring + var + count = count + chng = false + if typ.kind == nkIdent: + let + tname = typ.ident.s + ptname = getPtrType(tname) + if tname != ptname: + # If Nim type available, use that ident + result = nimState.getIdent(ptname, typ.info, exported = false) + chng = true + # One ptr reduced + count -= 1 + if count > 0: + # Nested nkPtrTy(typ) depending on count + # + # [ptr ...] typ + # + # nkPtrTy( + # nkPtrTy( + # typ + # ) + # ) + result = newNode(nkPtrTy) + var + parent = result + child: PNode + for i in 1 ..< count: + child = newNode(nkPtrTy) + parent.add child + parent = child + parent.add typ + elif not chng: + # Either no ptr, or none left after Nim type adjustment + result = typ + +proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = + # Create nkBracketExpr tree depending on input + let + (_, info) = nimState.getNameInfo(node, nskType) + ident = nimState.getIdent("array", info, exported = false) + + # array[size, typ] + # + # nkBracketExpr( + # nkIdent("array"), + # size, + # typ + # ) + result = newNode(nkBracketExpr) + result.add ident + result.add size + result.add typ + +proc getTypeArray(nimState: NimState, node: TSNode): PNode +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode + +proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter or object field + # + # For proc, param should not be exported + # + # pname: [ptr ..] typ + # + # nkIdentDefs( + # nkIdent(pname), + # typ, + # nkEmpty() + # ) + # + # For object, field should be exported + # + # pname*: [ptr ..] typ + # + # nkIdentDefs( + # nkPostfix( + # nkIdent("*"), + # nkIdent(pname) + # ), + # typ, + # nkEmpty() + # ) + result = newNode(nkIdentDefs) + + let + start = getStartAtom(node) + + # node[start] - param type + (tname, tinfo) = nimState.getNameInfo(node[start], nskType) + tident = nimState.getIdent(tname, tinfo, exported = false) + + if start == node.len - 1: + # Only for proc with no named param - create a param name based on offset + # + # int func(char, int); + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported) + result.add pident + result.add tident + result.add newNode(nkEmpty) + else: + let + fdecl = node[start+1].anyChildInTree("function_declarator") + adecl = node[start+1].anyChildInTree("array_declarator") + abst = node[start+1].getName() == "abstract_pointer_declarator" + if fdecl.isNil and adecl.isNil: + if abst: + # Only for proc with no named param with pointer type + # Create a param name based on offset + # + # int func(char *, int **); + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported) + acount = node[start+1].getXCount("abstract_pointer_declarator") + result.add pident + result.add nimState.newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + # Named param, simple type + let + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + + count = node[start+1].getPtrCount() + result.add pident + if count > 0: + result.add nimState.newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + # Named param, function pointer + let + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + result.add pident + result.add nimState.getTypeProc(name, node) + result.add newNode(nkEmpty) + elif not adecl.isNil: + # Named param, array type + let + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + result.add pident + result.add nimState.getTypeArray(node) + result.add newNode(nkEmpty) + else: + result = nil + +proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + # Create nkProcTy tree for specified proc type + let + fparam = newNode(nkFormalParams) + + # Add return type + fparam.add rtyp + + if not node.isNil: + for i in 0 ..< node.len: + # Add nkIdentDefs for each param + let + param = nimState.newIdentDefs(name, node[i], i, exported = false) + if not param.isNil: + fparam.add param + + # proc(pname: ptyp ..): rtyp + # + # nkProcTy( + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ), + # nkEmpty() + # ) + result = newNode(nkProcTy) + result.add fparam + result.add newNode(nkEmpty) + +proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = + # Create nkRecList tree for specified object + if not node.isNil: + # fname*: ftyp + # .. + # + # nkRecList( + # nkIdentDefs( # multiple depending on fields + # .. + # ) + # ) + result = newNode(nkRecList) + + for i in 0 ..< node.len: + # Add nkIdentDefs for each field + let + field = nimState.newIdentDefs(name, node[i], i, exported = true) + if not field.isNil: + result.add field + +proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "") = + # Add a type of object + # + # If `override` is set, use it as the name + # If `duplicate` is set, don't add the same name + decho("addTypeObject()") + let + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node, override) + name = $typeDef[0][1] + + if name != duplicate: + # type X* = object + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkEmpty() + # ) + # ) + let + obj = newNode(nkObjectTy) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + + let + fdlist = node.anyChildInTree("field_declaration_list") + if not fdlist.isNil and fdlist.len > 0: + # Add fields to object if present + obj.add nimState.newRecListTree(name, fdlist) + else: + obj.add newNode(nkEmpty) + + typeDef.add obj + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = "") = + # Add a type of a specified type + # + # If `toverride` is set, use it as the type name + # If `duplicate` is set, don't add the same name + decho("addTypeTyped()") + let + start = getStartAtom(node) + for i in start+1 ..< node.len: + # Add a type of a specific type + let + # node[i] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[i]) + + # node[start] = identifier = type name + (tname0, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) + + # Override type name + tname = if toverride.len != 0: toverride else: tname0 + + ident = nimState.getIdent(tname, tinfo, exported = false) + + # node[i] could have nested pointers + count = node[i].getPtrCount() + + # Skip typedef X X; + if $typeDef[0][1] != tname: + if count > 0: + # If pointers + typeDef.add nimState.newPtrTree(count, ident) + else: + typeDef.add ident + + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + else: + nimState.addTypeObject(node, duplicate = duplicate) + +proc getTypeArray(nimState: NimState, node: TSNode): PNode = + # Create array type tree + let + start = getStartAtom(node) + + # node[start] = identifier = type name + (name, info) = nimState.getNameInfo(node[start].getAtom(), nskType) + ident = nimState.getIdent(name, info, exported = false) + + # Top-most array declarator + adecl = node[start+1].firstChildInTree("array_declarator") + + # node[start+1] could have nested arrays + acount = adecl.getArrayCount() + innermost = adecl.mostNestedChildInTree() + + # node[start+1] could have nested pointers - type + tcount = node[start+1].getPtrCount() + + # Name could be nested pointer to array + # + # (.. + # (array_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = innermost[0].getAtom().tsNodeParent().getPtrCount(reverse = true) + + result = ident + var + cnode = adecl + + if tcount > 0: + # If pointers + result = nimState.newPtrTree(tcount, result) + + for i in 0 ..< acount: + let + size = nimState.getNodeVal(cnode[1]).getLit() + if size.kind != nkNilLit: + result = nimState.newArrayTree(cnode, result, size) + cnode = cnode[0] + + if ncount > 0: + result = nimState.newPtrTree(ncount, result) + +proc addTypeArray(nimState: NimState, node: TSNode) = + # Add a type of array type + decho("addTypeArray()") + let + # node[1] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + + typ = nimState.getTypeArray(node) + + typeDef.add typ + + # type X* = [ptr] array[x, [ptr] Y] + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkBracketExpr( + # nkIdent("array") + # nkXLit(x), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = + # Create proc type tree + let + # node[0] = identifier = return type name + (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + + # Parameter list + plist = node[1].anyChildInTree("parameter_list") + + # node[1] could have nested pointers + tcount = node[1].getPtrCount() + + # Name could be nested pointer to function + # + # (.. + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = node[1].getAtom().tsNodeParent().getPtrCount(reverse = true) + + # Return type + var + retType = nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = nimState.newPtrTree(tcount, retType) + + # Proc with return type and params + result = nimState.newProcTree(name, plist, retType) + if ncount > 1: + result = nimState.newPtrTree(ncount-1, result) + +proc addTypeProc(nimState: NimState, node: TSNode) = + # Add a type of proc type + decho("addTypeProc()") + let + # node[1] = identifier = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + name = $typeDef[0][1] + + procTy = nimState.getTypeProc(name, node) + + typeDef.add procTy + + # type X* = proc(a1: Y, a2: Z): P + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkProcTy( + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkEmpty() + # ) + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addType(nimState: NimState, node: TSNode) = + decho("addType()") + nimState.printDebug(node) + + if node.getName() == "struct_specifier": + # struct X; + # + # (struct_specifier + # (type_identifier) + # ) + # + # struct X {}; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # + # struct X { char *a1; }; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list + # (field_declaration + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (field_identifier) + # ) + # (field_declaration ...) + # ) + decho("addType(): case 1") + nimState.addTypeObject(node) + elif node.getName() == "type_definition": + if node.len >= 2: + let + fdlist = node[0].anyChildInTree("field_declaration_list") + if (fdlist.isNil or (not fdlist.isNil and fdlist.len == 0)) and + nimState.getNodeVal(node[1]) == "": + # typedef struct X; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_definition = "") + # ) + # + # typedef struct X {}; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # (type_definition = "") + # ) + decho("addType(): case 2") + nimState.addTypeObject(node[0]) + else: + let + fdecl = node[1].anyChildInTree("function_declarator") + adecl = node[1].anyChildInTree("array_declarator") + if fdlist.isNil(): + if adecl.isNil and fdecl.isNil: + # typedef X Y; + # typedef X *Y; + # typedef struct X Y; + # typedef struct X *Y; + # + # (type_definition + # (type_qualifier?) + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + decho("addType(): case 3") + nimState.addTypeTyped(node) + elif not fdecl.isNil: + # typedef X (*Y)(a1, a2, a3); + # typedef X *(*Y)(a1, a2, a3); + # typedef struct X (*Y)(a1, a2, a3); + # typedef struct X *(*Y)(a1, a2, a3); + # + # (type_definition + # (type_qualifier?) + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator + # (type_identifer) + # ) + # ) + # (parameter_list + # (parameter_declaration + # (struct_specifier|type_identifier|primitive_type|array_declarator|function_declarator) + # (identifier - optional) + # ) + # ) + # ) + # ) + # ) + decho("addType(): case 4") + nimState.addTypeProc(node) + elif not adecl.isNil: + # typedef struct X Y[a][..]; + # typedef struct X *Y[a][..]; + # typedef struct X *(*Y)[a][..]; + # + # (type_definition + # (type_qualifier?) + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + decho("addType(): case 5") + nimState.addTypeArray(node) + else: + if node.firstChildInTree("field_declaration_list").isNil: + # typedef struct X { .. } Y, *Z; + # + # (type_definition + # (struct_specifier + # (type_identifier) - named struct <==== + # (field_declaration_list + # (field_declaration - optional, multiple + # (type_identifier|primitive_type|) + # (function_declarator|array_declarator + # .. + # ) + # + # (field_identifier) + # ) + # ) + # ) + # + # (type_identifier) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + + # First add struct as object + decho("addType(): case 6") + nimState.addTypeObject(node[0]) + + if node.len > 1 and nimState.getNodeVal(node[1]) != "": + # Add any additional names + nimState.addTypeTyped(node, duplicate = nimState.getNodeVal(node[0].getAtom())) + else: + # Same as above except unnamed struct + # + # typedef struct { .. } Y, *Z; + + # Get any name that isn't a pointer + decho("addType(): case 7") + let + name = block: + var + name = "" + for i in 1 ..< node.len: + if node[i].getName() == "type_identifier": + name = nimState.getNodeVal(node[i].getAtom()) + + name + + # Now add struct as object with specified name + nimState.addTypeObject(node[0], override = name) + + if name.len != 0: + # Add any additional names except duplicate + nimState.addTypeTyped(node, toverride = name, duplicate = name) + +proc addEnum(nimState: NimState, node: TSNode) = + decho("addEnum()") + nimState.printDebug(node) + +proc addProc(nimState: NimState, node: TSNode) = + decho("addProc()") + nimState.printDebug(node) + +proc processNode(nimState: NimState, node: TSNode): bool = + result = true + + case node.getName() + of "preproc_def": + nimState.addConst(node) + of "type_definition": + if not node.firstChildInTree("enum_specifier").isNil(): + nimState.addEnum(node) + else: + nimState.addType(node) + of "struct_specifier": + nimState.addType(node) + of "enum_specifier": + nimState.addEnum(node) + of "declaration": + nimState.addProc(node) + else: + # Unknown + result = false + +proc searchTree(nimState: NimState, root: TSNode) = + # Search AST generated by tree-sitter for recognized elements + var + node = root + nextnode: TSNode + depth = 0 + processed = false + + while true: + if not node.isNil() and depth > -1: + processed = nimState.processNode(node) + else: + break + + if not processed and node.len() != 0: + nextnode = node[0] + depth += 1 + else: + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.isNil(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if node == root: + break + if not node.tsNodeNextNamedSibling().isNil(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc printNimHeader*(gState: State) = + gecho """# Generated at $1 +# Command line: +# $2 $3 + +{.hint[ConvFromXtoItselfNotNeeded]: off.} + +import nimterop/types +""" % [$now(), getAppFilename(), commandLineParams().join(" ")] + +proc printNim*(gState: State, fullpath: string, root: TSNode) = + var + nimState = new(NimState) + #fp = fullpath.replace("\\", "/") + + nimState.identifiers = newTable[string, string]() + + nimState.gState = gState + nimState.currentHeader = getCurrentHeader(fullpath) + nimState.sourceFile = fullpath + + # Nim compiler objects + nimState.identCache = newIdentCache() + nimState.config = newConfigRef() + + nimState.constSection = newNode(nkConstSection) + nimState.enumSection = newNode(nkStmtList) + nimState.procSection = newNode(nkStmtList) + nimState.typeSection = newNode(nkTypeSection) + + nimState.searchTree(root) + + var + tree = newNode(nkStmtList) + tree.add nimState.enumSection + tree.add nimState.constSection + tree.add nimState.typeSection + tree.add nimState.procSection + + gecho tree.renderTree() diff --git a/nimterop/compat.nim b/nimterop/compat.nim deleted file mode 100644 index 7e2bc98..0000000 --- a/nimterop/compat.nim +++ /dev/null @@ -1,34 +0,0 @@ -#[ -module for backward compatibility -put everything that requires `when (NimMajor, NimMinor, NimPatch)` here -]# - -import os - -when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): - proc myNormalizedPath*(path: string): string = path.normalizedPath() - - export relativePath - -else: - import std/[ospaths,strutils] - - proc myNormalizedPath*(path: string): string = - result = path.normalizedPath() - when defined(windows): - result = result.strip(trailing = false, chars = {'\\'}) - - proc relativePath*(file, base: string): string = - ## naive version of `os.relativePath` ; remove after nim >= 0.19.9 - runnableExamples: - import ospaths, unittest - check: - "/foo/bar/baz/log.txt".unixToNativePath.relativePath("/foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath - "foo/bar/baz/log.txt".unixToNativePath.relativePath("foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath - var base = base.myNormalizedPath - var file = file.myNormalizedPath - if not base.endsWith DirSep: base.add DirSep - doAssert file.startsWith base - result = file[base.len .. ^1] - - proc getCurrentCompilerExe*(): string = "nim" diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 3d77e99..4cd8c04 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,6 +2,8 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex +import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] + import "."/[build, globals, plugin, treesitter/api] const gReserved = """ @@ -26,6 +28,8 @@ when while xor yield""".split(Whitespace).toHashSet() +# Types related + const gTypeMap* = { # char "char": "cchar", @@ -91,6 +95,8 @@ proc getType*(str: string): string = if gTypeMap.hasKey(result): result = gTypeMap[result] +# Identifier related + proc checkIdentifier(name, kind, parent, origName: string) = let parentStr = if parent.nBl: parent & ":" else: "" @@ -165,6 +171,8 @@ proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool nimState.identifiers[nimName] = name result = true +# Overrides related + proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = doAssert name.nBl, "Blank identifier error" @@ -189,34 +197,121 @@ proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = for i in nimState.gState.onSymbolOverrideFinal(typ): result &= "\n" & nimState.getOverride(i, kind) -proc getPtrType*(str: string): string = - result = case str: - of "ptr cchar": - "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": - "pointer" - of "ptr ptr object": - "ptr pointer" - else: - str +# TSNode shortcuts -proc getLit*(str: string): string = - # Used to convert #define literals into const - let - str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() +proc isNil*(node: TSNode): bool = + node.tsNodeIsNull() + +proc len*(node: TSNode): int = + if not node.isNil: + result = node.tsNodeNamedChildCount().int + +proc `[]`*(node: TSNode, i: SomeInteger): TSNode = + if i < node.len(): + result = node.tsNodeNamedChild(i.uint32) - if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal - str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal - str.contains(re"^'[[:ascii:]]'$") or # char - str.contains(re"""^"[[:ascii:]]+"$"""): # char * - return str +proc getName*(node: TSNode): string {.inline.} = + if not node.isNil: + return $node.tsNodeType() + +proc getNodeVal*(gState: State, node: TSNode): string = + if not node.isNil: + return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() proc getNodeVal*(nimState: NimState, node: TSNode): string = - return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + nimState.gState.getNodeVal(node) + +proc getAtom*(node: TSNode): TSNode = + if not node.isNil: + # Get child node which is topmost atom + if node.getName() in gAtoms: + return node + elif node.len() != 0: + return node[0].getAtom() + +proc getStartAtom*(node: TSNode): int = + if not node.isNil: + # Skip const, volatile and other type qualifiers + for i in 0 .. node.len - 1: + if node[i].getAtom().getName() notin gAtoms: + result += 1 + else: + break + +proc getXCount*(node: TSNode, ntype: string, reverse = false): int = + if not node.isNil: + # Get number of ntype nodes nested in tree + var + cnode = node + while ntype in cnode.getName(): + result += 1 + if reverse: + cnode = cnode.tsNodeParent() + else: + if cnode.len() != 0: + cnode = cnode[0] + else: + break + +proc getPtrCount*(node: TSNode, reverse = false): int = + node.getXCount("pointer_declarator") + +proc getArrayCount*(node: TSNode, reverse = false): int = + node.getXCount("array_declarator") + +proc getDeclarator*(node: TSNode): TSNode = + if not node.isNil: + # Return if child is a function or array declarator + if node.getName() in ["function_declarator", "array_declarator"]: + return node + elif node.len() != 0: + return node[0].getDeclarator() + +proc firstChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type in tree - first children + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + cnode = cnode[0] + +proc anyChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type anywhere in tree - depth first + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + for i in 0 ..< cnode.len: + let + ccnode = cnode[i].anyChildInTree(ntype) + if not ccnode.isNil(): + return ccnode + if cnode != node: + cnode = cnode.tsNodeNextNamedSibling() + else: + break + +proc mostNestedChildInTree*(node: TSNode): TSNode = + # Search for the most nested child of node's type in tree + var + cnode = node + ntype = cnode.getName() + while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype: + cnode = cnode[0] + result = cnode + +proc inChildren*(node: TSNode, ntype: string): bool = + # Search for node type in immediate children + result = false + for i in 0 ..< node.len(): + if (node[i]).getName() == ntype: + result = true + break proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node result.line = 1 result.col = 1 for i in 0 .. node.tsNodeStartByte().int-1: @@ -225,6 +320,150 @@ proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = result.line += 1 result.col += 1 +proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = + for i in 0 ..< node.len(): + if node.getName() != "comment": + result += 1 + +proc getPxName*(node: TSNode, offset: int): string = + # Get the xth (grand)parent of the node + var + np = node + count = 0 + + while not np.isNil() and count < offset: + np = np.tsNodeParent() + count += 1 + + if count == offset and not np.isNil(): + return np.getName() + +proc printLisp*(gState: State, root: TSNode): string = + var + node = root + nextnode: TSNode + depth = 0 + + while true: + if not node.isNil() and depth > -1: + if gState.pretty: + result &= spaces(depth) + let + (line, col) = gState.getLineCol(node) + result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + let + val = gState.getNodeVal(node) + if "\n" notin val and " " notin val: + result &= &" \"{val}\"" + else: + break + + if node.tsNodeNamedChildCount() != 0: + if gState.pretty: + result &= "\n" + nextnode = node.tsNodeNamedChild(0) + depth += 1 + else: + if gState.pretty: + result &= ")\n" + else: + result &= ")" + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.isNil(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if gState.pretty: + result &= spaces(depth) & ")\n" + else: + result &= ")" + if node == root: + break + if not node.tsNodeNextNamedSibling().isNil(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +proc printTree*(nimState: NimState, pnode: PNode, offset = "") = + if nimState.gState.debug and pnode.kind != nkNone: + stdout.write "\n# " & offset & $pnode.kind & "(" + case pnode.kind + of nkCharLit: + stdout.write "'" & pnode.intVal.char & "')" + of nkIntLit..nkUInt64Lit: + stdout.write $pnode.intVal & ")" + of nkFloatLit..nkFloat128Lit: + stdout.write $pnode.floatVal & ")" + of nkStrLit..nkTripleStrLit: + stdout.write "\"" & pnode.strVal & "\")" + of nkSym: + stdout.write $pnode.sym & ")" + of nkIdent: + stdout.write "\"" & $pnode.ident.s & "\")" + else: + if pnode.sons.len != 0: + for i in 0 ..< pnode.sons.len: + nimState.printTree(pnode.sons[i], offset & " ") + if i != pnode.sons.len - 1: + stdout.write "," + stdout.write "\n# " & offset & ")" + else: + stdout.write ")" + if offset.len == 0: + necho "" + +proc printDebug*(nimState: NimState, node: TSNode) = + if nimState.gState.debug: + necho ("Input => " & nimState.getNodeVal(node)).getCommented() & "\n" & + nimState.gState.printLisp(node).getCommented() + +proc printDebug*(nimState: NimState, pnode: PNode) = + if nimState.gState.debug: + necho ("Output => " & $pnode).getCommented() + nimState.printTree(pnode) + +# Compiler shortcuts + +proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = + # Get Nim equivalent line:col info from node + let + (line, col) = nimState.gState.getLineCol(node) + + result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col) + +proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode = + # Get ident PNode for name + info + let + exp = getIdent(nimState.identCache, "*") + ident = getIdent(nimState.identCache, name) + + if exported: + result = newNode(nkPostfix) + result.add newIdentNode(exp, info) + result.add newIdentNode(ident, info) + else: + result = newIdentNode(ident, info) + +proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): + tuple[name: string, info: TLineInfo] = + # Shortcut to get identifier name and info (node value and line:col) + let + name = nimState.getNodeVal(node) + result.name = nimState.getIdentifier(name, kind, parent) + if kind == nskType: + result.name = result.name.getType() + result.info = nimState.getLineInfo(node) + proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) @@ -328,12 +567,6 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] -proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - if $node.tsNodeType() != "comment": - result += 1 - proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: for i in 0 .. node.tsNodeNamedChildCount()-1: @@ -364,18 +597,6 @@ proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = if ast.children.len == 1 and ast.children[0].name == ".": return ast.children[0] -proc getPxName*(node: TSNode, offset: int): string = - var - np = node - count = 0 - - while not np.tsNodeIsNull() and count < offset: - np = np.tsNodeParent() - count += 1 - - if count == offset and not np.tsNodeIsNull(): - return $np.tsNodeType() - proc getNimExpression*(nimState: NimState, expr: string): string = var clean = expr.multiReplace([("\n", " "), ("\r", "")]) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 9ee9de1..b234c1f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -5,6 +5,8 @@ import regex import "."/plugin when not declared(CIMPORT): + import compiler/[ast, idents, options] + import "."/treesitter/api const @@ -69,7 +71,16 @@ type NimState {.used.} = ref object identifiers*: TableRef[string, string] - commentStr*, constStr*, debugStr*, enumStr*, procStr*, skipStr*, typeStr*: string + # Legacy ast fields, remove when ast2 becomes default + constStr*, enumStr*, procStr*, typeStr*: string + + commentStr*, debugStr*, skipStr*: string + + # Nim compiler objects + when not declared(CIMPORT): + constSection*, enumSection*, procSection*, typeSection*: PNode + identCache*: IdentCache + config*: ConfigRef gState*: State @@ -110,3 +121,7 @@ when not declared(CIMPORT): template necho*(args: string) {.dirty.} = let gState = nimState.gState gecho args + + template decho*(str: untyped): untyped = + if nimState.gState.debug: + necho str.getCommented()
\ No newline at end of file diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index ba0e9fc..c76844c 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -7,6 +7,30 @@ import "."/[getters, globals, lisp, treesitter/api] type Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, nimState: NimState) {.nimcall.}]] +proc getPtrType(str: string): string = + result = case str: + of "ptr cchar": + "cstring" + of "ptr ptr cchar": + "ptr cstring" + of "ptr object": + "pointer" + of "ptr ptr object": + "ptr pointer" + else: + str + +proc getLit(str: string): string = + # Used to convert #define literals into const + let + str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() + + if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal + str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal + str.contains(re"^'[[:ascii:]]'$") or # char + str.contains(re"""^"[[:ascii:]]+"$"""): # char * + return str + proc initGrammar(): Grammar = # #define X Y result.add((""" diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 7dda1dc..a4415d2 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,56 +2,7 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast, globals, getters, grammar] - -proc printLisp(gState: State, root: TSNode) = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.tsNodeIsNull() and depth > -1: - if gState.pretty: - stdout.write spaces(depth) - let - (line, col) = gState.getLineCol(node) - stdout.write &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" - else: - break - - if node.tsNodeNamedChildCount() != 0: - if gState.pretty: - gecho "" - nextnode = node.tsNodeNamedChild(0) - depth += 1 - else: - if gState.pretty: - gecho ")" - else: - stdout.write ")" - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.tsNodeIsNull(): - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - if gState.pretty: - gecho spaces(depth) & ")" - else: - stdout.write ")" - if node == root: - break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break +import "."/[ast, ast2, globals, getters, grammar] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" @@ -91,9 +42,12 @@ proc process(gState: State, path: string, astTable: AstTable) = tree.tsTreeDelete() if gState.past: - gState.printLisp(root) + gecho gState.printLisp(root) elif gState.pnim: - gState.printNim(path, root, astTable) + if Feature.ast2 in gState.feature: + ast2.printNim(gState, path, root) + else: + ast.printNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code @@ -221,9 +175,9 @@ proc main( # Rerun nim check on stubbed wrapper (check, err) = execCmdEx(&"{gState.nim} check {outputFile}") - doAssert err == 0, data & "# Nim check with stub failed:\n\n" & check + doAssert err == 0, data & "\n# Nim check with stub failed:\n\n" & check else: - doAssert err == 0, outputFile.readFile() & "# Nim check failed:\n\n" & check + doAssert err == 0, outputFile.readFile() & "\n# Nim check failed:\n\n" & check # Print wrapper if temporarily redirected to file if check and output.len == 0: diff --git a/tests/include/tast2.h b/tests/include/tast2.h new file mode 100644 index 0000000..1a2789b --- /dev/null +++ b/tests/include/tast2.h @@ -0,0 +1,42 @@ + +#define A 1 +#define B 1.0 +#define C 0x10 +#define D "hello" +#define E 'c' + +struct A0; +struct A1 {}; +typedef struct A2; +typedef struct A3 {}; +typedef struct A4 A4, *A4p; +typedef const int A5; +typedef int *A6; +typedef A0 **A7; +typedef void *A8; + +typedef char *A9[3]; +typedef char *A10[3][6]; +typedef char *(*A11)[3]; + +typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int A13(int, int); + +struct A14 { volatile char a1; }; +struct A15 { char *a1; const int *a2[1]; }; + +typedef struct A16 { char f1; }; +typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p; +typedef struct { char *a1; int *a2[1]; } A19, *A19p; + +typedef struct A20 { char a1; } A20, A21, *A21p; + +//Expression +//typedef struct A21 { int **f1; int abc[123+132]; } A21; + +//Unions +//union UNION1 {int f1; }; +//typedef union UNION2 { int **f1; int abc[123+132]; } UNION2; + +// Anonymous +//typedef struct { char a1; }; diff --git a/tests/tast2.nim b/tests/tast2.nim new file mode 100644 index 0000000..ffdcd19 --- /dev/null +++ b/tests/tast2.nim @@ -0,0 +1,71 @@ +import tables + +import nimterop/[cimport] + +static: + cDebug() + +cImport("include/tast2.h", flags="-d -f:ast2") + +proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = + var + obj: t + count = 0 + for name, value in obj.fieldPairs(): + count += 1 + assert name in fields, $t & "." & name & " invalid" + assert $fields[name] == $typeof(value), + "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value) + assert count == fields.len, "Failed for " & $t + +assert A == 1 +assert B == 1.0 +assert C == 0x10 +assert D == "hello" +assert E == 'c' + +assert A0 is object +testFields(A0) +assert A1 is object +testFields(A1) +assert A2 is object +testFields(A2) +assert A3 is object +testFields(A3) +assert A4 is object +testFields(A4) +assert A4p is ptr A4 +assert A5 is cint +assert A6 is ptr cint +assert A7 is ptr ptr A0 +assert A8 is pointer + +assert A9 is array[3, cstring] +assert A10 is array[3, array[6, cstring]] +assert A11 is ptr array[3, cstring] + +assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint +assert A13 is proc(a1: cint, a2: cint): cint + +assert A14 is object +testFields(A14, {"a1": "cchar"}.toTable()) + +assert A15 is object +testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) + +assert A16 is object +testFields(A16, {"f1": "cchar"}.toTable()) + +assert A17 is object +testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A18 is A17 +assert A18p is ptr A17 + +assert A19 is object +testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A19p is ptr A19 + +assert A20 is object +testFields(A20, {"a1": "cchar"}.toTable()) +assert A21 is A20 +assert A21p is ptr A20 |