From 173e6d625c3ea96a95580303fedd011684635cc4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:18:04 -0600 Subject: Add string and char support Update some comments Rename exprparser main proc Don't export parse procs Add missing utils module Try to fix array type test Try fix cast test error Disable cast test for now Revert back comment test. Have to figure out how to test without vm --- nimterop/ast2.nim | 6 +-- nimterop/exprparser.nim | 101 ++++++++++++++++++++++++++++++++++++++++-------- nimterop/utils.nim | 18 +++++++++ tests/include/tast2.h | 6 +++ tests/tast2.nim | 12 ++++-- 5 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 nimterop/utils.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8c5da8c..08ea2d6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -18,7 +18,7 @@ proc getPtrType*(str: string): string = str proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.codeToNode(str) + result = nimState.parseCExpression(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -164,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - var + let colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index d27f23b..183bb18 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros +import strformat, strutils, macros, sets import regex @@ -12,11 +12,12 @@ type ExprParser* = ref object state*: NimState code*: string + name*: string ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string): ExprParser = - ExprParser(state: state, code: code) +proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = + ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: @@ -38,6 +39,8 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) + if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: + ident = ident & "." & exprParser.name if ident != "": result = exprParser.state.getIdent(ident) @@ -70,6 +73,58 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() +proc parseChar(charStr: string): uint8 {.inline.} = + ## Parses a character literal out of a string. This is needed + ## because treesitter gives unescaped characters when parsing + ## strings. + if charStr.len == 1: + return charStr[0].uint8 + + # Handle octal, hex, unicode? + if charStr.startsWith("\\x"): + result = parseHexInt(charStr.replace("\\x", "0x")).uint8 + elif charStr.len == 4: # Octal + result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8 + + if result == 0: + case charStr + of "\\0": + result = ord('\0') + of "\\a": + result = 0x07 + of "\\b": + result = 0x08 + of "\\e": + result = 0x1B + of "\\f": + result = 0x0C + of "\\n": + result = '\n'.uint8 + of "\\r": + result = 0x0D + of "\\t": + result = 0x09 + of "\\v": + result = 0x0B + of "\\\\": + result = 0x5C + of "\\'": + result = '\''.uint8 + of "\\\"": + result = '\"'.uint8 + of "\\?": + result = 0x3F + else: + discard + + if result > uint8.high: + result = uint8.high + +proc getCharLit(charStr: string): PNode {.inline.} = + ## Convert a character string into a proper Nim char lit node + result = newNode(nkCharLit) + result.intVal = parseChar(charStr).int64 + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) @@ -117,7 +172,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result.intVal = parseInt(number) -proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -141,17 +196,29 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkCharLit) - result.intVal = node.val[1].int64 +proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = + let val = node.val + result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = - let nodeVal = node.val - result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) +proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = + let + nodeVal = node.val + strVal = nodeVal[1 ..< nodeVal.len - 1] + + const + str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + reg = re(str) + + # Convert the c string escape sequences/etc to Nim chars + var nimStr = newStringOfCap(nodeVal.len) + for m in strVal.findAll(reg): + nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) + + result = newStrNode(nkStrLit, nimStr) -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -185,18 +252,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -415,13 +482,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): techo "NODE RESULT: ", result -proc codeToNode*(state: NimState, code: string): PNode = +proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol var tnode: PNode = nil - let exprParser = newExprParser(state, code) + let exprParser = newExprParser(state, code, name) try: withCodeAst(exprParser): result = exprParser.processTSNode(root, tnode) diff --git a/nimterop/utils.nim b/nimterop/utils.nim new file mode 100644 index 0000000..025256a --- /dev/null +++ b/nimterop/utils.nim @@ -0,0 +1,18 @@ +import compiler/[ast, lineinfos, msgs, options, parser, renderer] + +import "."/[globals, getters] + +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting for errors + if msg < warnMin: + raise newException(Exception, msgKindToString(msg)) + +proc parseString*(nimState: NimState, str: string): PNode = + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 7e15ac0..3c6148a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,6 +23,12 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) +#define CASTEXPR (int) 34 + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) diff --git a/tests/tast2.nim b/tests/tast2.nim index d65e34a..d00552c 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -93,11 +93,11 @@ macro testFields(t: typed, fields: static[string] = "") = for i in 0 ..< rl.len: let name = ($rl[i][0]).strip(chars = {'*'}) - typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "") + typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "").replace("typeof", "type") n = names.find(name) assert n != -1, $t & "." & name & " invalid" - assert types[n] == typ, - "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ + assert types[n].replace("typeof", "type") == typ, + "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ assert A == 2 assert B == 1.0 @@ -118,6 +118,12 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 +assert CASTEXPR == 34.chr + +assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" +assert NULLCHAR == '\0' +assert OCTCHAR == '\n' +assert HEXCHAR.int == 0xFE assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) -- cgit v1.2.3