aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGanesh Viswanathan <dev@genotrance.com>2020-05-09 02:34:59 -0500
committerGanesh Viswanathan <dev@genotrance.com>2020-05-09 02:34:59 -0500
commit701a8793c2a30dfcecdf24023be3c8091e724d2c (patch)
tree4cf9d996a91da7d5273d23d77b49f51547a484e6
parent5a6eb043d28fb03b19c14ce766f166f59fe269ab (diff)
downloadnimterop-pragmas.tar.gz
nimterop-pragmas.zip
Fix #202 - implement noHeaderpragmas
-rw-r--r--CHANGES.md8
-rw-r--r--README.md61
-rw-r--r--nimterop.nimble4
-rw-r--r--nimterop/ast.nim6
-rw-r--r--nimterop/ast2.nim105
-rw-r--r--nimterop/comphelp.nim84
-rw-r--r--nimterop/docs.nim4
-rw-r--r--nimterop/getters.nim459
-rw-r--r--nimterop/globals.nim37
-rw-r--r--nimterop/grammar.nim8
-rw-r--r--nimterop/toast.nim24
-rw-r--r--nimterop/tshelp.nim354
-rw-r--r--tests/include/tast2.h2
-rw-r--r--tests/tast2.nim18
-rw-r--r--tests/tpcre.nim2
15 files changed, 583 insertions, 593 deletions
diff --git a/CHANGES.md b/CHANGES.md
index b02453c..ef66c79 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -14,18 +14,18 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0
### Breaking changes
-- Nimterop now skips generating the `{.header.}` pragma by default in non-dynlib mode. This skips the header file `#include` in the generated code and allows creation of wrappers that do not require presence of the header during compile time. There are cases where this will not work so the `--includeHeader | -H` flag is available to revert to the legacy behavior when required. This change applies to both `ast2` and the legacy backend so if an existing wrapper breaks, test it with `-H` to see if it starts working again. [#169][i169]
-
-- Nimterop defaulted to C++ mode for preprocessing and tree-sitter parsing in all cases unless explicitly informed to use C mode. This has been changed and is now detected based on the file extension. This means some existing wrappers could break since they might contain C++ code or include C++ headers like `#include <string>` which will not work in C mode. Explicitly setting `mode = "cpp"` or `-mcpp` should fix such issues. [#176][i176]
+- Nimterop used to default to C++ mode for preprocessing and tree-sitter parsing in all cases unless explicitly informed to use C mode. This has been changed and is now detected based on the file extension. This means some existing wrappers could break since they might contain C++ code or include C++ headers like `#include <string>` which will not work in C mode. Explicitly setting `mode = "cpp"` or `-mcpp` should fix such issues. [#176][i176]
- Enums were originally being mapped to `distint int` - this has been changed to `distinct cint` since the sizes are incorrect on 64-bit and is especially noticeable when types or unions have enum fields.
-- `static inline` functions are no longer wrapped by the legacy backend. The `ast2` backend correctly generates wrappers for such functions but they are only generated when `--includeHeader | -H` is in effect. This is because such functions do not exist in the binary and can only be referenced when the header is compiled in.
+- `static inline` functions are no longer wrapped by the legacy backend. The `ast2` backend correctly generates wrappers for such functions but they are only generated when `--noHeader | -H` is not in effect. This is because such functions do not exist in the binary and can only be referenced when the header is compiled in.
- Support for Nim v0.19.6 has been dropped and the test matrix now covers v0.20.2, v1.0.6, v1.2.0 and devel.
### New functionality
+- Nimterop can now skip generating the `{.header.}` pragma when the `--noHeader | -H` flag is used. This skips the header file `#include` in the generated code and allows creation of wrappers that do not require presence of the header during compile time. Note that `static inline` functions will only be wrapped when the header is compiled in. This change applies to both `ast2` and the legacy backend, although `ast2` can also generate wrappers with both `{.header.}` and `{.dynlib.}` in effect enabling type size checking with `-d:checkAbi`. More information is available in the [README.md](README.md). [#169][i169]
+
- `ast2` includes support for various C constructs that were issues with the legacy backend. These changes should reduce the reliance on `cOverride()` and existing wrappers should attempt to clean up such sections where possible.
- N-dimensional arrays and pointers - [#54][i54]
- Synomyms for types - [#74][i74]
diff --git a/README.md b/README.md
index 8680048..869ac4e 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@ else:
Module documentation for the build API can be found [here](https://nimterop.github.io/nimterop/build.html). Refer to the ```tests``` directory for additional examples on how the library can be used. Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work.
-__Download / Search__
+#### Download / Search
The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth.
@@ -78,19 +78,19 @@ The above wrapper is generic and allows the end user to control how it works. No
- The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL if defined.
- If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly.
-__Pre build__
+#### Pre build
`getHeader()` provides a `headerPreBuild()` hook that gets called after the library is downloaded but before it is built. This allows for any manipulations of the source files or build scripts before build. [archive](https://github.com/genotrance/nimarchive/blob/master/nimarchive/archive.nim) has such an example.
The build API also includes various compile time helper procs that aid in file manipulation, Cmake shortcuts, library linking, etc. Refer to [build](https://nimterop.github.io/nimterop/build.html) for more details.
-__Build__
+#### Build
Nimterop currently supports `configure` and `cmake` based building of libraries, with `cmake` taking precedence if a project supports both. Nimterop verifies that the tool selected is available and notifies the user if any issues are found. Bash is required on Windows for `configure` and the binary shipped with Git has been tested.
Flags can be specified to these tools via `getHeader()` or directly via the underlying `configure()` and `cmake()` calls. Once the build scripts are ready, `getHeader()` then calls `make()`. At every step, `getHeader()` checks for the presence of created artifacts and does not redo steps that have been successfully completed.
-__Linking__
+#### Linking
- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If it is omitted, the dynamic library is returned in `headerLPath`.
- `getHeader()` searches for libraries based on the header name by default:
@@ -103,7 +103,7 @@ __Linking__
- See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example.
- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking.
-__User control__
+#### User control
The `-d:xxxYYY` Nim define flags have already been described above and can be specified on the command line or in a nim.cfg file. It is also possible to specify them within the wrapper itself using `setDefines()` if required. Further, all defines, regardless of how they are specified, can be generically checked using `isDefined()`.
@@ -132,7 +132,7 @@ cCompile("clib/src/*.c") # Compile in any implementation source files
Module documentation for the wrapper API can be found [here](https://nimterop.github.io/nimterop/cimport.html).
-__Preprocessing__
+#### Preprocessing
In order to leverage the preprocessor, certain projects might need `cDefine()` calls to set `#define` values. Simpler library may have documentation that cover this but larger ones will rely on build tools that discover and set values in a `config.h` which is loaded with `#include`. Projects might also require some `cIncludeDir()` calls to specify paths to directories that contain other headers. This might be within the library or refer to another library.
@@ -140,7 +140,7 @@ The wrapper API always runs headers through the C preprocessor before wrapping.
By default, the `$CC` environment variable is used for the compiler path. If not found, `toast` defaults to `gcc`.
-__Wrapping__
+#### Wrapping
The `cImport()` call invokes the `toast` binary with appropriate command line flags including any `cDefine()` and `cInclude()` parameters configured. The output of `toast` is then pulled into the module as Nim code and printed if `cDebug()` is specified. This allows for an end user to simply import the wrapper into their code and access the library API as Nim types and procs. Output is cached to save time on subsequent runs. It is also possible to just redirect the output to a file and import that instead if preferred.
@@ -148,13 +148,44 @@ The `recurse` flag can be set to enable the recursion capability which runs thro
There may be cases where the wrapper generated by `toast` for certain types or procs is not preferred, or may be skipped or altogether wrong due to limitations or bugs. In these instances, the `cOverride()` macro can be used to define consts, types or procs to use in place of the wrapper generated output. `cImport()` will forward this information to `toast` and the values will be inserted in context in the generated wrapper. This allows wrapper authors to work around tool limitations or to improve the wrapper output - say change `ptr X` to `var X` or to create more Nim friendly types or proc signatures.
-Several C libraries also use leading and/or trailing `_` in identifiers and since Nim does not allow this, the `cPlugin()` macro can be used to modify such symbols or `cSkipSymbol()` them altogether. Instead of a full `cPlugin()` section, it might also be preferred to set `flags = "-E_ -F_"` to the `cImport()` call to trim out such characters. These features can also be used to remove common prefixes like `SDL_` to generate a cleaner wrapper. `cPlugin()` is real Nim code though so anything Nim allows is fair game. Note that `cPlugin()` overrides any `-E -F` flags. Also, behind the scenes, `cOverride()` is communicated to `toast` via `cPlugin()`.
+Several C libraries also use leading and/or trailing `_` in identifiers and since Nim does not allow this, the `cPlugin()` macro can be used to modify such symbols or `cSkipSymbol()` them altogether. Instead of a full `cPlugin()` section, it might also be preferred to set `flags = "-E_ -F_"` to the `cImport()` call to trim out such characters. These features can also be used to remove common prefixes like `SDL_` to generate a cleaner wrapper. The `--replace | -G` flag can be used for replacements. `cPlugin()` is real Nim code though so anything Nim allows is fair game. Note that `cPlugin()` overrides any `-E -F -G` flags. Also, behind the scenes, `cOverride()` is communicated to `toast` via `cPlugin()`.
If the same `cPlugin()` is needed in multiple wrapper files, the code can be moved into a standalone file and be used with the `cPluginPath()` call.
-Lastly, `c2nImport()` provides access to calling `c2nim` from the wrapper instead of `toast`. Note that `c2nImport()` does not use any of the above described features like `cPlugin()` and needs to be controlled with the `flags` param.
+Lastly, `c2nImport()` provides access to calling `c2nim` from the wrapper instead of `toast`. Note that `c2nImport()` does not use any of the above described features like `cPlugin()` and needs to be controlled with `c2nim` specific flags via the `flags` param.
-__Compiling source__
+#### Header vs. Dynlib
+
+Nim provides some flexibility when it comes to using C/C++ libraries. In order to understand this better, some Nim pragmas need to be introduced. The main one is `{.importc.}` which informs Nim to use a symbol defined in a C library. This applies to both types and procs but how Nim should find the symbol is slightly different for each.
+
+For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol and to `#include "header.h"` in the generated code. However, types can be mostly recreated in pure Nim so it is also possible to omit both `{.importc.}` and `{.header}` and it will work just fine except with a different name in the generated C code. This allows the user to compile the wrapper without requiring `header.h` to be present.
+
+For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation.
+
+While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. The user will need to choose based on the library in question.
+
+Going further, the `{.dynlib: "path/to/libheader.so".}` pragma can be used to inform Nim to load the library at runtime and link the function instead of linking at compile time. This enables creation of a wrapper that does not need the library present at compile time.
+
+Now that this is understood, a user might want any combination of the above in the wrapper rendered by Nimterop. This can be controlled with various flags to `cImport()` and `toast`.
+- By default, generated wrappers will include the `{.header, importc.}` pragmas for types and procs. This can be disabled with the `--noHeader | -H` flag to `toast` or `flags = "-H"` param to `cImport()` which will remove `{.header}` for both and `{.importc.}` for types only.
+- By default, generated wrappers will assume that the user will link the library implementation themselves. The `--dynlib | -l` flag to `toast` or `dynlib = "headerLPath"` param to `cImport()` will configure the wrapper to generate `{.dynlib.}` pragmas for procs.
+
+This results in four cases:
+1. Default: `{.header, importc.}` for both types and procs
+2. With `--noHeader`, types will be pure Nim and procs will be just `{.importc.}`
+3. With `--dynlib`, types will still be `{.header, importc.}` but procs will be `{.dynlib, importc.}`
+4. With `--dynlib` and `--noHeader`, types will be pure Nim, procs will be `{.dynlib, importc.}`
+
+While `ast2` supports all these modes, the legacy backend does not support the third mixed case and will infer `--noHeader` when `--dynlib` is specified (case 4). Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib` for `ast2`.
+
+More documentation on on these pragmas can be found in the Nim manual:
+- [{.importc.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-importc-pragma)
+- [{.header.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-header-pragma)
+- [{.dynlib.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-dynlib-pragma-for-import)
+- [{.passL.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-passl-pragma)
+- [{.compile.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-pragma)
+
+#### Compiling the source
The job of building and compiling the underlying C library is best left to the build mechanism selected by the library author so using `getHeader()` is recommended. For simpler projects with a few `.c` files though, `cCompile()` should be more than enough. It is not recommended for larger projects which heavily rely on functionality offered by build tools. Recreating reliable logic in Nim can be tedious and one can expect minimal support from that author if their tested build mechanism is not used.
@@ -180,13 +211,13 @@ Options:
-d, --debug bool false enable debug output
-D=, --defines= strings {} definitions to pass to preprocessor
-l=, --dynlib= string "" import symbols from library in specified Nim string
- -f=, --feature= Features {} flags to enable experimental features
- -H, --includeHeader bool false add {.header.} pragma to wrapper
+ -f=, --feature= Features ast1 flags to enable experimental features
-I=, --includeDirs= strings {} include directory to pass to preprocessor
-m=, --mode= string "" language parser: c or cpp
--nim= string "nim" use a particular Nim executable
- -c, --nocomments bool false exclude top-level comments from output
- -o=, --output= string "" file to output content
+ -c, --noComments bool false exclude top-level comments from output
+ -H, --noHeader bool false skip {.header.} pragma in wrapper
+ -o=, --output= string "" file to output content - default: stdout
-a, --past bool false print AST output
-g, --pgrammar bool false print grammar
--pluginSourcePath= string "" nim file to build and load as a plugin
@@ -212,7 +243,7 @@ The tree-sitter library is limited though - it may fail on some advanced languag
It is debatable whether a syntax highlighting engine like `tree-sitter` is the most reliable method to convert C code into AST. However, it is lightweight, cross-platform with no dependencies and handles error conditions gracefully. It has produced usable wrappers for C libraries though things could get murky when considering C++ but that will be a topic for another day. Nimterop relies heavily on the preprocessor, as discussed next, so having an engine which can run anywhere has been worth the compromise. Only time will tell though.
-__Preprocessing__
+### Preprocessing
The wrapper API always runs headers through the C preprocessor before wrapping, unlike the command line interface where the `-p | --preprocess` flag is not set by default but *highly* recommended. This is because almost all platform, compiler and package discovery is handled by build tools like `configure` and `cmake` which then use preprocessor `#define` values to tweak what C code is applicable for that platform. While parsing preprocessor macros is possible in tools like `toast`, given how dependent the `#ifdef` branches are on values provided by these and many other build tools, preprocessing seems is best left to them than attempting to self-discover or intercept that information.
diff --git a/nimterop.nimble b/nimterop.nimble
index 984d0c5..d60224f 100644
--- a/nimterop.nimble
+++ b/nimterop.nimble
@@ -46,11 +46,13 @@ task docs, "Generate docs":
buildDocs(@["nimterop/all.nim"], "build/htmldocs")
task test, "Test":
+ rmFile("tests/timeit.txt")
+
buildTimeitTask()
buildToastTask()
execTest "tests/tast2.nim"
- execTest "tests/tast2.nim", "-d:HEADER"
+ execTest "tests/tast2.nim", "-d:NOHEADER"
execTest "tests/tnimterop_c.nim"
execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\""
diff --git a/nimterop/ast.nim b/nimterop/ast.nim
index 1f1cac6..96116c7 100644
--- a/nimterop/ast.nim
+++ b/nimterop/ast.nim
@@ -2,11 +2,11 @@ import hashes, macros, os, sets, strformat, strutils, tables
import regex
-import "."/[getters, globals, treesitter/api]
+import "."/[getters, globals, treesitter/api, tshelp]
proc getHeaderPragma*(gState: State): string =
result =
- if gState.isIncludeHeader():
+ if not gState.noHeader and gState.dynlib.Bl:
&", header: {gState.currentHeader}"
else:
""
@@ -211,7 +211,7 @@ proc parseNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable
gState.impShort = gState.currentHeader.replace("header", "imp")
gState.sourceFile = fullpath
- if gState.isIncludeHeader():
+ if not gState.noHeader and gState.dynlib.Bl:
gState.constStr &= &"\n {gState.currentHeader} {{.used.}} = \"{fp}\""
root.searchAst(astTable, gState)
diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim
index 6f05a34..1230e06 100644
--- a/nimterop/ast2.nim
+++ b/nimterop/ast2.nim
@@ -15,7 +15,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS
# If not, symbol needs to be skipped - only get here if `name` is blank
let
# Get cleaned name for symbol, set parent so that cOverride is ignored
- name = gState.getIdentifier(origname, kind, parent = "getOverrideOrSkip")
+ name = gState.getIdentifier(origname, kind, parent = "IgnoreSkipSymbol")
override = gState.getOverride(origname, kind)
@@ -244,7 +244,8 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas:
#
# If `fname`, use it instead of node.getAtom() for name
# If `pragmas`, add as nkPragmaExpr but not for `nskProc` since procs add pragmas elsewhere
- # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} when includeHeader
+ # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".}
+ # when `not noHeader`
let
atom = node.getAtom()
@@ -306,14 +307,14 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas:
# )
var
pragmas =
- if gState.isIncludeHeader():
+ if not gState.noHeader:
# Need to add header and importc
if istype and name == origname:
- # Need to add impShort since neither struct/union nor name change
- pragmas & gState.impShort
+ # Neither struct/union nor name change
+ pragmas & "importc" & (gState.impShort & "Hdr")
else:
# Add header shortcut, additional pragmas added later
- pragmas & (gState.impShort & "H")
+ pragmas & (gState.impShort & "Hdr")
else:
pragmas
@@ -323,7 +324,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas:
else:
ident
- if gState.isIncludeHeader():
+ if not gState.noHeader:
if not istype or name != origname:
# Add importc pragma since either struct/union or name changed
let
@@ -361,18 +362,20 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas:
prident = block:
var
prident: PNode
+ # Add {.importc.} pragma
if name != origname:
- # Add importc pragma since name changed
+ # Name changed
prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}"))
- if gState.isIncludeHeader():
- # Add header
- gState.addPragma(node, prident[1], gState.impShort & "H")
- elif gState.dynlib.nBl:
- # Add dynlib
- gState.addPragma(node, prident[1], "dynlib", gState.getIdent(gState.dynlib))
else:
- # Only need impShort since no name change
- prident = gState.newPragmaExpr(node, ident, gState.impShort)
+ prident = gState.newPragmaExpr(node, ident, "importc")
+
+ if gState.dynlib.nBl:
+ # Add {.dynlib.}
+ gState.addPragma(node, prident[1], gState.impShort & "Dyn")
+ elif not gState.noHeader:
+ # Add {.header.}
+ gState.addPragma(node, prident[1], gState.impShort & "Hdr")
+
if pragmas.nBl:
gState.addPragma(node, prident[1], pragmas)
prident
@@ -789,7 +792,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
npexpr = gState.newPragmaExpr(node, typedef[0], pragmas)
typedef[0] = npexpr
else:
- # includeHeader already added impShort in newXIdent()
+ # `not gState.noHeader` already added pragmas in newXIdent()
gState.addPragma(node, typeDef[0][1], pragmas)
# nkTypeSection.add
@@ -1552,23 +1555,20 @@ proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) =
gState.newPragma(node, "importc", newStrNode(nkStrLit, origname))
else:
# {.impnameC.} shortcut
- gState.newPragma(node, gState.impShort & "C")
+ gState.newPragma(node, "importc")
# Detect ... and add {.varargs.}
pvarargs = plist.getVarargs()
- # Need {.convention.} and {.header.} if applicable
- if name != origname:
- if gState.isIncludeHeader():
- # {.impnameHC.} shortcut
- gState.addPragma(node, prident, gState.impShort & "HC")
- else:
- # {.convention.}
- gState.addPragma(node, prident, gState.convention)
+ # Need {.convention.}
+ gState.addPragma(node, prident, gState.convention)
- if gState.dynlib.nBl:
- # {.dynlib.} for DLLs
- gState.addPragma(node, prident, "dynlib", gState.getIdent(gState.dynlib))
+ if gState.dynlib.nBl:
+ # Add {.dynlib.}
+ gState.addPragma(node, prident, gState.impShort & "Dyn")
+ elif not gState.noHeader:
+ # Add {.header.}
+ gState.addPragma(node, prident, gState.impShort & "Hdr")
if pvarargs:
# Add {.varargs.} for ...
@@ -1637,10 +1637,10 @@ proc addDef(gState: State, node: TSNode) =
commentNodes = gState.getCommentNodes(node)
if node[start+1].getName() == "function_declarator":
- if gState.isIncludeHeader():
+ if not gState.noHeader:
gState.addProc(node[start+1], node[start], commentNodes)
else:
- gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" %
+ gecho &"\n# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" %
gState.getNodeVal(node[start+1].getAtom())
proc processNode(gState: State, node: TSNode): bool =
@@ -1707,50 +1707,35 @@ proc searchTree(gState: State, root: TSNode) =
if node == root:
break
+###
+# gState.addPragma(root, impPragma, "importc")
+# gState.addPragma(root, impConvPragma, gState.convention)
+
proc setupPragmas(gState: State, root: TSNode, fullpath: string) =
# Create shortcut pragmas to reduce clutter
var
hdrPragma: PNode
- hdrConvPragma: PNode
- impPragma = newNode(nkPragma)
- impConvPragma = newNode(nkPragma)
+ dynPragma: PNode
- # {.pragma: impname, importc.}
- gState.addPragma(root, impPragma, "pragma", gState.getIdent(gState.impShort))
- gState.addPragma(root, impPragma, "importc")
-
- if gState.isIncludeHeader():
+ if not gState.noHeader:
# Path to header const
gState.constSection.add gState.newConstDef(
root, fname = gState.currentHeader, fval = '"' & fullpath & '"')
- # {.pragma: impnameH, header: "xxx".} for types when name != origname
- hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "H"))
+ # {.pragma: impnameHdr, header: "xxx".}
+ hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr"))
gState.addPragma(root, hdrPragma, "header", gState.getIdent(gState.currentHeader))
- # Add {.impnameH.} to {.impname.}
- gState.addPragma(root, impPragma, gState.impShort & "H")
-
- # {.pragma: impnameHC, impnameH, convention.} for procs when name != origname
- hdrConvPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "HC"))
- gState.addPragma(root, hdrConvPragma, gState.impShort & "H")
- gState.addPragma(root, hdrConvPragma, gState.convention)
-
- # {.pragma: impnameC, impname, convention.} for procs
- gState.addPragma(root, impConvPragma, "pragma", gState.getIdent(gState.impShort & "C"))
- gState.addPragma(root, impConvPragma, gState.impShort)
- gState.addPragma(root, impConvPragma, gState.convention)
-
if gState.dynlib.nBl:
- # {.dynlib.} for DLLs
- gState.addPragma(root, impConvPragma, "dynlib", gState.getIdent(gState.dynlib))
+ # {.pragma: impnameDyn, dynlib: libname.}
+ dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn"))
+ gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib))
- # Add all pragma shortcuts to output
+ # Add pragma shortcuts to output
if not hdrPragma.isNil:
gState.pragmaSection.add hdrPragma
- gState.pragmaSection.add hdrConvPragma
- gState.pragmaSection.add impPragma
- gState.pragmaSection.add impConvPragma
+ if not dynPragma.isNil:
+ gState.pragmaSection.add dynPragma
proc printNimHeader*(gState: State) =
# Top level output with context info
diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim
index 18915b0..e577717 100644
--- a/nimterop/comphelp.nim
+++ b/nimterop/comphelp.nim
@@ -1,6 +1,8 @@
-import compiler/[ast, lineinfos, msgs, options, parser, renderer]
+import macros, strutils
-import "."/[globals, getters]
+import compiler/[ast, idents, lineinfos, msgs, options, parser, pathutils, renderer]
+
+import "."/[globals, getters, treesitter/api, tshelp]
proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) =
# Raise exception in parseString() instead of exiting for errors
@@ -17,6 +19,84 @@ proc parseString*(gState: State, str: string): PNode =
except:
decho getCurrentExceptionMsg()
+proc printTree*(gState: State, pnode: PNode, offset = ""): string =
+ if not pnode.isNil and gState.debug and pnode.kind != nkNone:
+ result &= "\n# " & offset & $pnode.kind & "("
+ case pnode.kind
+ of nkCharLit:
+ result &= ($pnode.intVal.char).escape & ")"
+ of nkIntLit..nkUInt64Lit:
+ result &= $pnode.intVal & ")"
+ of nkFloatLit..nkFloat128Lit:
+ result &= $pnode.floatVal & ")"
+ of nkStrLit..nkTripleStrLit:
+ result &= pnode.strVal.escape & ")"
+ of nkSym:
+ result &= $pnode.sym & ")"
+ of nkIdent:
+ result &= "\"" & $pnode.ident.s & "\")"
+ else:
+ if pnode.sons.len != 0:
+ for i in 0 ..< pnode.sons.len:
+ result &= gState.printTree(pnode.sons[i], offset & " ")
+ if i != pnode.sons.len - 1:
+ result &= ","
+ result &= "\n# " & offset & ")"
+ else:
+ result &= ")"
+ if offset.len == 0:
+ result &= "\n"
+
+proc printDebug*(gState: State, pnode: PNode) =
+ if gState.debug and pnode.kind != nkNone:
+ gecho ("Output => " & $pnode).getCommented()
+ gecho gState.printTree(pnode)
+
+proc getDefaultLineInfo*(gState: State): TLineInfo =
+ result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, 0, 0)
+
+proc getLineInfo*(gState: State, node: TSNode): TLineInfo =
+ # Get Nim equivalent line:col info from node
+ let
+ (line, col) = gState.getLineCol(node)
+
+ result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, line, col)
+
+proc getIdent*(gState: State, name: string, info: TLineInfo, exported = true): PNode =
+ if name.nBl:
+ # Get ident PNode for name + info
+ let
+ exp = getIdent(gState.identCache, "*")
+ ident = getIdent(gState.identCache, name)
+
+ if exported:
+ result = newNode(nkPostfix)
+ result.add newIdentNode(exp, info)
+ result.add newIdentNode(ident, info)
+ else:
+ result = newIdentNode(ident, info)
+
+proc getIdent*(gState: State, name: string): PNode =
+ gState.getIdent(name, gState.getDefaultLineInfo(), exported = false)
+
+proc getIdentName*(node: PNode): string =
+ if not node.isNil:
+ for i in 0 ..< node.len:
+ if node[i].kind == nkIdent and $node[i] != "*":
+ result = $node[i]
+ if result.Bl and node.len > 0:
+ result = node[0].getIdentName()
+
+proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""):
+ tuple[name, origname: string, info: TLineInfo] =
+ # Shortcut to get identifier name and info (node value and line:col)
+ result.origname = gState.getNodeVal(node)
+ result.name = gState.getIdentifier(result.origname, kind, parent)
+ if result.name.nBl:
+ if kind == nskType:
+ result.name = result.name.getType()
+ result.info = gState.getLineInfo(node)
+
proc getPtrType*(str: string): string =
result = case str:
of "cchar":
diff --git a/nimterop/docs.nim b/nimterop/docs.nim
index d5bd50c..6c2bbfd 100644
--- a/nimterop/docs.nim
+++ b/nimterop/docs.nim
@@ -74,13 +74,13 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath
for file in files:
echo execAction(&"{nim} doc {defStr} {nimArgs} -o:{path} --project --index:on {baseDir & file}")
- echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}")
+ echo execAction(&"{nim} buildIndex {nimArgs} -o:{path}/theindex.html {path}")
when declared(getNimRootDir):
#[
this enables doc search, works at least locally with:
cd {path} && python -m SimpleHTTPServer 9009
]#
- echo execAction(&"{nim} js -o:{path}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim")
+ echo execAction(&"{nim} js {nimArgs} -o:{path}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim")
for i in 0 .. paramCount():
if paramStr(i) == "--publish":
diff --git a/nimterop/getters.nim b/nimterop/getters.nim
index 10c3b5e..12d08eb 100644
--- a/nimterop/getters.nim
+++ b/nimterop/getters.nim
@@ -2,9 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times
import regex
-import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer]
-
-import "."/[build, globals, plugin, treesitter/api]
+import "."/[build, globals, plugin]
const gReserved = """
addr and as asm
@@ -135,7 +133,12 @@ proc checkIdentifier(name, kind, parent, origName: string) =
doAssert (not name.contains("__")): errmsg & " consecutive underscores '_'"
- if parent.nBl:
+ # Cannot blank out symbols which are fields or params
+ #
+ # `IgnoreSkipSymbol` is used to `getIdentifier()` even if symbol is in `symOverride` list
+ # so that any prefix/suffix/replace or `onSymbol()` processing can occur. This is only used
+ # for `cOverride()` since it also depends on `symOverride`.
+ if parent.nBl and parent != "IgnoreSkipSymbol":
doAssert name.nBl, &"Blank identifier, originally '{parentStr}{origName}' ({kind}), cannot be empty"
proc getIdentifier*(gState: State, name: string, kind: NimSymKind, parent=""): string =
@@ -211,7 +214,7 @@ proc getOverride*(gState: State, name: string, kind: NimSymKind): string =
if gState.onSymbolOverride != nil:
var
- nname = gState.getIdentifier(name, kind, "Override")
+ nname = gState.getIdentifier(name, kind, "IgnoreSkipSymbol")
sym = Symbol(name: nname, kind: kind)
if nname.nBl:
gState.onSymbolOverride(sym)
@@ -236,327 +239,12 @@ proc getKeyword*(kind: NimSymKind): string =
# cOverride procs already include `proc` keyword
result = ($kind).replace("nsk", "").toLowerAscii()
-# TSNode shortcuts
-
-proc isNil*(node: TSNode): bool =
- node.tsNodeIsNull()
-
-proc len*(node: TSNode): int =
- if not node.isNil:
- result = node.tsNodeNamedChildCount().int
-
-proc `[]`*(node: TSNode, i: SomeInteger): TSNode =
- if i < type(i)(node.len()):
- result = node.tsNodeNamedChild(i.uint32)
-
-proc getName*(node: TSNode): string {.inline.} =
- if not node.isNil:
- return $node.tsNodeType()
-
-proc getNodeVal*(code: var string, node: TSNode): string =
- if not node.isNil:
- return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
-
-proc getNodeVal*(gState: State, node: TSNode): string =
- gState.code.getNodeVal(node)
-
-proc getAtom*(node: TSNode): TSNode =
- if not node.isNil:
- # Get child node which is topmost atom
- if node.getName() in gAtoms:
- return node
- elif node.len != 0:
- if node[0].getName() == "type_qualifier":
- # Skip const, volatile
- if node.len > 1:
- return node[1].getAtom()
- else:
- return
- else:
- return node[0].getAtom()
-
-proc getStartAtom*(node: TSNode): int =
- if not node.isNil:
- # Skip const, volatile and other type qualifiers
- for i in 0 .. node.len - 1:
- if node[i].getAtom().getName() notin gAtoms:
- result += 1
- else:
- break
-
-proc getXCount*(node: TSNode, ntype: string, reverse = false): int =
- if not node.isNil:
- # Get number of ntype nodes nested in tree
- var
- cnode = node
- while ntype in cnode.getName():
- result += 1
- if reverse:
- cnode = cnode.tsNodeParent()
- else:
- if cnode.len != 0:
- if cnode[0].getName() == "type_qualifier":
- # Skip const, volatile
- if cnode.len > 1:
- cnode = cnode[1]
- else:
- break
- else:
- cnode = cnode[0]
- else:
- break
-
-proc getPtrCount*(node: TSNode, reverse = false): int =
- node.getXCount("pointer_declarator")
-
-proc getArrayCount*(node: TSNode, reverse = false): int =
- node.getXCount("array_declarator")
-
-proc getDeclarator*(node: TSNode): TSNode =
- if not node.isNil:
- # Return if child is a function or array declarator
- if node.getName() in ["function_declarator", "array_declarator"]:
- return node
- elif node.len != 0:
- return node[0].getDeclarator()
-
-proc getVarargs*(node: TSNode): bool =
- # Detect ... and add {.varargs.}
- #
- # `node` is the param list
- #
- # ... is an unnamed node, second last node and ) is last node
- let
- nlen = node.tsNodeChildCount()
- if nlen > 1.uint32:
- let
- nval = node.tsNodeChild(nlen - 2.uint32).getName()
- if nval == "...":
- result = true
-
-proc firstChildInTree*(node: TSNode, ntype: string): TSNode =
- # Search for node type in tree - first children
- var
- cnode = node
- while not cnode.isNil:
- if cnode.getName() == ntype:
- return cnode
- cnode = cnode[0]
-
-proc anyChildInTree*(node: TSNode, ntype: string): TSNode =
- # Search for node type anywhere in tree - depth first
- var
- cnode = node
- while not cnode.isNil:
- if cnode.getName() == ntype:
- return cnode
- for i in 0 ..< cnode.len:
- let
- ccnode = cnode[i].anyChildInTree(ntype)
- if not ccnode.isNil:
- return ccnode
- if cnode != node:
- cnode = cnode.tsNodeNextNamedSibling()
- else:
- break
-
-proc mostNestedChildInTree*(node: TSNode): TSNode =
- # Search for the most nested child of node's type in tree
- var
- cnode = node
- ntype = cnode.getName()
- while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype:
- cnode = cnode[0]
- result = cnode
-
-proc inChildren*(node: TSNode, ntype: string): bool =
- # Search for node type in immediate children
- result = false
- for i in 0 ..< node.len:
- if (node[i]).getName() == ntype:
- result = true
- break
-
-proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
- # Get line number and column info for node
- let
- point = node.tsNodeStartPoint()
- result.line = point.row.int + 1
- result.col = point.column.int + 1
-
-proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
- getLineCol(gState.code, node)
-
-proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
- # Get line number and column info for node
- let
- point = node.tsNodeEndPoint()
- result.line = point.row.int + 1
- result.col = point.column.int + 1
-
-proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
- getEndLineCol(gState.code, node)
-
-proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
- for i in 0 ..< node.len:
- if node.getName() != "comment":
- result += 1
-
-proc getPxName*(node: TSNode, offset: int): string =
- # Get the xth (grand)parent of the node
- var
- np = node
- count = 0
-
- while not np.isNil and count < offset:
- np = np.tsNodeParent()
- count += 1
-
- if count == offset and not np.isNil:
- return np.getName()
-
-proc printLisp*(code: var string, root: TSNode): string =
- var
- node = root
- nextnode: TSNode
- depth = 0
-
- while true:
- if not node.isNil and depth > -1:
- result &= spaces(depth)
- let
- (line, col) = code.getLineCol(node)
- result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}"
- let
- val = code.getNodeVal(node)
- if "\n" notin val and " " notin val:
- result &= &" \"{val}\""
- else:
- break
-
- if node.len() != 0:
- result &= "\n"
- nextnode = node[0]
- depth += 1
- else:
- result &= ")\n"
- nextnode = node.tsNodeNextNamedSibling()
-
- if nextnode.isNil:
- while true:
- node = node.tsNodeParent()
- depth -= 1
- if depth == -1:
- break
- result &= spaces(depth) & ")\n"
- if node == root:
- break
- if not node.tsNodeNextNamedSibling().isNil:
- node = node.tsNodeNextNamedSibling()
- break
- else:
- node = nextnode
-
- if node == root:
- break
-
-proc printLisp*(gState: State, root: TSNode): string =
- printLisp(gState.code, root)
-
-proc getCommented*(str: string): string =
- "\n# " & str.strip().replace("\n", "\n# ")
-
-proc printTree*(gState: State, pnode: PNode, offset = ""): string =
- if not pnode.isNil and gState.debug and pnode.kind != nkNone:
- result &= "\n# " & offset & $pnode.kind & "("
- case pnode.kind
- of nkCharLit:
- result &= ($pnode.intVal.char).escape & ")"
- of nkIntLit..nkUInt64Lit:
- result &= $pnode.intVal & ")"
- of nkFloatLit..nkFloat128Lit:
- result &= $pnode.floatVal & ")"
- of nkStrLit..nkTripleStrLit:
- result &= pnode.strVal.escape & ")"
- of nkSym:
- result &= $pnode.sym & ")"
- of nkIdent:
- result &= "\"" & $pnode.ident.s & "\")"
- else:
- if pnode.sons.len != 0:
- for i in 0 ..< pnode.sons.len:
- result &= gState.printTree(pnode.sons[i], offset & " ")
- if i != pnode.sons.len - 1:
- result &= ","
- result &= "\n# " & offset & ")"
- else:
- result &= ")"
- if offset.len == 0:
- result &= "\n"
-
-proc printDebug*(gState: State, node: TSNode) =
- if gState.debug:
- gecho ("Input => " & gState.getNodeVal(node)).getCommented()
- gecho gState.printLisp(node).getCommented()
-
-proc printDebug*(gState: State, pnode: PNode) =
- if gState.debug and pnode.kind != nkNone:
- gecho ("Output => " & $pnode).getCommented()
- gecho gState.printTree(pnode)
-
-# Compiler shortcuts
-
-proc getDefaultLineInfo*(gState: State): TLineInfo =
- result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, 0, 0)
-
-proc getLineInfo*(gState: State, node: TSNode): TLineInfo =
- # Get Nim equivalent line:col info from node
- let
- (line, col) = gState.getLineCol(node)
-
- result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, line, col)
-
-proc getIdent*(gState: State, name: string, info: TLineInfo, exported = true): PNode =
- if name.nBl:
- # Get ident PNode for name + info
- let
- exp = getIdent(gState.identCache, "*")
- ident = getIdent(gState.identCache, name)
-
- if exported:
- result = newNode(nkPostfix)
- result.add newIdentNode(exp, info)
- result.add newIdentNode(ident, info)
- else:
- result = newIdentNode(ident, info)
-
-proc getIdent*(gState: State, name: string): PNode =
- gState.getIdent(name, gState.getDefaultLineInfo(), exported = false)
-
-proc getIdentName*(node: PNode): string =
- if not node.isNil:
- for i in 0 ..< node.len:
- if node[i].kind == nkIdent and $node[i] != "*":
- result = $node[i]
- if result.Bl and node.len > 0:
- result = node[0].getIdentName()
-
-proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""):
- tuple[name, origname: string, info: TLineInfo] =
- # Shortcut to get identifier name and info (node value and line:col)
- result.origname = gState.getNodeVal(node)
- result.name = gState.getIdentifier(result.origname, kind, parent)
- if result.name.nBl:
- if kind == nskType:
- result.name = result.name.getType()
- result.info = gState.getLineInfo(node)
-
proc getCurrentHeader*(fullpath: string): string =
("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")]))
proc getPreprocessor*(gState: State, fullpath: string): string =
var
- cmts = if gState.nocomments: "" else: "-CC"
+ cmts = if gState.noComments: "" else: "-CC"
cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """
rdata: seq[string] = @[]
@@ -644,118 +332,6 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool
if result.kind != exactlyOne:
result.name = result.name[0 .. ^2]
-proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string =
- ## Generate a comment from a set of comment nodes. Comment is guaranteed
- ## to be able to be rendered using nim doc
- if commentNodes.len > 0:
- result = "::"
- for commentNode in commentNodes:
- result &= "\n " & gState.getNodeVal(commentNode).strip()
-
- result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "")
- result = result.multiReplace([("\n", "\n "), ("`", "")]).strip()
-
-proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] =
- ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch``
- ## nodes before and after the current node
- ##
- ## Priority is (closest line number) > comment before > comment after.
- ## This priority might need to be changed based on the project, but
- ## for now it is good enough
-
- # Skip this if we don't want comments
- if gState.nocomments:
- return
-
- let (line, _) = gState.getLineCol(node)
-
- # Keep track of both directions from a node
- var
- prevSibling = node.tsNodePrevNamedSibling()
- nextSibling = node.tsNodeNextNamedSibling()
- nilNode: TSNode
-
- var
- i = 0
- prevSiblingDistance, nextSiblingDistance: int = int.high
- lowestDistance: int
- commentsFound = false
-
- while not commentsFound and i < maxSearch:
- # Distance from the current node will tell us approximately if the
- # comment belongs to the node. The closer it is in terms of line
- # numbers, the more we can be sure it's the comment we want
- if not prevSibling.isNil:
- if prevSibling.getName() == "comment":
- prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line)
- else:
- prevSiblingDistance = int.high
- if not nextSibling.isNil:
- if nextSibling.getName() == "comment":
- nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line)
- else:
- nextSiblingDistance = int.high
-
- lowestDistance = min(prevSiblingDistance, nextSiblingDistance)
-
- if prevSiblingDistance > maxSearch:
- # If the line is out of range, skip searching
- prevSibling = nilNode # Can't do `= nil`
-
- if nextSiblingDistance > maxSearch:
- # If the line is out of range, skip searching
- nextSibling = nilNode
-
- # Search above the current line for comments. When one is found
- # keep going to retrieve successive comments for cases with multiple
- # `//` style comments
- while (
- not prevSibling.isNil and
- prevSibling.getName() == "comment" and
- prevSiblingDistance == lowestDistance
- ):
- # Put the previous nodes in reverse order so the comments
- # make logical sense
- result.insert(prevSibling, 0)
- prevSibling = prevSibling.tsNodePrevNamedSibling()
- commentsFound = true
-
- # If we've already found comments above the current line, quit
- if commentsFound:
- break
-
- # Search below or at the current line for comments. When one is found
- # keep going to retrieve successive comments for cases with multiple
- # `//` style comments
- while (
- not nextSibling.isNil and
- nextSibling.getName() == "comment" and
- nextSiblingDistance == lowestDistance
- ):
- result.add(nextSibling)
- nextSibling = nextSibling.tsNodeNextNamedSibling()
- commentsFound = true
-
- if commentsFound:
- break
-
- # Go to next sibling pair
- if not prevSibling.isNil:
- prevSibling = prevSibling.tsNodePrevNamedSibling()
- if not nextSibling.isNil:
- nextSibling = nextSibling.tsNodeNextNamedSibling()
-
- i += 1
-
-proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
- if node.tsNodeNamedChildCount() != 0:
- for i in 0 .. node.tsNodeNamedChildCount()-1:
- let
- name = $node.tsNodeNamedChild(i).tsNodeType()
-
- if name != "comment":
- result.add(name)
-
proc getRegexForAstChildren*(ast: ref Ast): string =
result = "^"
for i in 0 .. ast.children.len-1:
@@ -837,20 +413,15 @@ proc getNimExpression*(gState: State, expr: string, name = ""): string =
("<<", " shl "), (">>", " shr ")
])
-proc getSplitComma*(joined: seq[string]): seq[string] =
- for i in joined:
- result = result.concat(i.split(","))
-
-template isIncludeHeader*(gState: State): bool =
- gState.dynlib.Bl and gState.includeHeader
-
proc getComments*(gState: State, strip = false): string =
- if not gState.nocomments and gState.commentStr.nBl:
+ if not gState.noComments and gState.commentStr.nBl:
result = "\n" & gState.commentStr
if strip:
result = result.replace("\n ", "\n")
gState.commentStr = ""
+# Plugin related
+
proc dll*(path: string): string =
let
(dir, name, _) = path.splitFile()
@@ -888,6 +459,12 @@ proc loadPlugin*(gState: State, sourcePath: string) =
gState.onSymbolOverrideFinal = cast[OnSymbolOverrideFinal](lib.symAddr("onSymbolOverrideFinal"))
+# Misc toast helpers
+
+proc getSplitComma*(joined: seq[string]): seq[string] =
+ for i in joined:
+ result = result.concat(i.split(","))
+
proc expandSymlinkAbs*(path: string): string =
try:
result = path.expandFilename().normalizedPath()
diff --git a/nimterop/globals.nim b/nimterop/globals.nim
index 21214f3..05f507c 100644
--- a/nimterop/globals.nim
+++ b/nimterop/globals.nim
@@ -43,39 +43,6 @@ type
zeroOrOne # ?
orWithNext # !
- PragmaMode* = enum
- # includeHeader = true, dynlib = false
- #
- # Types = {.header, .importc.}
- # Static inline = {.header, .importc.}
- # Procs = {.header, .importc.}
- # Lib = {.compile.} or {.passL.}
- useHeader
-
- # includeHeader = true, dynlib = true
- #
- # Types = {.header, .importc.}
- # Static inline = {.header, .importc.}
- # Procs = {.dynlib, importc.}
- # Libs = loaded by dynlib
- useHeaderDynlib
-
- # includeHeader = false, dynlib = false
- #
- # Types = {.bycopy.}
- # Static inline = not available
- # Procs = {.importc.}
- # Lib = {.compile.} or {.passL.}
- noHeader
-
- # includeHeader = false, dynlib = true
- #
- # Types = {.bycopy.}
- # Static inline = not available
- # Procs = {.dynlib, importc.}
- # Libs = loaded by dynlib
- noHeaderDynlib
-
Ast* = object
name*: string
kind*: Kind
@@ -95,10 +62,10 @@ type
dynlib*: string # `cImport(dynlib)` or `--dynlib | -l` to specify variable containing library name
feature*: seq[Feature] # `--feature | -f` feature flags enabled
includeDirs*: seq[string] # Paths added by `cIncludeDir()` and `--includeDirs | -I` for C/C++ preprocessor/compiler
- includeHeader*: bool # `--includeHeader | -H` to include {.header.} pragma to wrapper
mode*: string # `cImport(mode)` or `--mode | -m` to override detected compiler mode - c or cpp
nim*: string # `--nim` to specify full path to Nim compiler
- nocomments*: bool # `--nocomments | -c` to disable rendering comments in wrappers
+ noComments*: bool # `--noComments | -c` to disable rendering comments in wrappers
+ noHeader*: bool # `--noHeader | -H` to skip {.header.} pragma in wrapper
past*: bool # `--past | -a` to print tree-sitter AST of code
pluginSourcePath*: string # `--pluginSourcePath` specified path to plugin file to compile and load
pnim*: bool # `--pnim | -n` to render Nim wrapper for header
diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim
index eaa337a..e061a70 100644
--- a/nimterop/grammar.nim
+++ b/nimterop/grammar.nim
@@ -2,7 +2,7 @@ import macros, strformat, strutils, tables
import regex
-import "."/[ast, getters, globals, lisp, treesitter/api]
+import "."/[ast, getters, globals, lisp, treesitter/api, tshelp]
type
Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]]
@@ -180,7 +180,7 @@ proc initGrammar(): Grammar =
var
i = 0
- typ = gState.getIdentifier(gState.data[i].val, nskType, "Parent").getType()
+ typ = gState.getIdentifier(gState.data[i].val, nskType, "IgnoreSkipSymbol").getType()
name = ""
nname = ""
tptr = ""
@@ -202,7 +202,7 @@ proc initGrammar(): Grammar =
nname = gState.getIdentifier(name, nskType)
i += 1
- if gState.isIncludeHeader():
+ if not gState.noHeader and gState.dynlib.Bl:
pragmas.add gState.getImportC(name, nname)
let
@@ -318,7 +318,7 @@ proc initGrammar(): Grammar =
else:
var
pragmas: seq[string] = @[]
- if gState.isIncludeHeader():
+ if not gState.noHeader and gState.dynlib.Bl:
pragmas.add gState.getImportC(prefix & name, nname)
pragmas.add "bycopy"
if union.nBl:
diff --git a/nimterop/toast.nim b/nimterop/toast.nim
index 78708f5..d74838a 100644
--- a/nimterop/toast.nim
+++ b/nimterop/toast.nim
@@ -34,11 +34,11 @@ proc main(
defines: seq[string] = @[],
dynlib: string = "",
feature: seq[Feature] = @[Feature.ast1],
- includeHeader = false,
includeDirs: seq[string] = @[],
mode = "",
nim: string = "nim",
- nocomments = false,
+ noComments = false,
+ noHeader = false,
output = "",
past = false,
pgrammar = false,
@@ -61,11 +61,11 @@ proc main(
defines: defines,
dynlib: dynlib,
feature: feature,
- includeHeader: includeHeader,
includeDirs: includeDirs,
mode: mode,
nim: nim,
- nocomments: nocomments,
+ noComments: noComments,
+ noHeader: noHeader,
past: past,
pluginSourcePath: pluginSourcePath,
pnim: pnim,
@@ -77,10 +77,6 @@ proc main(
symOverride: symOverride
)
- # Fail if both includeHeader and dynlib
- doAssert not (includeHeader == true and dynlib.nBl),
- "`includeHeader` and `dynlib` cannot be used simultaneously"
-
# Set gDebug in build.nim
build.gDebug = debug
@@ -196,16 +192,16 @@ when isMainModule:
import cligen
dispatch(main, help = {
"check": "check generated wrapper with compiler",
- "convention": "calling convention for wrapped procs - default: cdecl",
+ "convention": "calling convention for wrapped procs",
"debug": "enable debug output",
"defines": "definitions to pass to preprocessor",
"dynlib": "import symbols from library in specified Nim string",
"feature": "flags to enable experimental features",
- "includeHeader": "add {.header.} pragma to wrapper",
"includeDirs": "include directory to pass to preprocessor",
"mode": "language parser: c or cpp",
- "nim": "use a particular Nim executable - default: $PATH/nim",
- "nocomments": "exclude top-level comments from output",
+ "nim": "use a particular Nim executable",
+ "noComments": "exclude top-level comments from output",
+ "noHeader": "skip {.header.} pragma in wrapper",
"output": "file to output content - default: stdout",
"past": "print AST output",
"pgrammar": "print grammar",
@@ -226,9 +222,9 @@ when isMainModule:
"defines": 'D',
"dynlib": 'l',
"feature": 'f',
- "includeHeader": 'H',
"includeDirs": 'I',
- "nocomments": 'c',
+ "noComments": 'c',
+ "noHeader": 'H',
"output": 'o',
"past": 'a',
"pgrammar": 'g',
diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim
index 109321c..762cf7c 100644
--- a/nimterop/tshelp.nim
+++ b/nimterop/tshelp.nim
@@ -1,4 +1,9 @@
-import "."/treesitter/[c, cpp]
+import sets, strformat, strutils
+
+import regex
+
+import "."/[getters, globals]
+import "."/treesitter/[api, c, cpp]
template withCodeAst*(code: string, mode: string, body: untyped): untyped =
## A simple template to inject the TSNode into a body of code
@@ -25,4 +30,349 @@ template withCodeAst*(code: string, mode: string, body: untyped): untyped =
body
defer:
- tree.tsTreeDelete() \ No newline at end of file
+ tree.tsTreeDelete()
+
+proc getCommented*(str: string): string =
+ "\n# " & str.strip().replace("\n", "\n# ")
+
+proc isNil*(node: TSNode): bool =
+ node.tsNodeIsNull()
+
+proc len*(node: TSNode): int =
+ if not node.isNil:
+ result = node.tsNodeNamedChildCount().int
+
+proc `[]`*(node: TSNode, i: SomeInteger): TSNode =
+ if i < type(i)(node.len()):
+ result = node.tsNodeNamedChild(i.uint32)
+
+proc getName*(node: TSNode): string {.inline.} =
+ if not node.isNil:
+ return $node.tsNodeType()
+
+proc getNodeVal*(code: var string, node: TSNode): string =
+ if not node.isNil:
+ return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
+
+proc getNodeVal*(gState: State, node: TSNode): string =
+ gState.code.getNodeVal(node)
+
+proc getAtom*(node: TSNode): TSNode =
+ if not node.isNil:
+ # Get child node which is topmost atom
+ if node.getName() in gAtoms:
+ return node
+ elif node.len != 0:
+ if node[0].getName() == "type_qualifier":
+ # Skip const, volatile
+ if node.len > 1:
+ return node[1].getAtom()
+ else:
+ return
+ else:
+ return node[0].getAtom()
+
+proc getStartAtom*(node: TSNode): int =
+ if not node.isNil:
+ # Skip const, volatile and other type qualifiers
+ for i in 0 .. node.len - 1:
+ if node[i].getAtom().getName() notin gAtoms:
+ result += 1
+ else:
+ break
+
+proc getXCount*(node: TSNode, ntype: string, reverse = false): int =
+ if not node.isNil:
+ # Get number of ntype nodes nested in tree
+ var
+ cnode = node
+ while ntype in cnode.getName():
+ result += 1
+ if reverse:
+ cnode = cnode.tsNodeParent()
+ else:
+ if cnode.len != 0:
+ if cnode[0].getName() == "type_qualifier":
+ # Skip const, volatile
+ if cnode.len > 1:
+ cnode = cnode[1]
+ else:
+ break
+ else:
+ cnode = cnode[0]
+ else:
+ break
+
+proc getPtrCount*(node: TSNode, reverse = false): int =
+ node.getXCount("pointer_declarator")
+
+proc getArrayCount*(node: TSNode, reverse = false): int =
+ node.getXCount("array_declarator")
+
+proc getDeclarator*(node: TSNode): TSNode =
+ if not node.isNil:
+ # Return if child is a function or array declarator
+ if node.getName() in ["function_declarator", "array_declarator"]:
+ return node
+ elif node.len != 0:
+ return node[0].getDeclarator()
+
+proc getVarargs*(node: TSNode): bool =
+ # Detect ... and add {.varargs.}
+ #
+ # `node` is the param list
+ #
+ # ... is an unnamed node, second last node and ) is last node
+ let
+ nlen = node.tsNodeChildCount()
+ if nlen > 1.uint32:
+ let
+ nval = node.tsNodeChild(nlen - 2.uint32).getName()
+ if nval == "...":
+ result = true
+
+proc firstChildInTree*(node: TSNode, ntype: string): TSNode =
+ # Search for node type in tree - first children
+ var
+ cnode = node
+ while not cnode.isNil:
+ if cnode.getName() == ntype:
+ return cnode
+ cnode = cnode[0]
+
+proc anyChildInTree*(node: TSNode, ntype: string): TSNode =
+ # Search for node type anywhere in tree - depth first
+ var
+ cnode = node
+ while not cnode.isNil:
+ if cnode.getName() == ntype:
+ return cnode
+ for i in 0 ..< cnode.len:
+ let
+ ccnode = cnode[i].anyChildInTree(ntype)
+ if not ccnode.isNil:
+ return ccnode
+ if cnode != node:
+ cnode = cnode.tsNodeNextNamedSibling()
+ else:
+ break
+
+proc mostNestedChildInTree*(node: TSNode): TSNode =
+ # Search for the most nested child of node's type in tree
+ var
+ cnode = node
+ ntype = cnode.getName()
+ while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype:
+ cnode = cnode[0]
+ result = cnode
+
+proc inChildren*(node: TSNode, ntype: string): bool =
+ # Search for node type in immediate children
+ result = false
+ for i in 0 ..< node.len:
+ if (node[i]).getName() == ntype:
+ result = true
+ break
+
+proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
+ # Get line number and column info for node
+ let
+ point = node.tsNodeStartPoint()
+ result.line = point.row.int + 1
+ result.col = point.column.int + 1
+
+proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
+ getLineCol(gState.code, node)
+
+proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
+ # Get line number and column info for node
+ let
+ point = node.tsNodeEndPoint()
+ result.line = point.row.int + 1
+ result.col = point.column.int + 1
+
+proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
+ getEndLineCol(gState.code, node)
+
+proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
+ for i in 0 ..< node.len:
+ if node.getName() != "comment":
+ result += 1
+
+proc getPxName*(node: TSNode, offset: int): string =
+ # Get the xth (grand)parent of the node
+ var
+ np = node
+ count = 0
+
+ while not np.isNil and count < offset:
+ np = np.tsNodeParent()
+ count += 1
+
+ if count == offset and not np.isNil:
+ return np.getName()
+
+proc printLisp*(code: var string, root: TSNode): string =
+ var
+ node = root
+ nextnode: TSNode
+ depth = 0
+
+ while true:
+ if not node.isNil and depth > -1:
+ result &= spaces(depth)
+ let
+ (line, col) = code.getLineCol(node)
+ result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}"
+ let
+ val = code.getNodeVal(node)
+ if "\n" notin val and " " notin val:
+ result &= &" \"{val}\""
+ else:
+ break
+
+ if node.len() != 0:
+ result &= "\n"
+ nextnode = node[0]
+ depth += 1
+ else:
+ result &= ")\n"
+ nextnode = node.tsNodeNextNamedSibling()
+
+ if nextnode.isNil:
+ while true:
+ node = node.tsNodeParent()
+ depth -= 1
+ if depth == -1:
+ break
+ result &= spaces(depth) & ")\n"
+ if node == root:
+ break
+ if not node.tsNodeNextNamedSibling().isNil:
+ node = node.tsNodeNextNamedSibling()
+ break
+ else:
+ node = nextnode
+
+ if node == root:
+ break
+
+proc printLisp*(gState: State, root: TSNode): string =
+ printLisp(gState.code, root)
+
+proc printDebug*(gState: State, node: TSNode) =
+ if gState.debug:
+ gecho ("Input => " & gState.getNodeVal(node)).getCommented()
+ gecho gState.printLisp(node).getCommented()
+
+proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string =
+ ## Generate a comment from a set of comment nodes. Comment is guaranteed
+ ## to be able to be rendered using nim doc
+ if commentNodes.len > 0:
+ result = "::"
+ for commentNode in commentNodes:
+ result &= "\n " & gState.getNodeVal(commentNode).strip()
+
+ result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "")
+ result = result.multiReplace([("\n", "\n "), ("`", "")]).strip()
+
+proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] =
+ ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch``
+ ## nodes before and after the current node
+ ##
+ ## Priority is (closest line number) > comment before > comment after.
+ ## This priority might need to be changed based on the project, but
+ ## for now it is good enough
+
+ # Skip this if we don't want comments
+ if gState.noComments:
+ return
+
+ let (line, _) = gState.getLineCol(node)
+
+ # Keep track of both directions from a node
+ var
+ prevSibling = node.tsNodePrevNamedSibling()
+ nextSibling = node.tsNodeNextNamedSibling()
+ nilNode: TSNode
+
+ var
+ i = 0
+ prevSiblingDistance, nextSiblingDistance: int = int.high
+ lowestDistance: int
+ commentsFound = false
+
+ while not commentsFound and i < maxSearch:
+ # Distance from the current node will tell us approximately if the
+ # comment belongs to the node. The closer it is in terms of line
+ # numbers, the more we can be sure it's the comment we want
+ if not prevSibling.isNil:
+ if prevSibling.getName() == "comment":
+ prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line)
+ else:
+ prevSiblingDistance = int.high
+ if not nextSibling.isNil:
+ if nextSibling.getName() == "comment":
+ nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line)
+ else:
+ nextSiblingDistance = int.high
+
+ lowestDistance = min(prevSiblingDistance, nextSiblingDistance)
+
+ if prevSiblingDistance > maxSearch:
+ # If the line is out of range, skip searching
+ prevSibling = nilNode # Can't do `= nil`
+
+ if nextSiblingDistance > maxSearch:
+ # If the line is out of range, skip searching
+ nextSibling = nilNode
+
+ # Search above the current line for comments. When one is found
+ # keep going to retrieve successive comments for cases with multiple
+ # `//` style comments
+ while (
+ not prevSibling.isNil and
+ prevSibling.getName() == "comment" and
+ prevSiblingDistance == lowestDistance
+ ):
+ # Put the previous nodes in reverse order so the comments
+ # make logical sense
+ result.insert(prevSibling, 0)
+ prevSibling = prevSibling.tsNodePrevNamedSibling()
+ commentsFound = true
+
+ # If we've already found comments above the current line, quit
+ if commentsFound:
+ break
+
+ # Search below or at the current line for comments. When one is found
+ # keep going to retrieve successive comments for cases with multiple
+ # `//` style comments
+ while (
+ not nextSibling.isNil and
+ nextSibling.getName() == "comment" and
+ nextSiblingDistance == lowestDistance
+ ):
+ result.add(nextSibling)
+ nextSibling = nextSibling.tsNodeNextNamedSibling()
+ commentsFound = true
+
+ if commentsFound:
+ break
+
+ # Go to next sibling pair
+ if not prevSibling.isNil:
+ prevSibling = prevSibling.tsNodePrevNamedSibling()
+ if not nextSibling.isNil:
+ nextSibling = nextSibling.tsNodeNextNamedSibling()
+
+ i += 1
+
+proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
+ if node.tsNodeNamedChildCount() != 0:
+ for i in 0 .. node.tsNodeNamedChildCount()-1:
+ let
+ name = $node.tsNodeNamedChild(i).tsNodeType()
+
+ if name != "comment":
+ result.add(name)
diff --git a/tests/include/tast2.h b/tests/include/tast2.h
index 0b1fca2..878a3a2 100644
--- a/tests/include/tast2.h
+++ b/tests/include/tast2.h
@@ -247,7 +247,7 @@ static inline int sitest1(int f1) {
// DUPLICATES
-#ifndef HEADER
+#ifdef NOHEADER
#define A 1
#define B 1.0
diff --git a/tests/tast2.nim b/tests/tast2.nim
index 6cdd606..bd040e2 100644
--- a/tests/tast2.nim
+++ b/tests/tast2.nim
@@ -11,17 +11,17 @@ static:
const
path = currentSourcePath.parentDir() / "include" / "tast2.h"
-when defined(HEADER):
- cDefine("HEADER")
+when defined(NOHEADER):
+ cDefine("NOHEADER")
const
flags = " -H"
- pHeader = @["header:" & path.replace("\\", "/")]
- pHeaderImp = @["importc"] & pHeader
+ pHeader: seq[string] = @[]
+ pHeaderImp: seq[string] = @[]
else:
const
flags = ""
- pHeader: seq[string] = @[]
- pHeaderImp: seq[string] = @[]
+ pHeader = @["header:" & path.replace("\\", "/")]
+ pHeaderImp = @["importc"] & pHeader
const
pHeaderImpBy = @["bycopy"] & pHeaderImp
@@ -68,7 +68,7 @@ macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool]
ast = t.getImpl()
prag = ast.getPragmas()
exprag = pragmas.toHashSet()
- when defined(HEADER):
+ when not defined(NOHEADER):
if not istype:
if "union" in exprag:
exprag.incl "importc:union " & $t
@@ -169,7 +169,7 @@ a1.f1 = 2
assert A2 is object
testFields(A2)
checkPragmas(A2, pHeaderInc, istype = false)
-when not defined(HEADER):
+when defined(NOHEADER):
# typedef struct X; is invalid
var a2: A2
@@ -475,6 +475,6 @@ assert nested is object
testFields(nested, "f1|f2|f3|f4|f5|f6|f7|f8!NT1|Type_tast2h1|NT3|Type_tast2h3|NU2|Union_tast2h1|NE1|Enum_tast2h2")
checkPragmas(nested, pHeaderImpBy)
-when defined(HEADER):
+when not defined(NOHEADER):
assert sitest1(5) == 10
assert sitest1(10) == 20
diff --git a/tests/tpcre.nim b/tests/tpcre.nim
index 51530c1..4a3badb 100644
--- a/tests/tpcre.nim
+++ b/tests/tpcre.nim
@@ -28,6 +28,8 @@ cPlugin:
proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
sym.name = sym.name.replace("pcre_", "")
+ if sym.name.startsWith("pcre16_") or sym.name.startsWith("pcre32_"):
+ sym.name = ""
const FLAGS {.strdefine.} = ""
cImport(pcreH, dynlib="dynpcre", flags="--mode=c " & FLAGS)