diff options
| author | Oskari Timperi <oskari.timperi@iki.fi> | 2018-04-07 00:19:17 +0300 |
|---|---|---|
| committer | Oskari Timperi <oskari.timperi@iki.fi> | 2018-04-07 00:19:17 +0300 |
| commit | 5bc364b94c90b1e6e3910bd29adc75e0affa0d10 (patch) | |
| tree | 32ecc1c3fe1af5a23afade2c4c31bfad5cce9b70 | |
| parent | 4b6946251d36b16048d6b5aa4aee493313cf7ca9 (diff) | |
| download | nimpb-5bc364b94c90b1e6e3910bd29adc75e0affa0d10.tar.gz nimpb-5bc364b94c90b1e6e3910bd29adc75e0affa0d10.zip | |
Remove generator in favor of nimpb_build
| -rw-r--r-- | README.md | 8 | ||||
| -rw-r--r-- | generator/README.md | 15 | ||||
| -rw-r--r-- | generator/descriptor_pb.nim | 368 | ||||
| -rw-r--r-- | generator/gen.nim | 655 | ||||
| -rw-r--r-- | generator/nim.cfg | 1 | ||||
| -rw-r--r-- | generator/plugin_pb.nim | 160 | ||||
| -rwxr-xr-x | generator/protoc-gen-nim | 3 | ||||
| -rw-r--r-- | generator/protoc_gen_nim.nim | 922 |
8 files changed, 5 insertions, 2127 deletions
@@ -1,8 +1,10 @@ # Protocol Buffers for Nim -A Nim library to serialize/deserialize Protocol Buffers and a `protoc` plugin for generating Nim code from `.proto` files. +A Nim library to serialize/deserialize Protocol Buffers. -At the moment this is at a very rough state. Do not use for any kind of production use. Anything can change at any time. You've been warned. +For generating Nim code usable with nimpb, you should use [nimpb_build](https://github.com/oswjk/nimpb-build). + +**NOTE** At the moment this is at a very rough state. Do not use for any kind of production use. Anything can change at any time. You've been warned. # Example @@ -23,7 +25,7 @@ message Test1 { } ``` -The `protoc` plugin will generate the following types (and procs for interacting with them): +You can use [nimpb_build](https://github.com/oswjk/nimpb-build) to generate code like this (procs not included in the example): ```nim type diff --git a/generator/README.md b/generator/README.md deleted file mode 100644 index be99e6f..0000000 --- a/generator/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# protoc_gen_nim - -A protoc plugin for generating Nim code out of .proto files. - -# Usage - -If you don't have the directory, where the generator plugin resides, in PATH, you can use - - protoc --plugin=protoc-gen-nim=/path/to/protoc_gen_nim --nim_out=protos -I. my.proto - -If the plugin is in PATH (and you're on a unix-ish system) then it's a bit easier - - protoc -I. --nim_out=protos my.proto - -The latter style is possible, because there exists a `protoc-gen-nim` wrapper script that calls the real executable. Protoc automatically finds this wrapper based on the `--nim_out` argument. diff --git a/generator/descriptor_pb.nim b/generator/descriptor_pb.nim deleted file mode 100644 index 923885f..0000000 --- a/generator/descriptor_pb.nim +++ /dev/null @@ -1,368 +0,0 @@ -import intsets - -import gen -import nimpb/nimpb - -const - FileDescriptorSetDesc = MessageDesc( - name: "FileDescriptorSet", - fields: @[ - FieldDesc( - name: "files", - number: 1, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "FileDescriptorProto", - packed: false, - oneofIdx: -1, - ) - ] - ) - - FileDescriptorProtoDesc = MessageDesc( - name: "FileDescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "package", - number: 2, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "dependency", - number: 3, - ftype: FieldType.String, - label: FieldLabel.Repeated, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "message_type", - number: 4, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "DescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "enum_type", - number: 5, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "EnumDescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "syntax", - number: 12, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - - DescriptorProtoDesc = MessageDesc( - name: "DescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "field", - number: 2, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "FieldDescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "nested_type", - number: 3, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "DescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "enum_type", - number: 4, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "EnumDescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "oneof_decl", - number: 8, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "OneofDescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "options", - number: 7, - ftype: FieldType.Message, - label: FieldLabel.Optional, - typeName: "MessageOptions", - packed: false, - oneofIdx: -1, - ), - ] - ) - - EnumDescriptorProtoDesc = MessageDesc( - name: "EnumDescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "value", - number: 2, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "EnumValueDescriptorProto", - packed: false, - oneofIdx: -1, - ), - ] - ) - - EnumValueDescriptorProtoDesc = MessageDesc( - name: "EnumValueDescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "number", - number: 2, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - - FieldDescriptorProtoDesc = MessageDesc( - name: "FieldDescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "number", - number: 3, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "label", - number: 4, - ftype: FieldType.Enum, - label: FieldLabel.Optional, - typeName: "FieldDescriptorProto_Label", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "type", - number: 5, - ftype: FieldType.Enum, - label: FieldLabel.Optional, - typeName: "FieldDescriptorProto_Type", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "type_name", - number: 6, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "options", - number: 8, - ftype: FieldType.Message, - label: FieldLabel.Optional, - typeName: "FieldOptions", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "oneof_index", - number: 9, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - - FieldDescriptorProto_LabelDesc = EnumDesc( - name: "FieldDescriptorProto_Label", - values: @[ - EnumValueDesc(name: "LABEL_OPTIONAL", number: 1), - EnumValueDesc(name: "LABEL_REQUIRED", number: 2), - EnumValueDesc(name: "LABEL_REPEATED", number: 3) - ] - ) - - FieldDescriptorProto_TypeDesc = EnumDesc( - name: "FieldDescriptorProto_Type", - values: @[ - EnumValueDesc(name: "TYPE_DOUBLE", number: 1), - EnumValueDesc(name: "TYPE_FLOAT", number: 2), - EnumValueDesc(name: "TYPE_INT64", number: 3), - EnumValueDesc(name: "TYPE_UINT64", number: 4), - EnumValueDesc(name: "TYPE_INT32", number: 5), - EnumValueDesc(name: "TYPE_FIXED64", number: 6), - EnumValueDesc(name: "TYPE_FIXED32", number: 7), - EnumValueDesc(name: "TYPE_BOOL", number: 8), - EnumValueDesc(name: "TYPE_STRING", number: 9), - EnumValueDesc(name: "TYPE_GROUP", number: 10), - EnumValueDesc(name: "TYPE_MESSAGE", number: 11), - EnumValueDesc(name: "TYPE_BYTES", number: 12), - EnumValueDesc(name: "TYPE_UINT32", number: 13), - EnumValueDesc(name: "TYPE_ENUM", number: 14), - EnumValueDesc(name: "TYPE_SFIXED32", number: 15), - EnumValueDesc(name: "TYPE_SFIXED64", number: 16), - EnumValueDesc(name: "TYPE_SINT32", number: 17), - EnumValueDesc(name: "TYPE_SINT64", number: 18), - ] - ) - - MessageOptionsDesc = MessageDesc( - name: "MessageOptions", - fields: @[ - FieldDesc( - name: "map_entry", - number: 7, - ftype: FieldType.Bool, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - - FieldOptionsDesc = MessageDesc( - name: "FieldOptions", - fields: @[ - FieldDesc( - name: "packed", - number: 2, - ftype: FieldType.Bool, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - - OneofDescriptorProtoDesc = MessageDesc( - name: "OneofDescriptorProto", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - -generateEnumType(FieldDescriptorProto_LabelDesc) -generateEnumProcs(FieldDescriptorProto_LabelDesc) - -generateEnumType(FieldDescriptorProto_TypeDesc) -generateEnumProcs(FieldDescriptorProto_TypeDesc) - -generateMessageType(EnumValueDescriptorProtoDesc) -generateMessageProcs(EnumValueDescriptorProtoDesc) - -generateMessageType(EnumDescriptorProtoDesc) -generateMessageProcs(EnumDescriptorProtoDesc) - -generateMessageType(FieldOptionsDesc) -generateMessageProcs(FieldOptionsDesc) - -generateMessageType(FieldDescriptorProtoDesc) -generateMessageProcs(FieldDescriptorProtoDesc) - -generateMessageType(OneofDescriptorProtoDesc) -generateMessageProcs(OneofDescriptorProtoDesc) - -generateMessageType(MessageOptionsDesc) -generateMessageProcs(MessageOptionsDesc) - -generateMessageType(DescriptorProtoDesc) -generateMessageProcs(DescriptorProtoDesc) - -generateMessageType(FileDescriptorProtoDesc) -generateMessageProcs(FileDescriptorProtoDesc) - -generateMessageType(FileDescriptorSetDesc) -generateMessageProcs(FileDescriptorSetDesc) diff --git a/generator/gen.nim b/generator/gen.nim deleted file mode 100644 index e954d03..0000000 --- a/generator/gen.nim +++ /dev/null @@ -1,655 +0,0 @@ -import macros -import strutils - -import nimpb/nimpb - -type - MessageDesc* = object - name*: string - fields*: seq[FieldDesc] - oneofs*: seq[string] - - FieldLabel* {.pure.} = enum - Optional = 1 - Required - Repeated - - FieldDesc* = object - name*: string - number*: int - ftype*: FieldType - label*: FieldLabel - typeName*: string - packed*: bool - oneofIdx*: int - - EnumDesc* = object - name*: string - values*: seq[EnumValueDesc] - - EnumValueDesc* = object - name*: string - number*: int - -proc findColonExpr(parent: NimNode, s: string): NimNode = - for child in parent: - if child.kind != nnkExprColonExpr: - continue - - if $child[0] == s: - return child - -proc getMessageName(desc: NimNode): string = - let node = findColonExpr(desc, "name") - result = $node[1] - -iterator fields(desc: NimNode): NimNode = - let node = findColonExpr(desc, "fields") - for field in node[1]: - yield field - -proc isRepeated(field: NimNode): bool = - let node = findColonExpr(field, "label") - let value = FieldLabel(node[1].intVal) - result = value == FieldLabel.Repeated - -proc isPacked(field: NimNode): bool = - let node = findColonExpr(field, "packed") - result = bool(node[1].intVal) - -proc getFieldType(field: NimNode): FieldType = - let node = findColonExpr(field, "ftype") - result = FieldType(node[1].intVal) - -proc isMessage(field: NimNode): bool = - result = getFieldType(field) == FieldType.Message - -proc isEnum(field: NimNode): bool = - result = getFieldType(field) == FieldType.Enum - -proc getFieldTypeName(field: NimNode): string = - let node = findColonExpr(field, "typeName") - result = $node[1] - -proc getFieldTypeAsString(field: NimNode): string = - if isMessage(field) or isEnum(field): - result = getFieldTypeName(field) - else: - case getFieldType(field) - of FieldType.Double: result = "float64" - of FieldType.Float: result = "float32" - of FieldType.Int64: result = "int64" - of FieldType.UInt64: result = "uint64" - of FieldType.Int32: result = "int32" - of FieldType.Fixed64: result = "uint64" - of FieldType.Fixed32: result = "uint32" - of FieldType.Bool: result = "bool" - of FieldType.String: result = "string" - of FieldType.Bytes: result = "bytes" - of FieldType.UInt32: result = "uint32" - of FieldType.SFixed32: result = "int32" - of FieldType.SFixed64: result = "int64" - of FieldType.SInt32: result = "int32" - of FieldType.SInt64: result = "int64" - else: result = "AYBABTU" - -proc getFullFieldType(field: NimNode): NimNode = - result = ident(getFieldTypeAsString(field)) - if isRepeated(field): - result = nnkBracketExpr.newTree(ident("seq"), result) - -proc getFieldName(field: NimNode): string = - let node = findColonExpr(field, "name") - result = $node[1] - -proc getFieldNumber(field: NimNode): int = - result = int(findColonExpr(field, "number")[1].intVal) - -proc defaultValue(field: NimNode): NimNode = - # TODO: check if there is a default value specified for the field - - if isRepeated(field): - return nnkPrefix.newTree(newIdentNode("@"), nnkBracket.newTree()) - - case getFieldType(field) - of FieldType.Double: result = newLit(0.0'f64) - of FieldType.Float: result = newLit(0.0'f32) - of FieldType.Int64: result = newLit(0'i64) - of FieldType.UInt64: result = newLit(0'u64) - of FieldType.Int32: result = newLit(0'i32) - of FieldType.Fixed64: result = newLit(0'u64) - of FieldType.Fixed32: result = newLit(0'u32) - of FieldType.Bool: result = newLit(false) - of FieldType.String: result = newLit("") - of FieldType.Group: result = newLit("NOTIMPLEMENTED") - of FieldType.Message: result = newCall(ident("new" & getFieldTypeAsString(field))) - of FieldType.Bytes: result = newCall(ident("bytes"), newLit("")) - of FieldType.UInt32: result = newLit(0'u32) - of FieldType.Enum: - let - descId = ident(getFieldTypeAsString(field) & "Desc") - nameId = ident(getFieldTypeAsString(field)) - result = quote do: - `nameId`(`descId`.values[0].number) - of FieldType.SFixed32: result = newLit(0'u32) - of FieldType.SFixed64: result = newLit(0'u32) - of FieldType.SInt32: result = newLit(0) - of FieldType.SInt64: result = newLit(0) - -proc wiretype(field: NimNode): WireType = - result = wiretype(getFieldType(field)) - -# TODO: maybe not the best name for this -proc getFieldNameAST(objname: NimNode, field: NimNode, oneof: string): NimNode = - result = - if oneof != "": - newDotExpr(newDotExpr(objname, ident(oneof)), ident(getFieldName(field))) - else: - newDotExpr(objname, ident(getFieldName(field))) - -proc fieldInitializer(objname: NimNode, field: NimNode, oneof: string): NimNode = - result = nnkAsgn.newTree( - getFieldNameAST(objname, field, oneof), - defaultValue(field) - ) - -proc oneofIndex(field: NimNode): int = - let node = findColonExpr(field, "oneofIdx") - if node == nil: - result = -1 - else: - result = int(node[1].intVal) - -proc oneofName(message, field: NimNode): string = - let index = oneofIndex(field) - - if index == -1: - return "" - - let oneofs = findColonExpr(message, "oneofs")[1] - - result = $oneofs[index] - -iterator oneofFields(message: NimNode, index: int): NimNode = - if index != -1: - for field in fields(message): - if oneofIndex(field) == index: - yield field - -proc generateOneofFields*(desc: NimNode, typeSection: NimNode) = - let - oneofs = findColonExpr(desc, "oneofs")[1] - messageName = getMessageName(desc) - - for index, oneof in oneofs: - let reclist = nnkRecList.newTree() - - for field in oneofFields(desc, index): - let ftype = getFullFieldType(field) - let name = ident(getFieldName(field)) - - add(reclist, newIdentDefs(postfix(name, "*"), ftype)) - - let typedef = nnkTypeDef.newTree( - nnkPragmaExpr.newTree( - postfix(ident(messageName & $oneof), "*"), - nnkPragma.newTree( - ident("union") - ) - ), - newEmptyNode(), - nnkObjectTy.newTree( - newEmptyNode(), - newEmptyNode(), - reclist - ) - ) - - add(typeSection, typedef) - -macro generateMessageType*(desc: typed): typed = - let - impl = getImpl(symbol(desc)) - typeSection = nnkTypeSection.newTree() - typedef = nnkTypeDef.newTree() - reclist = nnkRecList.newTree() - oneofs = findColonExpr(impl, "oneofs")[1] - - let name = getMessageName(impl) - - let typedefRef = nnkTypeDef.newTree(postfix(newIdentNode(name), "*"), newEmptyNode(), - nnkRefTy.newTree(newIdentNode(name & "Obj"))) - add(typeSection, typedefRef) - - add(typeSection, typedef) - - add(typedef, postfix(ident(name & "Obj"), "*")) - add(typedef, newEmptyNode()) - add(typedef, nnkObjectTy.newTree(newEmptyNode(), newEmptyNode(), reclist)) - - for field in fields(impl): - let ftype = getFullFieldType(field) - let name = ident(getFieldName(field)) - if oneofIndex(field) == -1: - add(reclist, newIdentDefs(postfix(name, "*"), ftype)) - - for oneof in oneofs: - add(reclist, newIdentDefs(postfix(ident($oneof), "*"), - ident(name & $oneof))) - - add(reclist, nnkIdentDefs.newTree( - ident("hasField"), ident("IntSet"), newEmptyNode())) - - generateOneofFields(impl, typeSection) - - result = newStmtList() - add(result, typeSection) - - when defined(debug): - hint(repr(result)) - -proc generateNewMessageProc(desc: NimNode): NimNode = - let - body = newStmtList( - newCall(ident("new"), ident("result")) - ) - resultId = ident("result") - - for field in fields(desc): - let oneofName = oneofName(desc, field) - add(body, fieldInitializer(resultId, field, oneofName)) - - add(body, newAssignment(newDotExpr(resultId, ident("hasField")), - newCall(ident("initIntSet")))) - - result = newProc(postfix(ident("new" & getMessageName(desc)), "*"), - @[ident(getMessageName(desc))], - body) - -proc fieldProcName(prefix: string, field: NimNode): string = - result = prefix & capitalizeAscii(getFieldName(field)) - -proc fieldProcIdent(prefix: string, field: NimNode): NimNode = - result = postfix(ident(fieldProcName(prefix, field)), "*") - -proc generateClearFieldProc(desc, field: NimNode): NimNode = - let - messageId = ident("message") - fname = getFieldNameAST(messageId, field, oneofName(desc, field)) - defvalue = defaultValue(field) - hasField = newDotExpr(messageId, ident("hasField")) - number = getFieldNumber(field) - procName = fieldProcIdent("clear", field) - mtype = ident(getMessageName(desc)) - - result = quote do: - proc `procName`(`messageId`: `mtype`) = - `fname` = `defvalue` - excl(`hasfield`, `number`) - - # When clearing a field that is contained in a oneof, we should also clear - # the other fields. - for sibling in oneofFields(desc, oneofIndex(field)): - if sibling == field: - continue - let - number = getFieldNumber(sibling) - exclNode = quote do: - excl(`hasField`, `number`) - add(body(result), exclNode) - -proc generateHasFieldProc(desc, field: NimNode): NimNode = - let - messageId = ident("message") - hasField = newDotExpr(messageId, ident("hasField")) - number = getFieldNumber(field) - mtype = ident(getMessageName(desc)) - procName = fieldProcIdent("has", field) - - result = quote do: - proc `procName`(`messageId`: `mtype`): bool = - contains(`hasfield`, `number`) - -proc generateSetFieldProc(desc, field: NimNode): NimNode = - let - messageId = ident("message") - hasField = newDotExpr(messageId, ident("hasField")) - number = getFieldNumber(field) - valueId = ident("value") - fname = getFieldNameAST(messageId, field, oneofName(desc, field)) - procName = fieldProcIdent("set", field) - mtype = ident(getMessageName(desc)) - ftype = getFullFieldType(field) - - result = quote do: - proc `procName`(`messageId`: `mtype`, `valueId`: `ftype`) = - `fname` = `valueId` - incl(`hasfield`, `number`) - - # When setting a field that is in a oneof, we need to unset the other fields - for sibling in oneofFields(desc, oneofIndex(field)): - if sibling == field: - continue - let - number = getFieldNumber(sibling) - exclNode = quote do: - excl(`hasField`, `number`) - add(body(result), exclNode) - -proc generateAddToFieldProc(desc, field: NimNode): NimNode = - let - procName = fieldProcIdent("add", field) - messageId = ident("message") - mtype = ident(getMessageName(desc)) - valueId = ident("value") - ftype = ident(getFieldTypeAsString(field)) - hasField = newDotExpr(messageId, ident("hasField")) - number = getFieldNumber(field) - fname = newDotExpr(messageId, ident(getFieldName(field))) - - result = quote do: - proc `procName`(`messageId`: `mtype`, `valueId`: `ftype`) = - add(`fname`, `valueId`) - incl(`hasfield`, `number`) - -proc ident(wt: WireType): NimNode = - result = newDotExpr(ident("WireType"), ident($wt)) - -proc genWriteField(message, field: NimNode): NimNode = - result = newStmtList() - - let - number = getFieldNumber(field) - writer = ident("write" & getFieldTypeAsString(field)) - messageId = ident("message") - fname = getFieldNameAST(messageId, field, oneofName(message, field)) - wiretype = ident(wiretype(field)) - sizeproc = ident("sizeOf" & getFieldTypeAsString(field)) - hasproc = ident(fieldProcName("has", field)) - - if not isRepeated(field): - result.add quote do: - if `hasproc`(message): - writeTag(stream, `number`, `wiretype`) - `writer`(stream, `fname`) - if isMessage(field): - insert(result[0][0][1], 1, quote do: - writeVarint(stream, `sizeproc`(`fname`)) - ) - else: - let valueId = ident("value") - if isPacked(field): - result.add quote do: - writeTag(stream, `number`, WireType.LengthDelimited) - writeVarInt(stream, packedFieldSize(`fname`, `wiretype`)) - for `valueId` in `fname`: - `writer`(stream, `valueId`) - else: - result.add quote do: - for `valueId` in `fname`: - writeTag(stream, `number`, `wiretype`) - `writer`(stream, `valueId`) - if isMessage(field): - insert(result[^1][^1], 1, quote do: - writeVarint(stream, `sizeproc`(`valueId`)) - ) - -proc generateWriteMessageProc(desc: NimNode): NimNode = - let - messageId = ident("message") - mtype = ident(getMessageName(desc)) - procName = postfix(ident("write" & getMessageName(desc)), "*") - body = newStmtList() - stream = ident("stream") - sizeproc = postfix(ident("sizeOf" & getMessageName(desc)), "*") - - for field in fields(desc): - add(body, genWriteField(desc, field)) - - result = quote do: - proc `sizeproc`(`messageId`: `mtype`): uint64 - - proc `procName`(`stream`: ProtobufStream, `messageId`: `mtype`) = - `body` - -proc generateReadMessageProc(desc: NimNode): NimNode = - let - procName = postfix(ident("read" & getMessageName(desc)), "*") - newproc = ident("new" & getMessageName(desc)) - streamId = ident("stream") - mtype = ident(getMessageName(desc)) - tagId = ident("tag") - wiretypeId = ident("wiretype") - resultId = ident("result") - - result = quote do: - proc `procName`(`streamId`: ProtobufStream): `mtype` = - `resultId` = `newproc`() - while not atEnd(stream): - let - `tagId` = readTag(`streamId`) - `wiretypeId` = wireType(`tagId`) - case fieldNumber(`tagId`) - else: - skipField(`streamId`, `wiretypeId`) - - let caseNode = body(result)[1][1][1] - - # TODO: check wiretypes and fail if it doesn't match - for field in fields(desc): - let - number = getFieldNumber(field) - reader = ident("read" & getFieldTypeAsString(field)) - setproc = - if isRepeated(field): - ident("add" & capitalizeAscii(getFieldName(field))) - else: - ident("set" & capitalizeAscii(getFieldName(field))) - if isRepeated(field): - if isNumeric(getFieldType(field)): - insert(caseNode, 1, nnkOfBranch.newTree(newLit(number), quote do: - if `wiretypeId` == WireType.LengthDelimited: - let - size = readVarint(stream) - start = getPosition(stream).uint64 - var consumed = 0'u64 - while consumed < size: - `setproc`(`resultId`, `reader`(stream)) - consumed = getPosition(stream).uint64 - start - if consumed != size: - raise newException(Exception, "packed field size mismatch") - else: - `setproc`(`resultId`, `reader`(stream)) - )) - elif isMessage(field): - insert(caseNode, 1, nnkOfBranch.newTree(newLit(number), quote do: - let size = readVarint(stream) - let data = readStr(stream, int(size)) - let stream2 = newProtobufStream(newStringStream(data)) - `setproc`(`resultId`, `reader`(stream2)) - )) - else: - insert(caseNode, 1, nnkOfBranch.newTree(newLit(number), quote do: - `setproc`(`resultId`, `reader`(stream)) - )) - else: - if isMessage(field): - insert(caseNode, 1, nnkOfBranch.newTree(newLit(number), quote do: - let size = readVarint(stream) - let data = readStr(stream, int(size)) - let stream2 = newProtobufStream(newStringStream(data)) - `setproc`(`resultId`, `reader`(stream2)) - )) - else: - insert(caseNode, 1, nnkOfBranch.newTree(newLit(number), quote do: - `setproc`(`resultId`, `reader`(stream)) - )) - -proc generateSizeOfMessageProc(desc: NimNode): NimNode = - let - name = getMessageName(desc) - body = newStmtList() - messageId = ident("message") - resultId = ident("result") - procName = postfix(ident("sizeOf" & getMessageName(desc)), "*") - mtype = ident(getMessageName(desc)) - - result = quote do: - proc `procName`(`messageId`: `mtype`): uint64 = - `resultId` = 0 - - let procBody = body(result) - - for field in fields(desc): - let - hasproc = ident(fieldProcName("has", field)) - sizeofproc = ident("sizeOf" & getFieldTypeAsString(field)) - fname = getFieldNameAST(messageId, field, oneofName(desc, field)) - number = getFieldNumber(field) - wiretype = ident(wiretype(field)) - - # TODO: packed - if isRepeated(field): - if isPacked(field): - procBody.add quote do: - if `hasproc`(`messageId`): - let - tagSize = sizeOfUint32(uint32(makeTag(`number`, WireType.LengthDelimited))) - dataSize = packedFieldSize(`fname`, `wiretype`) - sizeOfSize = sizeOfUint64(dataSize) - `resultId` = tagSize + dataSize + sizeOfSize - else: - procBody.add quote do: - for value in `fname`: - let - sizeOfField = `sizeofproc`(value) - tagSize = sizeOfUint32(uint32(makeTag(`number`, `wiretype`))) - `resultId` = `resultId` + - sizeOfField + - sizeOfUint64(sizeOfField) + - tagSize - else: - let sizeOfFieldId = ident("sizeOfField") - - procBody.add quote do: - if `hasproc`(`messageId`): - let - `sizeOfFieldId` = `sizeofproc`(`fname`) - tagSize = sizeOfUint32(uint32(makeTag(`number`, `wiretype`))) - `resultId` = `resultId` + sizeOfField + tagSize - - if isMessage(field): - # For messages we need to include the size of the encoded size - let asgn = procBody[^1][0][1][1] - asgn[1] = infix(asgn[1], "+", newCall(ident("sizeOfUint64"), - sizeOfFieldId)) - -proc generateSerializeProc(desc: NimNode): NimNode = - let - mtype = ident(getMessageName(desc)) - procName = postfix(ident("serialize"), "*") - writer = ident("write" & getMessageName(desc)) - resultId = ident("result") - - result = quote do: - proc `procName`(message: `mtype`): string = - let - ss = newStringStream() - pbs = newProtobufStream(ss) - `writer`(pbs, message) - `resultId` = ss.data - -proc generateDeserializeProc(desc: NimNode): NimNode = - let - mtype = ident(getMessageName(desc)) - procName = postfix(ident("new" & getMessageName(desc)), "*") - reader = ident("read" & getMessageName(desc)) - resultId = ident("result") - - result = quote do: - proc `procName`(data: string): `mtype` = - let - ss = newStringStream(data) - pbs = newProtobufStream(ss) - `resultId` = `reader`(pbs) - -macro generateMessageProcs*(x: typed): typed = - let - desc = getImpl(symbol(x)) - - result = newStmtList( - generateNewMessageProc(desc), - ) - - for field in fields(desc): - add(result, generateClearFieldProc(desc, field)) - add(result, generateHasFieldProc(desc, field)) - add(result, generateSetFieldProc(desc, field)) - - if isRepeated(field): - add(result, generateAddToFieldProc(desc, field)) - - add(result, generateWriteMessageProc(desc)) - add(result, generateReadMessageProc(desc)) - add(result, generateSizeOfMessageProc(desc)) - add(result, generateSerializeProc(desc)) - add(result, generateDeserializeProc(desc)) - - when defined(debug): - hint(repr(result)) - -macro generateEnumType*(x: typed): typed = - let - impl = getImpl(symbol(x)) - name = $findColonExpr(impl, "name")[1] - values = findColonExpr(impl, "values")[1] - - let enumTy = nnkEnumTy.newTree(newEmptyNode()) - - for valueNode in values: - let - name = $findColonExpr(valueNode, "name")[1] - number = findColonExpr(valueNode, "number")[1] - - add(enumTy, nnkEnumFieldDef.newTree(ident(name), number)) - - result = newStmtList(nnkTypeSection.newTree( - nnkTypeDef.newTree( - nnkPragmaExpr.newTree( - postfix(ident(name), "*"), - nnkPragma.newTree(ident("pure")) - ), - newEmptyNode(), - enumTy - ) - )) - - when defined(debug): - hint(repr(result)) - -macro generateEnumProcs*(x: typed): typed = - let - impl = getImpl(symbol(x)) - name = $findColonExpr(impl, "name")[1] - nameId = ident(name) - values = findColonExpr(impl, "values")[1] - readProc = postfix(ident("read" & name), "*") - writeProc = postfix(ident("write" & name), "*") - sizeProc = postfix(ident("sizeOf" & name), "*") - resultId = ident("result") - - result = newStmtList() - - add(result, quote do: - proc `readProc`(stream: ProtobufStream): `nameId` = - `resultId` = `nameId`(readUInt32(stream)) - - proc `writeProc`(stream: ProtobufStream, value: `nameId`) = - writeEnum(stream, value) - - proc `sizeProc`(value: `nameId`): uint64 = - `resultId` = sizeOfUInt32(uint32(value)) - ) - - when defined(debug): - hint(repr(result)) diff --git a/generator/nim.cfg b/generator/nim.cfg deleted file mode 100644 index a119208..0000000 --- a/generator/nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---path:"../src" diff --git a/generator/plugin_pb.nim b/generator/plugin_pb.nim deleted file mode 100644 index fa9fc94..0000000 --- a/generator/plugin_pb.nim +++ /dev/null @@ -1,160 +0,0 @@ -import intsets - -import gen -import nimpb/nimpb - -import descriptor_pb - -const - VersionDesc = MessageDesc( - name: "Version", - fields: @[ - FieldDesc( - name: "major", - number: 1, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "minor", - number: 2, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "patch", - number: 3, - ftype: FieldType.Int32, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "suffix", - number: 4, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ) - ] - ) - - CodeGeneratorRequestDesc = MessageDesc( - name: "CodeGeneratorRequest", - fields: @[ - FieldDesc( - name: "file_to_generate", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Repeated, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "parameter", - number: 2, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "proto_file", - number: 15, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "FileDescriptorProto", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "compiler_version", - number: 3, - ftype: FieldType.Message, - label: FieldLabel.Optional, - typeName: "Version", - packed: false, - oneofIdx: -1, - ) - ] - ) - - CodeGeneratorResponseDesc = MessageDesc( - name: "CodeGeneratorResponse", - fields: @[ - FieldDesc( - name: "error", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "file", - number: 15, - ftype: FieldType.Message, - label: FieldLabel.Repeated, - typeName: "CodeGeneratorResponse_File", - packed: false, - oneofIdx: -1, - ), - ] - ) - - CodeGeneratorResponse_FileDesc = MessageDesc( - name: "CodeGeneratorResponse_File", - fields: @[ - FieldDesc( - name: "name", - number: 1, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "insertion_point", - number: 2, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - FieldDesc( - name: "content", - number: 15, - ftype: FieldType.String, - label: FieldLabel.Optional, - typeName: "", - packed: false, - oneofIdx: -1, - ), - ] - ) - -generateMessageType(VersionDesc) -generateMessageProcs(VersionDesc) - -generateMessageType(CodeGeneratorRequestDesc) -generateMessageProcs(CodeGeneratorRequestDesc) - -generateMessageType(CodeGeneratorResponse_FileDesc) -generateMessageProcs(CodeGeneratorResponse_FileDesc) - -generateMessageType(CodeGeneratorResponseDesc) -generateMessageProcs(CodeGeneratorResponseDesc) diff --git a/generator/protoc-gen-nim b/generator/protoc-gen-nim deleted file mode 100755 index e3c526c..0000000 --- a/generator/protoc-gen-nim +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -MYPATH=$(dirname "$0") -exec "$MYPATH/protoc_gen_nim" diff --git a/generator/protoc_gen_nim.nim b/generator/protoc_gen_nim.nim deleted file mode 100644 index 62ef75d..0000000 --- a/generator/protoc_gen_nim.nim +++ /dev/null @@ -1,922 +0,0 @@ -import algorithm -import os -import pegs -import sequtils -import sets -import strformat -import strutils -import tables - -import descriptor_pb -import plugin_pb - -import nimpb/nimpb - -import gen - -type - Names = distinct seq[string] - - Enum = ref object - names: Names - values: seq[tuple[name: string, number: int]] - - Field = ref object - number: int - name: string - label: FieldDescriptorProto_Label - ftype: FieldDescriptorProto_Type - typeName: string - packed: bool - oneof: Oneof - mapEntry: Message - - Message = ref object - names: Names - fields: seq[Field] - oneofs: seq[Oneof] - mapEntry: bool - - Oneof = ref object - name: string - fields: seq[Field] - - ProcessedFile = ref object - name: string - data: string - - ProtoFile = ref object - fdesc: FileDescriptorProto - enums: seq[Enum] - messages: seq[Message] - syntax: Syntax - - Syntax {.pure.} = enum - Proto2 - Proto3 - -when defined(debug): - proc log(msg: string) = - stderr.write(msg) - stderr.write("\n") -else: - proc log(msg: string) = discard - -proc initNamesFromTypeName(typename: string): Names = - if typename[0] != '.': - raise newException(Exception, "relative names not supported") - let parts = split(typename[1..^1], ".") - result = Names(parts) - -proc `$`(names: Names): string = - let n = seq[string](names) - result = join(n, "_") - -proc add(names: var Names, s: string) = - add(seq[string](names), s) - -proc `&`(names: Names, s: string): Names = - result = names - add(result, s) - -proc isRepeated(field: Field): bool = - result = field.label == FieldDescriptorProtoLabel.LabelRepeated - -proc isMessage(field: Field): bool = - result = field.ftype == FieldDescriptorProtoType.TypeMessage - -proc isEnum(field: Field): bool = - result = field.ftype == FieldDescriptorProtoType.TypeEnum - -proc isNumeric(field: Field): bool = - case field.ftype - of FieldDescriptorProtoType.TypeDouble, FieldDescriptorProtoType.TypeFloat, - FieldDescriptorProtoType.TypeInt64, FieldDescriptorProtoType.TypeUInt64, - FieldDescriptorProtoType.TypeInt32, FieldDescriptorProtoType.TypeFixed64, - FieldDescriptorProtoType.TypeFixed32, FieldDescriptorProtoType.TypeBool, - FieldDescriptorProtoType.TypeUInt32, FieldDescriptorProtoType.TypeEnum, - FieldDescriptorProtoType.TypeSFixed32, FieldDescriptorProtoType.TypeSFixed64, - FieldDescriptorProtoType.TypeSInt32, FieldDescriptorProtoType.TypeSInt64: - result = true - else: discard - -proc isMapEntry(message: Message): bool = - result = message.mapEntry - -proc isMapEntry(field: Field): bool = - result = field.mapEntry != nil - -proc nimTypeName(field: Field): string = - case field.ftype - of FieldDescriptorProtoType.TypeDouble: result = "float64" - of FieldDescriptorProtoType.TypeFloat: result = "float32" - of FieldDescriptorProtoType.TypeInt64: result = "int64" - of FieldDescriptorProtoType.TypeUInt64: result = "uint64" - of FieldDescriptorProtoType.TypeInt32: result = "int32" - of FieldDescriptorProtoType.TypeFixed64: result = "uint64" - of FieldDescriptorProtoType.TypeFixed32: result = "uint32" - of FieldDescriptorProtoType.TypeBool: result = "bool" - of FieldDescriptorProtoType.TypeString: result = "string" - of FieldDescriptorProtoType.TypeGroup: result = "" - of FieldDescriptorProtoType.TypeMessage: result = field.typeName - of FieldDescriptorProtoType.TypeBytes: result = "bytes" - of FieldDescriptorProtoType.TypeUInt32: result = "uint32" - of FieldDescriptorProtoType.TypeEnum: result = field.typeName - of FieldDescriptorProtoType.TypeSFixed32: result = "int32" - of FieldDescriptorProtoType.TypeSFixed64: result = "int64" - of FieldDescriptorProtoType.TypeSInt32: result = "int32" - of FieldDescriptorProtoType.TypeSInt64: result = "int64" - -proc mapKeyType(field: Field): string = - for f in field.mapEntry.fields: - if f.name == "key": - return f.nimTypeName - -proc mapValueType(field: Field): string = - for f in field.mapEntry.fields: - if f.name == "value": - return f.nimTypeName - -proc `$`(ft: FieldDescriptorProtoType): string = - case ft - of FieldDescriptorProtoType.TypeDouble: result = "Double" - of FieldDescriptorProtoType.TypeFloat: result = "Float" - of FieldDescriptorProtoType.TypeInt64: result = "Int64" - of FieldDescriptorProtoType.TypeUInt64: result = "UInt64" - of FieldDescriptorProtoType.TypeInt32: result = "Int32" - of FieldDescriptorProtoType.TypeFixed64: result = "Fixed64" - of FieldDescriptorProtoType.TypeFixed32: result = "Fixed32" - of FieldDescriptorProtoType.TypeBool: result = "Bool" - of FieldDescriptorProtoType.TypeString: result = "String" - of FieldDescriptorProtoType.TypeGroup: result = "Group" - of FieldDescriptorProtoType.TypeMessage: result = "Message" - of FieldDescriptorProtoType.TypeBytes: result = "Bytes" - of FieldDescriptorProtoType.TypeUInt32: result = "UInt32" - of FieldDescriptorProtoType.TypeEnum: result = "Enum" - of FieldDescriptorProtoType.TypeSFixed32: result = "SFixed32" - of FieldDescriptorProtoType.TypeSFixed64: result = "SFixed64" - of FieldDescriptorProtoType.TypeSInt32: result = "SInt32" - of FieldDescriptorProtoType.TypeSInt64: result = "SInt64" - -proc defaultValue(field: Field): string = - if isMapEntry(field): - return &"newTable[{field.mapKeyType}, {field.mapValueType}]()" - elif isRepeated(field): - return "@[]" - - case field.ftype - of FieldDescriptorProtoType.TypeDouble: result = "0" - of FieldDescriptorProtoType.TypeFloat: result = "0" - of FieldDescriptorProtoType.TypeInt64: result = "0" - of FieldDescriptorProtoType.TypeUInt64: result = "0" - of FieldDescriptorProtoType.TypeInt32: result = "0" - of FieldDescriptorProtoType.TypeFixed64: result = "0" - of FieldDescriptorProtoType.TypeFixed32: result = "0" - of FieldDescriptorProtoType.TypeBool: result = "false" - of FieldDescriptorProtoType.TypeString: result = "\"\"" - of FieldDescriptorProtoType.TypeGroup: result = "" - of FieldDescriptorProtoType.TypeMessage: result = "nil" - of FieldDescriptorProtoType.TypeBytes: result = "bytes(\"\")" - of FieldDescriptorProtoType.TypeUInt32: result = "0" - of FieldDescriptorProtoType.TypeEnum: result = &"{field.typeName}(0)" - of FieldDescriptorProtoType.TypeSFixed32: result = "0" - of FieldDescriptorProtoType.TypeSFixed64: result = "0" - of FieldDescriptorProtoType.TypeSInt32: result = "0" - of FieldDescriptorProtoType.TypeSInt64: result = "0" - -proc wiretypeStr(field: Field): string = - result = "WireType." - case field.ftype - of FieldDescriptorProtoType.TypeDouble: result &= "Fixed64" - of FieldDescriptorProtoType.TypeFloat: result &= "Fixed32" - of FieldDescriptorProtoType.TypeInt64: result &= "Varint" - of FieldDescriptorProtoType.TypeUInt64: result &= "Varint" - of FieldDescriptorProtoType.TypeInt32: result &= "Varint" - of FieldDescriptorProtoType.TypeFixed64: result &= "Fixed64" - of FieldDescriptorProtoType.TypeFixed32: result &= "Fixed32" - of FieldDescriptorProtoType.TypeBool: result &= "Varint" - of FieldDescriptorProtoType.TypeString: result &= "LengthDelimited" - of FieldDescriptorProtoType.TypeGroup: result &= "" - of FieldDescriptorProtoType.TypeMessage: result &= "LengthDelimited" - of FieldDescriptorProtoType.TypeBytes: result &= "LengthDelimited" - of FieldDescriptorProtoType.TypeUInt32: result &= "Varint" - of FieldDescriptorProtoType.TypeEnum: result &= &"Varint" - of FieldDescriptorProtoType.TypeSFixed32: result &= "Fixed32" - of FieldDescriptorProtoType.TypeSFixed64: result &= "Fixed64" - of FieldDescriptorProtoType.TypeSInt32: result &= "Varint" - of FieldDescriptorProtoType.TypeSInt64: result &= "Varint" - -proc fieldTypeStr(field: Field): string = - result = "FieldType." & $field.ftype - -proc isKeyword(s: string): bool = - case s - of "addr", "and", "as", "asm", "bind", "block", "break", "case", "cast", - "concept", "const", "continue", "converter", "defer", "discard", - "distinct", "div", "do", "elif", "else", "end", "enum", "except", - "export", "finally", "for", "from", "func", "if", "import", "in", - "include", "interface", "is", "isnot", "iterator", "let", "macro", - "method", "mixin", "mod", "nil", "not", "notin", "object", "of", "or", - "out", "proc", "ptr", "raise", "ref", "return", "shl", "shr", "static", - "template", "try", "tuple", "type", "using", "var", "when", "while", - "xor", "yield": - result = true - else: - result = false - -proc writeProc(field: Field): string = - if isMapEntry(field): - result = &"write{field.typeName}KV" - elif isMessage(field): - result = "writeMessage" - elif isEnum(field): - result = "writeEnum" - else: - result = &"write{field.typeName}" - -proc readProc(field: Field): string = - if isMapEntry(field): - result = &"read{field.typeName}KV" - elif isEnum(field): - result = &"readEnum[{field.typeName}]" - else: - result = &"read{field.typeName}" - -proc sizeOfProc(field: Field): string = - if isMapEntry(field): - result = &"sizeOf{field.typeName}KV" - elif isEnum(field): - result = &"sizeOfEnum[{field.typeName}]" - else: - result = &"sizeOf{field.typeName}" - -proc newField(file: ProtoFile, message: Message, desc: FieldDescriptorProto): Field = - new(result) - - result.name = desc.name - result.number = desc.number - result.label = desc.label - result.ftype = desc.type - result.typeName = "" - result.packed = false - result.mapEntry = nil - - # Identifiers cannot start/end with underscore - removePrefix(result.name, '_') - removeSuffix(result.name, '_') - - # Consecutive underscores are not allowed - result.name = replace(result.name, peg"'_' '_'+", "_") - - if isKeyword(result.name): - result.name = "f" & result.name - - if isRepeated(result) and isNumeric(result): - if hasOptions(desc): - if hasPacked(desc.options): - result.packed = desc.options.packed - else: - result.packed = - if file.syntax == Syntax.Proto2: - false - else: - true - else: - result.packed = - if file.syntax == Syntax.Proto2: - false - else: - true - - if hasOneof_index(desc): - result.oneof = message.oneofs[desc.oneof_index] - add(result.oneof.fields, result) - - if isMessage(result) or isEnum(result): - result.typeName = $initNamesFromTypeName(desc.type_name) - else: - result.typeName = $result.ftype - - log(&"newField {result.name} {$result.ftype} {result.typeName} PACKED={result.packed} SYNTAX={file.syntax}") - -proc newOneof(name: string): Oneof = - new(result) - result.fields = @[] - result.name = name - -proc newMessage(file: ProtoFile, names: Names, desc: DescriptorProto): Message = - new(result) - - result.names = names - result.fields = @[] - result.oneofs = @[] - result.mapEntry = false - - if hasMapEntry(desc.options): - result.mapEntry = desc.options.mapEntry - - log(&"newMessage {$result.names}") - - for oneof in desc.oneof_decl: - add(result.oneofs, newOneof(oneof.name)) - - for field in desc.field: - add(result.fields, newField(file, result, field)) - -proc fixMapEntry(file: ProtoFile, message: Message): bool = - for field in message.fields: - for msg in file.messages: - if $msg.names == field.typeName: - if msg.mapEntry: - log(&"fixing map {field.name} {msg.names}") - field.mapEntry = msg - result = true - -proc newEnum(names: Names, desc: EnumDescriptorProto): Enum = - new(result) - - result.names = names & desc.name - result.values = @[] - - log(&"newEnum {$result.names}") - - for value in desc.value: - add(result.values, (value.name, int(value.number))) - - type EnumValue = tuple[name: string, number: int] - - sort(result.values, proc (x, y: EnumValue): int = - system.cmp(x.number, y.number) - ) - -iterator messages(desc: DescriptorProto, names: Names): tuple[names: Names, desc: DescriptorProto] = - var stack: seq[tuple[names: Names, desc: DescriptorProto]] = @[] - - for nested in desc.nested_type: - add(stack, (names, nested)) - - while len(stack) > 0: - let (names, submsg) = pop(stack) - - let subnames = names & submsg.name - yield (subnames, submsg) - - for desc in submsg.nested_type: - add(stack, (subnames, desc)) - -iterator messages(fdesc: FileDescriptorProto, names: Names): tuple[names: Names, desc: DescriptorProto] = - for desc in fdesc.message_type: - let subnames = names & desc.name - yield (subnames, desc) - - for x in messages(desc, subnames): - yield x - -proc quoteReserved(name: string): string = - case name - of "type": result = &"`{name}`" - else: result = name - -proc accessor(field: Field): string = - if field.oneof != nil: - result = &"{field.oneof.name}.{quoteReserved(field.name)}" - else: - result = quoteReserved(field.name) - -proc dependencies(field: Field): seq[string] = - result = @[] - - if isMessage(field) or isEnum(field): - add(result, field.typeName) - -proc dependencies(message: Message): seq[string] = - result = @[] - - for field in message.fields: - add(result, dependencies(field)) - -proc toposort(graph: TableRef[string, HashSet[string]]): seq[string] = - type State = enum Unknown, Gray, Black - - var - enter = toSeq(keys(graph)) - state = newTable[string, State]() - order: seq[string] = @[] - - proc dfs(node: string) = - state[node] = Gray - if node in graph: - for k in graph[node]: - let sk = - if k in state: - state[k] - else: - Unknown - - if sk == Gray: - # cycle - continue - elif sk == Black: - continue - - let idx = find(enter, k) - if idx != -1: - delete(enter, idx) - - dfs(k) - insert(order, node, 0) - state[node] = Black - - while len(enter) > 0: - dfs(pop(enter)) - - result = order - -iterator sortDependencies(messages: seq[Message]): Message = - let - deps = newTable[string, HashSet[string]]() - byname = newTable[string, Message]() - - for message in messages: - deps[$message.names] = toSet(dependencies(message)) - byname[$message.names] = message - - let order = reversed(toposort(deps)) - - for name in order: - if name in byname: - yield byname[name] - -proc parseFile(name: string, fdesc: FileDescriptorProto): ProtoFile = - log(&"parsing {name}") - - new(result) - - result.fdesc = fdesc - result.messages = @[] - result.enums = @[] - - if hasSyntax(fdesc): - if fdesc.syntax == "proto2": - result.syntax = Syntax.Proto2 - elif fdesc.syntax == "proto3": - result.syntax = Syntax.Proto3 - else: - raise newException(Exception, "unrecognized syntax: " & fdesc.syntax) - else: - result.syntax = Syntax.Proto2 - - let basename = - if hasPackage(fdesc): - Names(split(fdesc.package, ".")) - else: - Names(@[]) - - for e in fdesc.enum_type: - add(result.enums, newEnum(basename, e)) - - for name, message in messages(fdesc, basename): - add(result.messages, newMessage(result, name, message)) - - for e in message.enum_type: - add(result.enums, newEnum(name, e)) - -proc addLine(s: var string, line: string) = - if not isNilOrWhitespace(line): - s &= line - s &= "\n" - -iterator genType(e: Enum): string = - yield &"{e.names}* {{.pure.}} = enum" - for item in e.values: - let (name, number) = item - yield indent(&"{name} = {number}", 4) - -proc fullType(field: Field): string = - if isMapEntry(field): - result = &"TableRef[{field.mapKeyType}, {field.mapValueType}]" - else: - result = field.nimTypeName - if isRepeated(field): - result = &"seq[{result}]" - -proc mapKeyField(message: Message): Field = - for field in message.fields: - if field.name == "key": - return field - -proc mapValueField(message: Message): Field = - for field in message.fields: - if field.name == "value": - return field - -iterator genType(message: Message): string = - if not isMapEntry(message): - yield &"{message.names}* = ref {message.names}Obj" - yield &"{message.names}Obj* = object of RootObj" - yield indent(&"hasField: IntSet", 4) - - for field in message.fields: - if isMapEntry(field): - yield indent(&"{field.name}: TableRef[{mapKeyType(field)}, {mapValueType(field)}]", 4) - elif field.oneof == nil: - yield indent(&"{quoteReserved(field.name)}: {field.fullType}", 4) - - for oneof in message.oneofs: - yield indent(&"{oneof.name}: {message.names}_{oneof.name}_OneOf", 4) - - for oneof in message.oneofs: - yield "" - yield &"{message.names}_{oneof.name}_OneOf* {{.union.}} = object" - for field in oneof.fields: - yield indent(&"{quoteReserved(field.name)}: {field.fullType}", 4) - -iterator genNewMessageProc(msg: Message): string = - yield &"proc new{msg.names}*(): {msg.names} =" - yield indent("new(result)", 4) - yield indent("result.hasField = initIntSet()", 4) - for field in msg.fields: - yield indent(&"result.{field.accessor} = {defaultValue(field)}", 4) - yield "" - -iterator oneofSiblings(field: Field): Field = - if field.oneof != nil: - for sibling in field.oneof.fields: - if sibling == field: - continue - yield sibling - -iterator genClearFieldProc(msg: Message, field: Field): string = - yield &"proc clear{field.name}*(message: {msg.names}) =" - yield indent(&"message.{field.accessor} = {defaultValue(field)}", 4) - var numbers: seq[int] = @[field.number] - for sibling in oneofSiblings(field): - add(numbers, sibling.number) - yield indent(&"excl(message.hasField, [{join(numbers, \", \")}])", 4) - yield "" - -iterator genHasFieldProc(msg: Message, field: Field): string = - yield &"proc has{field.name}*(message: {msg.names}): bool =" - var check = indent(&"result = contains(message.hasField, {field.number})", 4) - if isRepeated(field) or isMapEntry(field): - check = &"{check} or (len(message.{field.accessor}) > 0)" - yield check - yield "" - -iterator genSetFieldProc(msg: Message, field: Field): string = - yield &"proc set{field.name}*(message: {msg.names}, value: {field.fullType}) =" - yield indent(&"message.{field.accessor} = value", 4) - yield indent(&"incl(message.hasField, {field.number})", 4) - var numbers: seq[int] = @[] - for sibling in oneofSiblings(field): - add(numbers, sibling.number) - if len(numbers) > 0: - yield indent(&"excl(message.hasField, [{join(numbers, \", \")}])", 4) - yield "" - -iterator genAddToFieldProc(msg: Message, field: Field): string = - yield &"proc add{field.name}*(message: {msg.names}, value: {field.nimTypeName}) =" - yield indent(&"add(message.{field.name}, value)", 4) - yield indent(&"incl(message.hasField, {field.number})", 4) - yield "" - -iterator genFieldAccessorProcs(msg: Message, field: Field): string = - yield &"proc {quoteReserved(field.name)}*(message: {msg.names}): {field.fullType} {{.inline.}} =" - yield indent(&"message.{field.accessor}", 4) - yield "" - - yield &"proc `{field.name}=`*(message: {msg.names}, value: {field.fullType}) {{.inline.}} =" - yield indent(&"set{field.name}(message, value)", 4) - yield "" - -iterator genWriteMapKVProc(msg: Message): string = - let - key = mapKeyField(msg) - value = mapValueField(msg) - - yield &"proc write{msg.names}KV(stream: ProtobufStream, key: {key.fullType}, value: {value.fullType}) =" - yield indent(&"{key.writeProc}(stream, key, {key.number})", 4) - yield indent(&"{value.writeProc}(stream, value, {value.number})", 4) - yield "" - -iterator genWriteMessageProc(msg: Message): string = - yield &"proc write{msg.names}*(stream: ProtobufStream, message: {msg.names}) =" - for field in msg.fields: - if isMapEntry(field): - yield indent(&"for key, value in message.{field.name}:", 4) - yield indent(&"writeTag(stream, {field.number}, {wiretypeStr(field)})", 8) - yield indent(&"writeVarint(stream, {field.sizeOfProc}(key, value))", 8) - yield indent(&"{field.writeProc}(stream, key, value)", 8) - elif isRepeated(field): - if field.packed: - yield indent(&"if has{field.name}(message):", 4) - yield indent(&"writeTag(stream, {field.number}, WireType.LengthDelimited)", 8) - yield indent(&"writeVarint(stream, packedFieldSize(message.{field.name}, {field.fieldTypeStr}))", 8) - yield indent(&"for value in message.{field.name}:", 8) - yield indent(&"{field.writeProc}(stream, value)", 12) - else: - yield indent(&"for value in message.{field.name}:", 4) - yield indent(&"{field.writeProc}(stream, value, {field.number})", 8) - else: - yield indent(&"if has{field.name}(message):", 4) - yield indent(&"{field.writeProc}(stream, message.{field.accessor}, {field.number})", 8) - - if len(msg.fields) == 0: - yield indent("discard", 4) - - yield "" - -iterator genReadMapKVProc(msg: Message): string = - let - key = mapKeyField(msg) - value = mapValueField(msg) - - yield &"proc read{msg.names}KV(stream: ProtobufStream, tbl: TableRef[{key.fullType}, {value.fullType}]) =" - - yield indent(&"var", 4) - yield indent(&"key: {key.fullType}", 8) - yield indent("gotKey = false", 8) - yield indent(&"value: {value.fullType}", 8) - yield indent("gotValue = false", 8) - yield indent("while not atEnd(stream):", 4) - yield indent("let", 8) - yield indent("tag = readTag(stream)", 12) - yield indent("wireType = wireType(tag)", 12) - yield indent("case fieldNumber(tag)", 8) - yield indent(&"of {key.number}:", 8) - yield indent(&"key = {key.readProc}(stream)", 12) - yield indent("gotKey = true", 12) - yield indent(&"of {value.number}:", 8) - if isMessage(value): - yield indent("let", 12) - yield indent("size = readVarint(stream)", 16) - yield indent("data = safeReadStr(stream, int(size))", 16) - yield indent("pbs = newProtobufStream(newStringStream(data))", 16) - yield indent(&"value = {value.readProc}(pbs)", 12) - else: - yield indent(&"value = {value.readProc}(stream)", 12) - yield indent("gotValue = true", 12) - yield indent("else: skipField(stream, wireType)", 8) - yield indent("if not gotKey:", 4) - yield indent(&"raise newException(Exception, \"missing key\")", 8) - yield indent("if not gotValue:", 4) - yield indent(&"raise newException(Exception, \"missing value\")", 8) - yield indent("tbl[key] = value", 4) - yield "" - -iterator genReadMessageProc(msg: Message): string = - yield &"proc read{msg.names}*(stream: ProtobufStream): {msg.names} =" - yield indent(&"result = new{msg.names}()", 4) - if len(msg.fields) > 0: - yield indent("while not atEnd(stream):", 4) - yield indent("let", 8) - yield indent("tag = readTag(stream)", 12) - yield indent("wireType = wireType(tag)", 12) - yield indent("case fieldNumber(tag)", 8) - yield indent("of 0:", 8) - yield indent("raise newException(InvalidFieldNumberError, \"Invalid field number: 0\")", 12) - for field in msg.fields: - let - setter = - if isRepeated(field): - &"add{field.name}" - else: - &"set{field.name}" - yield indent(&"of {field.number}:", 8) - if isRepeated(field): - if isMapEntry(field): - yield indent(&"expectWireType(wireType, {field.wiretypeStr})", 12) - yield indent("let", 12) - yield indent("size = readVarint(stream)", 16) - yield indent("data = safeReadStr(stream, int(size))", 16) - yield indent("pbs = newProtobufStream(newStringStream(data))", 16) - yield indent(&"{field.readProc}(pbs, result.{field.name})", 12) - elif isNumeric(field): - yield indent(&"expectWireType(wireType, {field.wiretypeStr}, WireType.LengthDelimited)", 12) - yield indent("if wireType == WireType.LengthDelimited:", 12) - yield indent("let", 16) - yield indent("size = readVarint(stream)", 20) - yield indent("start = uint64(getPosition(stream))", 20) - yield indent("var consumed = 0'u64", 16) - yield indent("while consumed < size:", 16) - yield indent(&"{setter}(result, {field.readProc}(stream))", 20) - yield indent("consumed = uint64(getPosition(stream)) - start", 20) - yield indent("if consumed != size:", 16) - yield indent("raise newException(Exception, \"packed field size mismatch\")", 20) - yield indent("else:", 12) - yield indent(&"{setter}(result, {field.readProc}(stream))", 16) - elif isMessage(field): - yield indent(&"expectWireType(wireType, {field.wiretypeStr})", 12) - yield indent("let", 12) - yield indent("size = readVarint(stream)", 16) - yield indent("data = safeReadStr(stream, int(size))", 16) - yield indent("pbs = newProtobufStream(newStringStream(data))", 16) - yield indent(&"{setter}(result, {field.readProc}(pbs))", 12) - else: - yield indent(&"expectWireType(wireType, {field.wiretypeStr})", 12) - yield indent(&"{setter}(result, {field.readProc}(stream))", 12) - else: - yield indent(&"expectWireType(wireType, {field.wiretypeStr})", 12) - if isMessage(field): - yield indent("let", 12) - yield indent("size = readVarint(stream)", 16) - yield indent("data = safeReadStr(stream, int(size))", 16) - yield indent("pbs = newProtobufStream(newStringStream(data))", 16) - yield indent(&"{setter}(result, {field.readProc}(pbs))", 12) - else: - yield indent(&"{setter}(result, {field.readProc}(stream))", 12) - yield indent("else: skipField(stream, wireType)", 8) - yield "" - -iterator genSizeOfMapKVProc(message: Message): string = - let - key = mapKeyField(message) - value = mapValueField(message) - - yield &"proc sizeOf{message.names}KV(key: {key.fullType}, value: {value.fullType}): uint64 =" - - # Key (cannot be message or other complex field) - yield indent(&"result = result + sizeOfTag({key.number}, {key.wiretypeStr})", 4) - yield indent(&"result = result + {key.sizeOfProc}(key)", 4) - - # Value - yield indent(&"result = result + sizeOfTag({value.number}, {value.wiretypeStr})", 4) - if isMessage(value): - yield indent(&"result = result + sizeOfLengthDelimited({value.sizeOfProc}(value))", 4) - else: - yield indent(&"result = result + {value.sizeOfProc}(value)", 4) - - yield "" - -iterator genSizeOfMessageProc(msg: Message): string = - yield &"proc sizeOf{msg.names}*(message: {msg.names}): uint64 =" - for field in msg.fields: - if isMapEntry(field): - yield indent(&"if has{field.name}(message):", 4) - yield indent(&"var sizeOfKV = 0'u64", 8) - yield indent(&"for key, value in message.{field.name}:", 8) - yield indent(&"sizeOfKV = sizeOfKV + {field.sizeOfProc}(key, value)", 12) - yield indent(&"result = result + sizeOfTag({field.number}, {field.wiretypeStr})", 8) - yield indent(&"result = result + sizeOfLengthDelimited(sizeOfKV)", 8) - elif isRepeated(field): - if isNumeric(field): - yield indent(&"if has{field.name}(message):", 4) - yield indent(&"result = result + sizeOfTag({field.number}, WireType.LengthDelimited)", 8) - yield indent(&"result = result + sizeOfLengthDelimited(packedFieldSize(message.{field.name}, {field.fieldTypeStr}))", 8) - else: - yield indent(&"for value in message.{field.name}:", 4) - yield indent(&"result = result + sizeOfTag({field.number}, {field.wiretypeStr})", 8) - if isMessage(field): - yield indent(&"result = result + sizeOfLengthDelimited({field.sizeOfProc}(value))", 8) - else: - yield indent(&"result = result + {field.sizeOfProc}(value)", 8) - else: - yield indent(&"if has{field.name}(message):", 4) - yield indent(&"result = result + sizeOfTag({field.number}, {field.wiretypeStr})", 8) - if isMessage(field): - yield indent(&"result = result + sizeOfLengthDelimited({field.sizeOfProc}(message.{field.accessor}))", 8) - else: - yield indent(&"result = result + {field.sizeOfProc}(message.{field.accessor})", 8) - - if len(msg.fields) == 0: - yield indent("result = 0", 4) - - yield "" - -iterator genMessageProcForwards(msg: Message): string = - if not isMapEntry(msg): - yield &"proc new{msg.names}*(): {msg.names}" - yield &"proc write{msg.names}*(stream: ProtobufStream, message: {msg.names})" - yield &"proc read{msg.names}*(stream: ProtobufStream): {msg.names}" - yield &"proc sizeOf{msg.names}*(message: {msg.names}): uint64" - else: - let - key = mapKeyField(msg) - value = mapValueField(msg) - - yield &"proc write{msg.names}KV(stream: ProtobufStream, key: {key.fullType}, value: {value.fullType})" - yield &"proc read{msg.names}KV(stream: ProtobufStream, tbl: TableRef[{key.fullType}, {value.fullType}])" - yield &"proc sizeOf{msg.names}KV(key: {key.fullType}, value: {value.fullType}): uint64" - -iterator genProcs(msg: Message): string = - if isMapEntry(msg): - for line in genSizeOfMapKVProc(msg): yield line - for line in genWriteMapKVProc(msg): yield line - for line in genReadMapKVProc(msg): yield line - else: - for line in genNewMessageProc(msg): yield line - - for field in msg.fields: - for line in genClearFieldProc(msg, field): yield line - for line in genHasFieldProc(msg, field): yield line - for line in genSetFieldProc(msg, field): yield line - - if isRepeated(field) and not isMapEntry(field): - for line in genAddToFieldProc(msg, field): yield line - - for line in genFieldAccessorProcs(msg, field): yield line - - for line in genSizeOfMessageProc(msg): yield line - for line in genWriteMessageProc(msg): yield line - for line in genReadMessageProc(msg): yield line - - yield &"proc serialize*(message: {msg.names}): string =" - yield indent("let", 4) - yield indent("ss = newStringStream()", 8) - yield indent("pbs = newProtobufStream(ss)", 8) - yield indent(&"write{msg.names}(pbs, message)", 4) - yield indent("result = ss.data", 4) - yield "" - - yield &"proc new{msg.names}*(data: string): {msg.names} =" - yield indent("let", 4) - yield indent("ss = newStringStream(data)", 8) - yield indent("pbs = newProtobufStream(ss)", 8) - yield indent(&"result = read{msg.names}(pbs)", 4) - yield "" - -proc processFile(filename: string, fdesc: FileDescriptorProto, - otherFiles: TableRef[string, ProtoFile]): ProcessedFile = - var (dir, name, _) = splitFile(filename) - var pbfilename = (dir / name) & "_pb.nim" - - log(&"processing {filename}: {pbfilename}") - - new(result) - result.name = pbfilename - result.data = "" - - let parsed = parseFile(filename, fdesc) - - var hasMaps = false - for message in parsed.messages: - let tmp = fixMapEntry(parsed, message) - if tmp: - hasMaps = true - - addLine(result.data, "# Generated by protoc_gen_nim. Do not edit!") - addLine(result.data, "") - addLine(result.data, "import intsets") - if hasMaps: - addLine(result.data, "import tables") - addLine(result.data, "export tables") - addLine(result.data, "") - addLine(result.data, "import nimpb/nimpb") - addLine(result.data, "") - - for dep in fdesc.dependency: - var (dir, depname, _) = splitFile(dep) - - if dir == "google/protobuf": - dir = "nimpb/wkt" - - var deppbname = (dir / depname) & "_pb" - addLine(result.data, &"import {deppbname}") - - if hasDependency(fdesc): - addLine(result.data, "") - - addLine(result.data, "type") - - for e in parsed.enums: - for line in genType(e): addLine(result.data, indent(line, 4)) - - for message in parsed.messages: - for line in genType(message): addLine(result.data, indent(line, 4)) - - addLine(result.data, "") - - for message in sortDependencies(parsed.messages): - for line in genMessageProcForwards(message): - addLine(result.data, line) - addLine(result.data, "") - - for message in sortDependencies(parsed.messages): - for line in genProcs(message): - addLine(result.data, line) - addLine(result.data, "") - -proc generateCode(request: CodeGeneratorRequest, response: CodeGeneratorResponse) = - let otherFiles = newTable[string, ProtoFile]() - - for file in request.proto_file: - add(otherFiles, file.name, parseFile(file.name, file)) - - for filename in request.file_to_generate: - for fdesc in request.proto_file: - if fdesc.name == filename: - let results = processFile(filename, fdesc, otherFiles) - let f = newCodeGeneratorResponse_File() - setName(f, results.name) - setContent(f, results.data) - addFile(response, f) - -let pbsi = newProtobufStream(newFileStream(stdin)) -let pbso = newProtobufStream(newFileStream(stdout)) - -let request = readCodeGeneratorRequest(pbsi) -let response = newCodeGeneratorResponse() - -generateCode(request, response) - -writeCodeGeneratorResponse(pbso, response) |
