Add initial version of the protoc plugin

4 files changed, 751 insertions, 0 deletions

diff --git a/generator/descriptor_pb.nim b/generator/descriptor_pb.nim
new file mode 100644
index 0000000..5d07365
--- /dev/null
+++ b/generator/descriptor_pb.nim
@@ -0,0 +1,276 @@
+import intsets
+
+import protobuf/types
+import protobuf/gen
+import protobuf/stream
+
+const
+    FileDescriptorSetDesc = MessageDesc(
+        name: "FileDescriptorSet",
+        fields: @[
+            FieldDesc(
+                name: "files",
+                number: 1,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "FileDescriptorProto",
+                packed: false
+            )
+        ]
+    )
+
+    FileDescriptorProtoDesc = MessageDesc(
+        name: "FileDescriptorProto",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "package",
+                number: 2,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "dependency",
+                number: 3,
+                ftype: FieldType.String,
+                label: FieldLabel.Repeated,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "message_type",
+                number: 4,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "DescriptorProto",
+                packed: false
+            ),
+            FieldDesc(
+                name: "enum_type",
+                number: 5,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "EnumDescriptorProto",
+                packed: false
+            ),
+        ]
+    )
+
+    DescriptorProtoDesc = MessageDesc(
+        name: "DescriptorProto",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "field",
+                number: 2,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "FieldDescriptorProto",
+                packed: false
+            ),
+            FieldDesc(
+                name: "nested_type",
+                number: 3,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "DescriptorProto",
+                packed: false
+            ),
+            FieldDesc(
+                name: "enum_type",
+                number: 4,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "EnumDescriptorProto",
+                packed: false
+            ),
+        ]
+    )
+
+    EnumDescriptorProtoDesc = MessageDesc(
+        name: "EnumDescriptorProto",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "value",
+                number: 2,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "EnumValueDescriptorProto",
+                packed: false
+            ),
+        ]
+    )
+
+    EnumValueDescriptorProtoDesc = MessageDesc(
+        name: "EnumValueDescriptorProto",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "number",
+                number: 2,
+                ftype: FieldType.Int32,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+        ]
+    )
+
+    FieldDescriptorProtoDesc = MessageDesc(
+        name: "FieldDescriptorProto",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "number",
+                number: 3,
+                ftype: FieldType.Int32,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "label",
+                number: 4,
+                ftype: FieldType.Enum,
+                label: FieldLabel.Optional,
+                typeName: "FieldDescriptorProto_Label",
+                packed: false
+            ),
+            FieldDesc(
+                name: "type",
+                number: 5,
+                ftype: FieldType.Enum,
+                label: FieldLabel.Optional,
+                typeName: "FieldDescriptorProto_Type",
+                packed: false
+            ),
+            FieldDesc(
+                name: "type_name",
+                number: 6,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "options",
+                number: 8,
+                ftype: FieldType.Message,
+                label: FieldLabel.Optional,
+                typeName: "FieldOptions",
+                packed: false
+            ),
+        ]
+    )
+
+    FieldDescriptorProto_LabelDesc = EnumDesc(
+        name: "FieldDescriptorProto_Label",
+        values: @[
+            EnumValueDesc(name: "LABEL_OPTIONAL", number: 1),
+            EnumValueDesc(name: "LABEL_REQUIRED", number: 2),
+            EnumValueDesc(name: "LABEL_REPEATED", number: 3)
+        ]
+    )
+
+    FieldDescriptorProto_TypeDesc = EnumDesc(
+        name: "FieldDescriptorProto_Type",
+        values: @[
+            EnumValueDesc(name: "TYPE_DOUBLE", number: 1),
+            EnumValueDesc(name: "TYPE_FLOAT", number: 2),
+            EnumValueDesc(name: "TYPE_INT64", number: 3),
+            EnumValueDesc(name: "TYPE_UINT64", number: 4),
+            EnumValueDesc(name: "TYPE_INT32", number: 5),
+            EnumValueDesc(name: "TYPE_FIXED64", number: 6),
+            EnumValueDesc(name: "TYPE_FIXED32", number: 7),
+            EnumValueDesc(name: "TYPE_BOOL", number: 8),
+            EnumValueDesc(name: "TYPE_STRING", number: 9),
+            EnumValueDesc(name: "TYPE_GROUP", number: 10),
+            EnumValueDesc(name: "TYPE_MESSAGE", number: 11),
+            EnumValueDesc(name: "TYPE_BYTES", number: 12),
+            EnumValueDesc(name: "TYPE_UINT32", number: 13),
+            EnumValueDesc(name: "TYPE_ENUM", number: 14),
+            EnumValueDesc(name: "TYPE_SFIXED32", number: 15),
+            EnumValueDesc(name: "TYPE_SFIXED64", number: 16),
+            EnumValueDesc(name: "TYPE_SINT32", number: 17),
+            EnumValueDesc(name: "TYPE_SINT64", number: 18),
+        ]
+    )
+
+    FieldOptionsDesc = MessageDesc(
+        name: "FieldOptions",
+        fields: @[
+            FieldDesc(
+                name: "packed",
+                number: 2,
+                ftype: FieldType.Bool,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+        ]
+    )
+
+generateEnumType(FieldDescriptorProto_LabelDesc)
+generateEnumProcs(FieldDescriptorProto_LabelDesc)
+
+generateEnumType(FieldDescriptorProto_TypeDesc)
+generateEnumProcs(FieldDescriptorProto_TypeDesc)
+
+generateMessageType(EnumValueDescriptorProtoDesc)
+generateMessageProcs(EnumValueDescriptorProtoDesc)
+
+generateMessageType(EnumDescriptorProtoDesc)
+generateMessageProcs(EnumDescriptorProtoDesc)
+
+generateMessageType(FieldOptionsDesc)
+generateMessageProcs(FieldOptionsDesc)
+
+generateMessageType(FieldDescriptorProtoDesc)
+generateMessageProcs(FieldDescriptorProtoDesc)
+
+generateMessageType(DescriptorProtoDesc)
+generateMessageProcs(DescriptorProtoDesc)
+
+generateMessageType(FileDescriptorProtoDesc)
+generateMessageProcs(FileDescriptorProtoDesc)
+
+generateMessageType(FileDescriptorSetDesc)
+generateMessageProcs(FileDescriptorSetDesc)
diff --git a/generator/nim.cfg b/generator/nim.cfg
new file mode 100644
index 0000000..a119208
--- /dev/null
+++ b/generator/nim.cfg
@@ -0,0 +1 @@
+--path:"../src"
diff --git a/generator/plugin_pb.nim b/generator/plugin_pb.nim
new file mode 100644
index 0000000..6a3759d
--- /dev/null
+++ b/generator/plugin_pb.nim
@@ -0,0 +1,148 @@
+import intsets
+
+import protobuf/types
+import protobuf/gen
+import protobuf/stream
+
+import descriptor_pb
+
+const
+    VersionDesc = MessageDesc(
+        name: "Version",
+        fields: @[
+            FieldDesc(
+                name: "major",
+                number: 1,
+                ftype: FieldType.Int32,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "minor",
+                number: 2,
+                ftype: FieldType.Int32,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "patch",
+                number: 3,
+                ftype: FieldType.Int32,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "suffix",
+                number: 4,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            )
+        ]
+    )
+
+    CodeGeneratorRequestDesc = MessageDesc(
+        name: "CodeGeneratorRequest",
+        fields: @[
+            FieldDesc(
+                name: "file_to_generate",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Repeated,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "parameter",
+                number: 2,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "proto_file",
+                number: 15,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "FileDescriptorProto",
+                packed: false
+            ),
+            FieldDesc(
+                name: "compiler_version",
+                number: 3,
+                ftype: FieldType.Message,
+                label: FieldLabel.Optional,
+                typeName: "Version",
+                packed: false
+            )
+        ]
+    )
+
+    CodeGeneratorResponseDesc = MessageDesc(
+        name: "CodeGeneratorResponse",
+        fields: @[
+            FieldDesc(
+                name: "error",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "file",
+                number: 15,
+                ftype: FieldType.Message,
+                label: FieldLabel.Repeated,
+                typeName: "CodeGeneratorResponse_File",
+                packed: false
+            ),
+        ]
+    )
+
+    CodeGeneratorResponse_FileDesc = MessageDesc(
+        name: "CodeGeneratorResponse_File",
+        fields: @[
+            FieldDesc(
+                name: "name",
+                number: 1,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "insertion_point",
+                number: 2,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+            FieldDesc(
+                name: "content",
+                number: 15,
+                ftype: FieldType.String,
+                label: FieldLabel.Optional,
+                typeName: "",
+                packed: false
+            ),
+        ]
+    )
+
+generateMessageType(VersionDesc)
+generateMessageProcs(VersionDesc)
+
+generateMessageType(CodeGeneratorRequestDesc)
+generateMessageProcs(CodeGeneratorRequestDesc)
+
+generateMessageType(CodeGeneratorResponse_FileDesc)
+generateMessageProcs(CodeGeneratorResponse_FileDesc)
+
+generateMessageType(CodeGeneratorResponseDesc)
+generateMessageProcs(CodeGeneratorResponseDesc)
diff --git a/generator/protoc_gen_nim.nim b/generator/protoc_gen_nim.nim
new file mode 100644
index 0000000..6d8f192
--- /dev/null
+++ b/generator/protoc_gen_nim.nim
@@ -0,0 +1,326 @@
+import algorithm
+import os
+import sequtils
+import sets
+import strformat
+import strutils
+import tables
+
+import descriptor_pb
+import plugin_pb
+
+import protobuf/stream
+import protobuf/types
+import protobuf/gen
+
+type
+    Names = distinct seq[string]
+
+    Enum = ref object
+        names: Names
+        values: seq[tuple[name: string, number: int]]
+
+    Field = ref object
+        number: int
+        name: string
+        label: FieldLabel
+        ftype: FieldType
+        typeName: string
+        packed: bool
+
+    Message = ref object
+        names: Names
+        fields: seq[Field]
+
+    ProcessedFile = ref object
+        name: string
+        data: string
+
+    ProtoFile = ref object
+        fdesc: FileDescriptorProto
+        enums: seq[Enum]
+        messages: seq[Message]
+
+when defined(debug):
+    proc log(msg: string) =
+        stderr.write(msg)
+        stderr.write("\n")
+else:
+    proc log(msg: string) = discard
+
+proc initNames(n: seq[string]): Names =
+    result = Names(n)
+
+proc initNamesFromTypeName(typename: string): Names =
+    if typename[0] != '.':
+        raise newException(Exception, "relative names not supported")
+    let parts = split(typename[1..^1], ".")
+    result = Names(parts)
+
+proc `$`(names: Names): string =
+    let n = seq[string](names)
+    result = join(n, "_")
+
+proc add(names: var Names, s: string) =
+    add(seq[string](names), s)
+
+proc add(names: var Names, other: Names) =
+    names = Names(seq[string](names) & seq[string](other))
+
+proc `&`(names: Names, s: string): Names =
+    result = names
+    add(result, s)
+
+proc `==`(a, b: Names): bool =
+    result = seq[string](a) == seq[string](b)
+
+proc convertFieldType(t: FieldDescriptorProto_Type): FieldType =
+    result = FieldType(int(t))
+
+proc convertFieldLabel(t: FieldDescriptorProto_Label): FieldLabel =
+    result = FieldLabel(int(t))
+
+proc newField(desc: FieldDescriptorProto): Field =
+    new(result)
+
+    result.name = desc.name
+    result.number = desc.number
+    result.label = convertFieldLabel(desc.label)
+    result.ftype = convertFieldType(desc.type)
+    result.typeName = ""
+    result.packed = desc.options.packed
+
+    if result.ftype == FieldType.Message or result.ftype == FieldType.Enum:
+        result.typeName = $initNamesFromTypeName(desc.type_name)
+
+    log(&"newField {result.name} {$result.ftype} {result.typeName}")
+
+proc newMessage(names: Names, desc: DescriptorProto): Message =
+    new(result)
+
+    result.names = names
+    result.fields = @[]
+
+    log(&"newMessage {$result.names}")
+
+    for field in desc.field:
+        add(result.fields, newField(field))
+
+proc newEnum(names: Names, desc: EnumDescriptorProto): Enum =
+    new(result)
+
+    result.names = names & desc.name
+    result.values = @[]
+
+    log(&"newEnum {$result.names}")
+
+    for value in desc.value:
+        add(result.values, (value.name, int(value.number)))
+
+iterator messages(desc: DescriptorProto, names: Names): tuple[names: Names, desc: DescriptorProto] =
+    var stack: seq[tuple[names: Names, desc: DescriptorProto]] = @[]
+
+    for nested in desc.nested_type:
+        add(stack, (names, nested))
+
+    while len(stack) > 0:
+        let (names, submsg) = pop(stack)
+
+        let subnames = names & submsg.name
+        yield (subnames, submsg)
+
+        for desc in submsg.nested_type:
+            add(stack, (subnames, desc))
+
+iterator messages(fdesc: FileDescriptorProto, names: Names): tuple[names: Names, desc: DescriptorProto] =
+    for desc in fdesc.message_type:
+        let subnames = names & desc.name
+        yield (subnames, desc)
+
+        for x in messages(desc, subnames):
+            yield x
+
+proc dependencies(field: Field): seq[string] =
+    result = @[]
+
+    if field.ftype == FieldType.Message or field.ftype == FieldType.Enum:
+        add(result, field.typeName)
+
+proc dependencies(message: Message): seq[string] =
+    result = @[]
+
+    for field in message.fields:
+        add(result, dependencies(field))
+
+proc toposort(graph: TableRef[string, HashSet[string]]): seq[string] =
+    type State = enum Unknown, Gray, Black
+
+    var
+        enter = toSeq(keys(graph))
+        state = newTable[string, State]()
+        order: seq[string] = @[]
+
+    proc dfs(node: string) =
+        state[node] = Gray
+        if node in graph:
+            for k in graph[node]:
+                let sk =
+                    if k in state:
+                        state[k]
+                    else:
+                        Unknown
+
+                if sk == Gray:
+                    raise newException(Exception, "cycle detected")
+                elif sk == Black:
+                    continue
+
+                let idx = find(enter, k)
+                if idx != -1:
+                    delete(enter, idx)
+
+                dfs(k)
+        insert(order, node, 0)
+        state[node] = Black
+
+    while len(enter) > 0:
+        dfs(pop(enter))
+
+    result = order
+
+iterator sortDependencies(messages: seq[Message]): Message =
+    let
+        deps = newTable[string, HashSet[string]]()
+        byname = newTable[string, Message]()
+
+    for message in messages:
+        deps[$message.names] = toSet(dependencies(message))
+        byname[$message.names] = message
+
+    let order = reversed(toposort(deps))
+
+    for name in order:
+        if name in byname:
+            yield byname[name]
+
+proc parseFile(name: string, fdesc: FileDescriptorProto): ProtoFile =
+    log(&"parsing {name}")
+
+    new(result)
+
+    result.fdesc = fdesc
+    result.messages = @[]
+    result.enums = @[]
+
+    let basename: Names = Names(@[])
+
+    for e in fdesc.enum_type:
+        add(result.enums, newEnum(basename, e))
+
+    for name, message in messages(fdesc, basename):
+        add(result.messages, newMessage(name, message))
+
+        for e in message.enum_type:
+            add(result.enums, newEnum(name, e))
+
+proc addLine(s: var string, line: string, indent: int = 0) =
+    if indent > 0:
+        s &= repeat(' ', indent)
+    s &= line
+    s &= "\n"
+
+proc generateDesc(field: Field): string =
+    result = ""
+    addLine(result, "FieldDesc(", 12)
+    addLine(result, &"name: \"{field.name}\",", 16)
+    addLine(result, &"number: {field.number},", 16)
+    addLine(result, &"ftype: FieldType.{field.ftype},", 16)
+    addLine(result, &"label: FieldLabel.{field.label},", 16)
+    addLine(result, &"typeName: \"{field.typeName}\",", 16)
+    addLine(result, &"packed: {field.packed},", 16)
+    addLine(result, "),", 12)
+
+proc generateDesc(message: Message): string =
+    result = ""
+    addLine(result, &"{message.names}Desc = MessageDesc(", 4)
+    addLine(result, &"name: \"{message.names}\",", 8)
+    addLine(result, "fields: @[", 8)
+    for field in message.fields:
+        result &= generateDesc(field)
+    addLine(result, "]", 8)
+    addLine(result, ")", 4)
+
+proc generateDesc(e: Enum): string =
+    result = ""
+    addLine(result, &"{e.names}Desc = EnumDesc(", 4)
+    addLine(result, &"name: \"{e.names}\",", 8)
+    addLine(result, "values: @[", 8)
+    for v in e.values:
+        addLine(result, &"EnumValueDesc(name: \"{v.name}\", number: {v.number}),", 12)
+    addLine(result, "]", 8)
+    addLine(result, ")", 4)
+
+proc processFile(filename: string, fdesc: FileDescriptorProto,
+                 otherFiles: TableRef[string, ProtoFile]): ProcessedFile =
+    var (dir, name, _) = splitFile(filename)
+    var pbfilename = (dir / name) & "_pb.nim"
+
+    log(&"processing {filename}: {pbfilename}")
+
+    # TODO: handle file dependencies
+
+    new(result)
+    result.name = pbfilename
+    result.data = ""
+
+    addLine(result.data, "# Generated by protoc_gen_nim. Do not edit!")
+    addLine(result.data, "")
+    addLine(result.data, "import intsets")
+    addLine(result.data, "")
+    addLine(result.data, "import protobuf/gen")
+    addLine(result.data, "import protobuf/stream")
+    addLine(result.data, "import protobuf/types")
+    addLine(result.data, "")
+
+    let parsed = parseFile(filename, fdesc)
+
+    addLine(result.data, "const")
+
+    for e in parsed.enums:
+        result.data &= generateDesc(e)
+
+    for message in sortDependencies(parsed.messages):
+        result.data &= generateDesc(message)
+
+    for e in parsed.enums:
+        addLine(result.data, &"generateEnumType({e.names}Desc)")
+        addLine(result.data, &"generateEnumProcs({e.names}Desc)")
+
+    for message in sortDependencies(parsed.messages):
+        addLine(result.data, &"generateMessageType({message.names}Desc)")
+        addLine(result.data, &"generateMessageProcs({message.names}Desc)")
+
+proc generateCode(request: CodeGeneratorRequest, response: CodeGeneratorResponse) =
+    let otherFiles = newTable[string, ProtoFile]()
+
+    for file in request.proto_file:
+        add(otherFiles, file.name, parseFile(file.name, file))
+
+    for filename in request.file_to_generate:
+        for fdesc in request.proto_file:
+            if fdesc.name == filename:
+                let results = processFile(filename, fdesc, otherFiles)
+                let f = newCodeGeneratorResponse_File()
+                setName(f, results.name)
+                setContent(f, results.data)
+                addFile(response, f)
+
+let pbsi = newProtobufStream(newFileStream(stdin))
+let pbso = newProtobufStream(newFileStream(stdout))
+
+let request = readCodeGeneratorRequest(pbsi)
+let response = newCodeGeneratorResponse()
+
+generateCode(request, response)
+
+writeCodeGeneratorResponse(pbso, response)