Add protobuf (proto3) defn's to generated docs.

avro/python/docs.py

@@ -6,7 +6,13 @@
 @dataclass
 class Schema:
     name: str
-    path: PurePath
+    path: Path
+
+    def raw_proto3(self) -> str:
+        proto3_dir = self.path.parent.parent / "proto3"
+        proto3_file = proto3_dir / f"{self.name}.proto"
+        with proto3_file.open("r") as f:
+            return f.read()
 
 
 @dataclass
@@ -57,6 +63,10 @@ def h3(title: str) -> str:
     return f"### {title}\n\n"
 
 
+def h4(title: str) -> str:
+    return f"#### {title}\n\n"
+
+
 def list_item(text: str, indent=0) -> str:
     indent = ' ' * (2*indent)
     return f'{indent}- {text}\n'
@@ -67,13 +77,14 @@ def link(text: str, url: str) -> str:
 
 
 def to_anchor(text: str) -> str:
+    text = text.replace("(", "").replace(")", "")
     return "#" + text.lower().replace(" ", "-")
 
 
 def code(text: str, type='') -> str:
     s = f"```{type}\n"
     s += text + "\n"
-    s += "```\n"
+    s += "```\n\n"
     return s
 
 
@@ -120,22 +131,40 @@ def generate_versions(self, output_dir: Path, proto_url_path: str):
         for ver in self.versions:
             ver_path = output_dir / f"version-{ver.version}.md"
             body = ""
+            protoc_body = ""
             with ver_path.open("w") as f:
                 f.write(h1(f"Interface Specification Version {ver.version}"))
                 f.write(h2("Overview"))
+                f.write("Table of schemas, grouped by protocol:\n\n")
                 for proto in ver.protocols:
-                    sum = f"{link(proto.name, to_anchor(proto.name))} includes schemas:"
-                    f.write(list_item(sum))
+                    href = link(proto.name + " (Avro IDL)", to_anchor(proto.name))
+                    p3_section = proto.name + " (proto3)"
+                    p3_href = link("protobuf", to_anchor(p3_section))
+                    f.write(list_item(f"{href} (see also {p3_href}):"))
+
+                    body += h3(proto.name)
+                    body += f"See also the {p3_href}\n\n"
+                    protoc_body += h3(p3_section)
+
+                    # Print table of contents by protocol
                     for schema in proto.schemas:
                         spath = Path(proto_url_path) / f"v{ver.version}"
                         spath = spath / "schema" / schema.path.name
-                        f.write(list_item(link(schema.name, str(spath)), indent=1))
-                    body += h3(proto.name)
+                        li = list_item(link(schema.name, str(spath)), indent=1)
+                        f.write(li)
+
+                        p3_name = schema.name + " (proto3)"
+                        protoc_body += h4(p3_name)
+                        protoc_body += code(schema.raw_proto3(), type='protobuf')
+
                     body += code(proto.raw_text(), type='avdl')
 
                 f.write(h2("Protocols"))
                 f.write(body)
 
+                f.write(h2("Proto3 Definitions"))
+                f.write(protoc_body)
+
     def generate_markdown(self, output_dir: Path):
         self.generate_index(output_dir)
         self.generate_versions(output_dir, "/protocol")

avro/python/generate.py

@@ -1,13 +1,20 @@
 import os
 from pathlib import Path
 import subprocess
-from avrotize.avrotoproto import convert_avro_to_proto
+from avrotize.avrotoproto import convert_avro_to_proto, json
 
 from docs import Docs
 
 # Main script to generate all the things from the Avro IDL definitions.
 
 
+def namespace(schema_file: Path) -> str | None:
+    """ Extract namespace from Avro schema file. """
+    with open(schema_file, "r") as f:
+        obj = json.load(f)
+        return obj.get("namespace", None)
+
+
 class Schemas:
     """ Helper class for enumerating schemas in this repo. Provides a
         builder-style API (methods that return reference to self to allow
@@ -54,6 +61,14 @@ def gen_proto3(self):
             proto_dir = pp.parent.parent / "proto3"
             print(f"--> Generating proto3 for {schema_file.stem} in {proto_dir}")
             convert_avro_to_proto(schema_file, proto_dir)
+            # workaround: avrotize func. above always names file <namespace>.proto,
+            #  which causes all except the last schema to be overwritten. Rename that
+            #  output file here, until we can fix the avrotize lib.
+            ns = namespace(schema_file)
+            if ns:
+                proto_file = proto_dir / f"{ns}.proto"
+                new_file = proto_dir / f"{schema_file.stem}.proto"
+                proto_file.rename(new_file)
 
 
 def main():

docs/generated/version-0.1-example.md

@@ -2,17 +2,21 @@
 
 ## Overview
 
-- [trip](#trip) includes schemas:
+Table of schemas, grouped by protocol:
+
+- [trip (Avro IDL)](#trip) (see also [protobuf](#trip-proto3)):
   - [TripType](/protocol/v0.1-example/schema/TripType.avsc)
   - [GeomType](/protocol/v0.1-example/schema/GeomType.avsc)
   - [TripStart](/protocol/v0.1-example/schema/TripStart.avsc)
   - [TripEnd](/protocol/v0.1-example/schema/TripEnd.avsc)
-- [sanity](#sanity) includes schemas:
+- [sanity (Avro IDL)](#sanity) (see also [protobuf](#sanity-proto3)):
   - [Sanity](/protocol/v0.1-example/schema/Sanity.avsc)
 ## Protocols
 
 ### trip
 
+See also the [protobuf](#trip-proto3)
+
 ```avdl
 @namespace("v0.1-example")
 
@@ -77,8 +81,11 @@ protocol Trip {
 }
 
 ```
+
 ### sanity
 
+See also the [protobuf](#sanity-proto3)
+
 ```avdl
 @namespace("v0.1-example")
 protocol Sanity {
@@ -93,3 +100,98 @@ protocol Sanity {
 
 
 ```
+
+## Proto3 Definitions
+
+### trip (proto3)
+
+#### TripType (proto3)
+
+```protobuf
+syntax = "proto3";
+
+package v0.1-example;
+
+enum TripType {
+  Hiking = 0;
+  Biking = 1;
+}
+
+
+```
+
+#### GeomType (proto3)
+
+```protobuf
+syntax = "proto3";
+
+package v0.1-example;
+
+enum GeomType {
+  Point = 0;
+  Polygon = 1;
+}
+
+
+```
+
+#### TripStart (proto3)
+
+```protobuf
+syntax = "proto3";
+
+package v0.1-example;
+
+message TripStart {
+  string name = 1;
+  string timestamp = 2;
+  string trip_id = 3;
+  TripTypeEnum TripType = 4;
+  int32 num_travellers = 5;
+  GeomTypeEnum GeomType = 6;
+  repeated array coordinates = 7;
+  enum TripTypeEnum {
+    Hiking = 0;
+    Biking = 1;
+  }
+  enum GeomTypeEnum {
+    Point = 0;
+    Polygon = 1;
+  }
+}
+
+```
+
+#### TripEnd (proto3)
+
+```protobuf
+syntax = "proto3";
+
+package v0.1-example;
+
+message TripEnd {
+  string name = 1;
+  string timestamp = 2;
+  string trip_id = 3;
+}
+
+```
+
+### sanity (proto3)
+
+#### Sanity (proto3)
+
+```protobuf
+syntax = "proto3";
+
+package v0.1-example;
+
+message Sanity {
+  string name = 1;
+  int32 count = 2;
+  float realnum = 3;
+  repeated string cities = 4;
+}
+
+```
+

protocol/v0.1-example/proto3/GeomType.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package v0.1-example;
+
+enum GeomType {
+  Point = 0;
+  Polygon = 1;
+}
+

protocol/v0.1-example/proto3/Sanity.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package v0.1-example;
+
+message Sanity {
+  string name = 1;
+  int32 count = 2;
+  float realnum = 3;
+  repeated string cities = 4;
+}

protocol/v0.1-example/proto3/TripEnd.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package v0.1-example;
+
+message TripEnd {
+  string name = 1;
+  string timestamp = 2;
+  string trip_id = 3;
+}

protocol/v0.1-example/proto3/TripType.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package v0.1-example;
+
+enum TripType {
+  Hiking = 0;
+  Biking = 1;
+}
+