proto: update proto generation to use buf (tendermint#7975)

* Hard-code go_package option for .proto files Signed-off-by: Thane Thomson <[email protected]> * Automatically relocate generated ABCI types after proto-gen Signed-off-by: Thane Thomson <[email protected]> * Skip building gogoproto (i.e. only build our types) Signed-off-by: Thane Thomson <[email protected]> * Remove unnecessary proto generation scripts Signed-off-by: Thane Thomson <[email protected]> * Upgrade buf config from v1beta1 to v1 Signed-off-by: Thane Thomson <[email protected]> * Add simple proto generation script Signed-off-by: Thane Thomson <[email protected]> * Replace buf-based protobuf generation with simple protoc-based approach Signed-off-by: Thane Thomson <[email protected]> * Remove custom buf-based Docker image generation config and Dockerfile Signed-off-by: Thane Thomson <[email protected]> * Adopt Cosmos SDK's approach to Protobuf linting and breakage checking in CI Signed-off-by: Thane Thomson <[email protected]> * Suppress command echo when running proto checks Signed-off-by: Thane Thomson <[email protected]> * Fix proto-check workflow YAML indentation Signed-off-by: Thane Thomson <[email protected]> * Restore proto-format target Signed-off-by: Thane Thomson <[email protected]> * Replace custom BASH script with make equivalent Signed-off-by: Thane Thomson <[email protected]> * Remove proto linting/breaking changes CI checks after discussion today Signed-off-by: Thane Thomson <[email protected]> * Remove dangling reference to CI workflow that no longer exists Signed-off-by: Thane Thomson <[email protected]> * Update contributing guidelines relating to protos Signed-off-by: Thane Thomson <[email protected]> * Use buf instead for generating protos Signed-off-by: Thane Thomson <[email protected]> * Remove unused buf config for gogoprotobuf Signed-off-by: Thane Thomson <[email protected]> * Add reminder for if we migrate fully to buf Signed-off-by: Thane Thomson <[email protected]> * Restore protopackage script for tendermint#8065 Signed-off-by: Thane Thomson <[email protected]> * Fix permissions on protopackage script Signed-off-by: Thane Thomson <[email protected]> * Update contributing guidelines to show building of protos using buf Signed-off-by: Thane Thomson <[email protected]> * Fix breaking changes check and add disclaimer Signed-off-by: Thane Thomson <[email protected]> * Expand on contributing guidelines for clarity Signed-off-by: Thane Thomson <[email protected]> * Re-remove old proto workflows Signed-off-by: Thane Thomson <[email protected]> * Add buf-based proto linting workflow in CI Signed-off-by: Thane Thomson <[email protected]> * Superficially reorder proto targets Signed-off-by: Thane Thomson <[email protected]> * Fix proto lints Signed-off-by: Thane Thomson <[email protected]> * Fix GA workflow YAML indentation Signed-off-by: Thane Thomson <[email protected]> * Temporarily use forked version of mlc Use forked version of markdown-link-check until gaurav-nelson/github-action-markdown-link-check#126 lands. Signed-off-by: Thane Thomson <[email protected]> * Temporarily disable markdown link checker Signed-off-by: Thane Thomson <[email protected]> * Remove gogo protos - superseded by version from buf registry Signed-off-by: Thane Thomson <[email protected]>
PeggyJV · Mar 11, 2022 · 72bbe64 · 72bbe64
1 parent 658a766
commit 72bbe64
Show file tree

Hide file tree

Showing 33 changed files with 160 additions and 370 deletions.
diff --git a/.github/workflows/markdown-links.yml b/.github/workflows/markdown-links.yml
@@ -1,17 +1,19 @@
-name: Check Markdown links
+# TODO: Re-enable when https://github.com/gaurav-nelson/github-action-markdown-link-check/pull/126 lands.
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches: [master]
-
-jobs:
-  markdown-link-check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: gaurav-nelson/[email protected]
-        with:
-          check-modified-files-only: 'yes'
+#name: Check Markdown links
+#
+#on:
+#  push:
+#    branches:
+#      - master
+#  pull_request:
+#    branches: [master]
+#
+#jobs:
+#  markdown-link-check:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - uses: actions/checkout@v3
+#      - uses: gaurav-nelson/[email protected]
+#        with:
+#          check-modified-files-only: 'yes'
diff --git a/.github/workflows/proto-check.yml b/.github/workflows/proto-check.yml
diff --git a/.github/workflows/proto-dockerfile.yml b/.github/workflows/proto-dockerfile.yml
diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml
@@ -0,0 +1,21 @@
+name: Protobuf Lint
+on:
+  pull_request:
+    paths:
+      - 'proto/**'
+  push:
+    branches:
+      - master
+    paths:
+      - 'proto/**'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bufbuild/[email protected]
+      - uses: bufbuild/buf-lint-action@v1
+        with:
+          input: 'proto'
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -105,11 +105,33 @@ specify exactly the dependency you want to update, eg.
 
 ## Protobuf
 
-We use [Protocol Buffers](https://developers.google.com/protocol-buffers) along with [gogoproto](https://github.com/gogo/protobuf) to generate code for use across Tendermint Core.
+We use [Protocol Buffers](https://developers.google.com/protocol-buffers) along
+with [`gogoproto`](https://github.com/gogo/protobuf) to generate code for use
+across Tendermint Core.
 
-For linting, checking breaking changes and generating proto stubs, we use [buf](https://buf.build/). If you would like to run linting and check if the changes you have made are breaking then you will need to have docker running locally. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`.
+To generate proto stubs, lint, and check protos for breaking changes, you will
+need to install [buf](https://buf.build/) and `gogoproto`. Then, from the root
+of the repository, run:
 
-We use [Docker](https://www.docker.com/) to generate the protobuf stubs. To generate the stubs yourself, make sure docker is running then run `make proto-gen`. This command uses the spec repo to get the necessary protobuf files for generating the go code. If you are modifying the proto files manually for changes in the core data structures, you will need to clone them into the go repo and comment out lines 22-37 of the file `./scripts/protocgen.sh`.
+```bash
+# Lint all of the .proto files in proto/tendermint
+make proto-lint
+
+# Check if any of your local changes (prior to committing to the Git repository)
+# are breaking
+make proto-check-breaking
+
+# Generate Go code from the .proto files in proto/tendermint
+make proto-gen
+```
+
+To automatically format `.proto` files, you will need
+[`clang-format`](https://clang.llvm.org/docs/ClangFormat.html) installed. Once
+installed, you can run:
+
+```bash
+make proto-format
+```
 
 ### Visual Studio Code
 

diff --git a/Makefile b/Makefile
@@ -13,8 +13,6 @@ endif
 
 LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
-BUILD_IMAGE := ghcr.io/tendermint/docker-build-proto
-DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(BUILD_IMAGE)
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -73,47 +71,57 @@ install:
 
 $(BUILDDIR)/:
 	mkdir -p $@
-# The Docker image containing the generator, formatter, and linter.
-# This is generated by proto/Dockerfile. To update tools, make changes
-# there and run the Build & Push Proto Builder Image workflow.
-IMAGE := ghcr.io/tendermint/docker-build-proto:latest
-DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
-HTTPS_GIT := https://github.com/tendermint/tendermint.git
 
 ###############################################################################
 ###                                Protobuf                                 ###
 ###############################################################################
 
-proto-all: proto-lint proto-check-breaking
-.PHONY: proto-all
+check-proto-deps:
+ifeq (,$(shell which buf))
+	$(error "buf is required for Protobuf building, linting and breakage checking. See https://docs.buf.build/installation for installation instructions.")
+endif
+ifeq (,$(shell which protoc-gen-gogofaster))
+	$(error "gogofaster plugin for protoc is required. Run 'go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest' to install")
+endif
+.PHONY: check-proto-deps
+
+check-proto-format-deps:
+ifeq (,$(shell which clang-format))
+	$(error "clang-format is required for Protobuf formatting. See instructions for your platform on how to install it.")
+endif
+.PHONY: check-proto-format-deps
 
-proto-gen:
+proto-gen: check-proto-deps
 	@echo "Generating Protobuf files"
-	@$(DOCKER_PROTO_BUILDER) buf generate --template=./buf.gen.yaml --config ./buf.yaml
+	@buf generate
+	@mv ./proto/tendermint/abci/types.pb.go ./abci/types/
 .PHONY: proto-gen
 
-# TODO: Should be removed when work on ABCI++ is complete.
-# For more information, see https://github.com/tendermint/tendermint/issues/8066
-abci-proto-gen:
-	./scripts/abci-gen.sh
-.PHONY: abci-proto-gen
-
-proto-lint:
-	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json --config ./buf.yaml
+# These targets are provided for convenience and are intended for local
+# execution only.
+proto-lint: check-proto-deps
+	@echo "Linting Protobuf files"
+	@buf lint
 .PHONY: proto-lint
 
-proto-format:
+proto-format: check-proto-format-deps
 	@echo "Formatting Protobuf files"
-	@$(DOCKER_PROTO_BUILDER) find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \;
+	@find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \;
 .PHONY: proto-format
 
-proto-check-breaking:
-	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git --config ./buf.yaml
+proto-check-breaking: check-proto-deps
+	@echo "Checking for breaking changes in Protobuf files against local branch"
+	@echo "Note: This is only useful if your changes have not yet been committed."
+	@echo "      Otherwise read up on buf's \"breaking\" command usage:"
+	@echo "      https://docs.buf.build/breaking/usage"
+	@buf breaking --against ".git"
 .PHONY: proto-check-breaking
 
-proto-check-breaking-ci:
-	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT) --config ./buf.yaml
-.PHONY: proto-check-breaking-ci
+# TODO: Should be removed when work on ABCI++ is complete.
+# For more information, see https://github.com/tendermint/tendermint/issues/8066
+abci-proto-gen:
+	./scripts/abci-gen.sh
+.PHONY: abci-proto-gen
 
 ###############################################################################
 ###                              Build ABCI                                 ###

diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -1,14 +1,9 @@
-# The version of the generation template (required).
-# The only currently-valid value is v1beta1.
-version: v1beta1
-
-# The plugins to run.
+version: v1
 plugins:
-  # The name of the plugin.
   - name: gogofaster
-    # The directory where the generated proto output will be written.
-    # The directory is relative to where the generation tool was run.
-    out: proto
-    # Set options to assign import paths to the well-known types
-    # and to enable service generation.
-    opt: Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative
+    out: ./proto/
+    opt:
+      - Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types
+      - Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration
+      - plugins=grpc
+      - paths=source_relative
diff --git a/buf.work.yaml b/buf.work.yaml
@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - proto
diff --git a/proto/Dockerfile b/proto/Dockerfile
diff --git a/proto/buf.lock b/proto/buf.lock
@@ -0,0 +1,7 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: gogo
+    repository: protobuf
+    commit: 4df00b267f944190a229ce3695781e99
diff --git a/buf.yaml → proto/buf.yaml b/buf.yaml → proto/buf.yaml
@@ -1,16 +1,11 @@
-version: v1beta1
-
-build:
-  roots:
-    - proto
-    - third_party/proto
+version: v1
+deps:
+  - buf.build/gogo/protobuf
+breaking:
+  use:
+    - FILE
 lint:
   use:
     - BASIC
     - FILE_LOWER_SNAKE_CASE
     - UNARY_RPC
-  ignore:
-    - gogoproto
-breaking:
-  use:
-    - FILE
diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package tendermint.blocksync;
 
+option go_package = "github.com/tendermint/tendermint/proto/tendermint/blocksync";
+
 import "tendermint/types/block.proto";
 
 // BlockRequest requests a block for a specific height

diff --git a/proto/tendermint/consensus/types.proto b/proto/tendermint/consensus/types.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package tendermint.consensus;
 
+option go_package = "github.com/tendermint/tendermint/proto/tendermint/consensus";
+
 import "gogoproto/gogo.proto";
 import "tendermint/types/types.proto";
 import "tendermint/libs/bits/types.proto";

diff --git a/proto/tendermint/consensus/wal.proto b/proto/tendermint/consensus/wal.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package tendermint.consensus;
 
+option go_package = "github.com/tendermint/tendermint/proto/tendermint/consensus";
+
 import "gogoproto/gogo.proto";
 import "tendermint/consensus/types.proto";
 import "tendermint/types/events.proto";

diff --git a/proto/tendermint/crypto/keys.proto b/proto/tendermint/crypto/keys.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package tendermint.crypto;
 
+option go_package = "github.com/tendermint/tendermint/proto/tendermint/crypto";
+
 import "gogoproto/gogo.proto";
 
 // PublicKey defines the keys available for use with Tendermint Validators

diff --git a/proto/tendermint/crypto/proof.proto b/proto/tendermint/crypto/proof.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 package tendermint.crypto;
 
+option go_package = "github.com/tendermint/tendermint/proto/tendermint/crypto";
+
 import "gogoproto/gogo.proto";
 
 message Proof {