Skip to content

Commit 4cee3ef

Browse files
keejonkeejon
committed
poc: use Buf and protocompile for protobuf schemas
1 parent c6a0948 commit 4cee3ef

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+5856
-79
lines changed

‎MANIFEST.in

+1
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ include setup.py
99
include setup.cfg
1010
include LICENSE
1111
include MANIFEST.in
12+
include *.so
1213

1314
recursive-exclude examples *~ *.pyc \.*

‎go/protopace/.gitignore

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# If you prefer the allow list template instead of the deny list, see community template:
2+
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
3+
#
4+
# Binaries for programs and plugins
5+
*.exe
6+
*.exe~
7+
*.dll
8+
*.so
9+
*.dylib
10+
11+
# Test binary, built with `go test -c`
12+
*.test
13+
14+
# Output of the go coverage tool, specifically when used with LiteIDE
15+
*.out
16+
17+
# Dependency directories (remove the comment below to include it)
18+
# vendor/
19+
20+
# Go workspace file
21+
go.work
22+
go.work.sum
23+
24+
# env file
25+
.env

‎go/protopace/Makefile

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Change these variables as necessary.
2+
MAIN_PACKAGE_PATH := .
3+
BINARY_NAME := protopace
4+
BUILD_DIR := ../../karapace/protobuf/protopace/bin
5+
6+
# ==================================================================================== #
7+
# HELPERS
8+
# ==================================================================================== #
9+
10+
## help: print this help message
11+
.PHONY: help
12+
help:
13+
@echo 'Usage:'
14+
@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /'
15+
16+
.PHONY: confirm
17+
confirm:
18+
@echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ]
19+
20+
.PHONY: no-dirty
21+
no-dirty:
22+
git diff --exit-code
23+
24+
25+
# ==================================================================================== #
26+
# QUALITY CONTROL
27+
# ==================================================================================== #
28+
29+
## tidy: format code and tidy modfile
30+
.PHONY: tidy
31+
tidy:
32+
go fmt ./...
33+
go mod tidy -v
34+
35+
## audit: run quality control checks
36+
.PHONY: audit
37+
audit:
38+
go mod verify
39+
go vet ./...
40+
go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
41+
go run golang.org/x/vuln/cmd/govulncheck@latest ./...
42+
go test -race -buildvcs -vet=off ./...
43+
44+
45+
# ==================================================================================== #
46+
# DEVELOPMENT
47+
# ==================================================================================== #
48+
49+
## test: run all tests
50+
.PHONY: test
51+
test:
52+
go test -v -race -buildvcs ./...
53+
54+
## test/cover: run all tests and display coverage
55+
.PHONY: test/cover
56+
test/cover:
57+
go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./...
58+
go tool cover -html=/tmp/coverage.out
59+
60+
## build: build the application
61+
.PHONY: build
62+
build:
63+
# Include additional build steps, like TypeScript, SCSS or Tailwind compilation here...
64+
go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH}
65+
66+
## run: run the application
67+
.PHONY: run
68+
run: build
69+
/tmp/bin/${BINARY_NAME}
70+
71+
## run/live: run the application with reloading on file changes
72+
.PHONY: run/live
73+
run/live:
74+
go run github.com/cosmtrek/[email protected] \
75+
--build.cmd "make build" --build.bin "/tmp/bin/${BINARY_NAME}" --build.delay "100" \
76+
--build.exclude_dir "" \
77+
--build.include_ext "go, tpl, tmpl, html, css, scss, js, ts, sql, jpeg, jpg, gif, png, bmp, svg, webp, ico" \
78+
--misc.clean_on_exit "true"
79+
80+
81+
# ==================================================================================== #
82+
# OPERATIONS
83+
# ==================================================================================== #
84+
85+
## release: cross-compile to build dir on mac
86+
.PHONY: release
87+
release:
88+
CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
89+
CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
90+
91+
docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/amd64 golang:1.22 env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-amd64.so
92+
cp ${BINARY_NAME}-linux-amd64.so ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.so
93+
cp ${BINARY_NAME}-linux-amd64.h ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.h
94+
rm ${BINARY_NAME}-linux-amd64.so
95+
rm ${BINARY_NAME}-linux-amd64.h
96+
97+
docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/arm64 golang:1.22 env GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-arm64.so
98+
cp ${BINARY_NAME}-linux-arm64.so ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.so
99+
cp ${BINARY_NAME}-linux-arm64.h ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.h
100+
rm ${BINARY_NAME}-linux-arm64.so
101+
rm ${BINARY_NAME}-linux-arm64.h

‎go/protopace/compatibility.go

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package main
2+
3+
import (
4+
"context"
5+
6+
"github.com/Aiven-Open/karapace/go/protopace/schema"
7+
8+
"github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking"
9+
"github.com/bufbuild/buf/private/bufpkg/bufconfig"
10+
"github.com/bufbuild/buf/private/pkg/tracing"
11+
"go.uber.org/zap"
12+
)
13+
14+
func Check(schema schema.Schema, previousSchema schema.Schema) error {
15+
handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer)
16+
ctx := context.Background()
17+
image, err := schema.CompileBufImage()
18+
if err != nil {
19+
return err
20+
}
21+
previousImage, err := previousSchema.CompileBufImage()
22+
if err != nil {
23+
return err
24+
}
25+
checkConfig, _ := bufconfig.NewEnabledCheckConfig(
26+
bufconfig.FileVersionV2,
27+
nil,
28+
[]string{
29+
"FIELD_NO_DELETE",
30+
"FILE_SAME_PACKAGE",
31+
"FIELD_SAME_NAME",
32+
"FIELD_SAME_JSON_NAME",
33+
"FILE_NO_DELETE",
34+
"ENUM_NO_DELETE",
35+
},
36+
nil,
37+
nil,
38+
)
39+
config := bufconfig.NewBreakingConfig(checkConfig, false)
40+
return handler.Check(ctx, config, previousImage, image)
41+
}

‎go/protopace/compatibility_test.go

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"testing"
6+
7+
s "github.com/Aiven-Open/karapace/go/protopace/schema"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestCompatibility(t *testing.T) {
12+
assert := assert.New(t)
13+
14+
data, _ := os.ReadFile("./fixtures/dependency.proto")
15+
dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil)
16+
assert.NoError(err)
17+
assert.NotNil(dependencySchema)
18+
19+
data, _ = os.ReadFile("./fixtures/test.proto")
20+
testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
21+
assert.NoError(err)
22+
assert.NotNil(testSchema)
23+
24+
data, _ = os.ReadFile("./fixtures/test_previous.proto")
25+
previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
26+
assert.NoError(err)
27+
assert.NotNil(previousSchema)
28+
29+
err = Check(*testSchema, *testSchema)
30+
assert.NoError(err)
31+
32+
err = Check(*testSchema, *previousSchema)
33+
assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".")
34+
}

‎go/protopace/fixtures/dependency.proto

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
syntax = "proto3";
2+
package my.awesome.customer.v1;
3+
4+
message NestedValue {
5+
string value = 1;
6+
}
7+
8+
enum Status {
9+
UNKNOWN = 0;
10+
ACTIVE = 1;
11+
INACTIVE = 2;
12+
}

‎go/protopace/fixtures/test.proto

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
syntax = "proto3";
2+
3+
package my.awesome.customer.v1;
4+
5+
import "my/awesome/customer/v1/nested_value.proto";
6+
import "google/protobuf/timestamp.proto";
7+
8+
option ruby_package = "My::Awesome::Customer::V1";
9+
option csharp_namespace = "my.awesome.customer.V1";
10+
option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
11+
option java_multiple_files = true;
12+
option java_outer_classname = "EventValueProto";
13+
option java_package = "com.my.awesome.customer.v1";
14+
option objc_class_prefix = "TDD";
15+
option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
16+
option php_namespace = "My\\Awesome\\Customer\\V1";
17+
18+
message Local {
19+
message NestedValue {
20+
string foo = 1;
21+
}
22+
}
23+
24+
message EventValue {
25+
NestedValue nested_value = 1;
26+
google.protobuf.Timestamp created_at = 2;
27+
Status status = 3;
28+
Local.NestedValue local_nested_value = 4;
29+
int32 foo = 5;
30+
}

‎go/protopace/fixtures/test_previous.proto

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
syntax = "proto3";
2+
3+
package my.awesome.customer.v1;
4+
5+
import "my/awesome/customer/v1/nested_value.proto";
6+
import "google/protobuf/timestamp.proto";
7+
8+
option ruby_package = "My::Awesome::Customer::V1";
9+
option csharp_namespace = "my.awesome.customer.V1";
10+
option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
11+
option java_multiple_files = true;
12+
option java_outer_classname = "EventValueProto";
13+
option java_package = "com.my.awesome.customer.v1";
14+
option objc_class_prefix = "TDD";
15+
option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
16+
option php_namespace = "My\\Awesome\\Customer\\V1";
17+
18+
message Local {
19+
message NestedValue {
20+
string foo = 1;
21+
}
22+
}
23+
24+
message EventValue {
25+
NestedValue nested_value = 1;
26+
google.protobuf.Timestamp created_at = 2;
27+
Status status = 3;
28+
Local.NestedValue local_nested_value = 4;
29+
string foo = 5;
30+
}

0 commit comments

Comments
 (0)