From 9ae5f6e62afe6c78d1f6d8b7baae7b3ce1a56637 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 25 Mar 2023 23:28:07 +0100 Subject: [PATCH 1/8] Create fluxpipe-lambda.go Signed-off-by: Lorenzo Mangani --- fluxpipe-lambda.go | 108 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 fluxpipe-lambda.go diff --git a/fluxpipe-lambda.go b/fluxpipe-lambda.go new file mode 100644 index 0000000..a02b519 --- /dev/null +++ b/fluxpipe-lambda.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/influxdata/flux" + "github.com/influxdata/flux/csv" + _ "github.com/influxdata/flux/fluxinit/static" + "github.com/influxdata/flux/lang" + "github.com/influxdata/flux/memory" + "github.com/influxdata/flux/runtime" + + _fluxhttp "github.com/influxdata/flux/dependencies/http" + "github.com/influxdata/flux/dependencies/secret" + "github.com/influxdata/flux/dependencies/url" + + "github.com/aws/aws-lambda-go/lambda" + +) + +var APPNAME = "fluxpipe-lambda" + +func runQuery(ctx context.Context, script string) (flux.Query, error) { + + program, err := lang.Compile(ctx, script, runtime.Default, time.Now()) + if err != nil { + return nil, err + } + + q, err := program.Start(ctx, memory.DefaultAllocator) + if err != nil { + return nil, err + } + return q, nil +} + +// NewCustomDependencies produces a Custom set of dependencies including EnvironmentSecretService. +func NewCustomDependencies() flux.Deps { + validator := url.PassValidator{} + return flux.Deps{ + Deps: flux.WrappedDeps{ + HTTPClient: _fluxhttp.NewLimitedDefaultClient(validator), + // Default to having no filesystem, no secrets, and no url validation (always pass). + FilesystemService: nil, + SecretService: secret.EnvironmentSecretService{}, + URLValidator: validator, + }, + } +} + +func exec(inputString string) (string, string) { + + // CustomDeps produces a Custom set of dependencies including EnvironmentSecretService. + customValidator := url.PassValidator{} + customDeps := flux.Deps{ + Deps: flux.WrappedDeps{ + HTTPClient: _fluxhttp.NewLimitedDefaultClient(customValidator), + FilesystemService: nil, + SecretService: secret.EnvironmentSecretService{}, + URLValidator: customValidator, + }, + } + + // ctx := flux.NewDefaultDependencies().Inject(context.Background()) + ctx := customDeps.Inject(context.Background()) + + q, err := runQuery(ctx, inputString) + if err != nil { + fmt.Println("unexpected error while creating query: %s", err) + return "", string(fmt.Sprintf(`{"code":"invalid","message":"%v"}`, err)) + } + + results := flux.NewResultIteratorFromQuery(q) + defer results.Release() + + buf := bytes.NewBuffer(nil) + encoder := csv.NewMultiResultEncoder(csv.DefaultEncoderConfig()) + + if _, err := encoder.Encode(buf, results); err != nil { + panic(err) + } + + q.Done() + + if q.Err() != nil { + fmt.Println("unexpected error from query execution: %s", q.Err()) + return "", string(fmt.Sprintf(`{"code":"invalid","message":"%v"}`, q.Err())) + + } else { + return buf.String(), "" + } +} + +type FluxEvent struct { + Query string `json:"query"` +} + +func HandleRequest(ctx context.Context, query FluxEvent) (string, error) { + return exec(name.Query) +} + +func main() { + lambda.Start(HandleRequest) +} From 22f6d3445344cd2f58257543d1e18c2e7729cc4d Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 25 Mar 2023 23:48:31 +0100 Subject: [PATCH 2/8] Update fluxpipe-lambda.go Signed-off-by: Lorenzo Mangani --- fluxpipe-lambda.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluxpipe-lambda.go b/fluxpipe-lambda.go index a02b519..1454b70 100644 --- a/fluxpipe-lambda.go +++ b/fluxpipe-lambda.go @@ -18,7 +18,7 @@ import ( "github.com/influxdata/flux/dependencies/secret" "github.com/influxdata/flux/dependencies/url" - "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-lambda-go/lambda" ) From 9c489599c721be5604c330110700993824e80876 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:01:07 +0100 Subject: [PATCH 3/8] mod tidy --- go.mod | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 22f9413..6050c66 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/lmangani/fluxpipe go 1.18 require ( + github.com/aws/aws-lambda-go v1.13.3 github.com/influxdata/flux v0.193.0 github.com/labstack/echo/v4 v4.2.1 ) diff --git a/go.sum b/go.sum index cfd38dd..ea0cde6 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= From 3c8899b1f43c098d8f640b795fe32033ed654d01 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:04:37 +0100 Subject: [PATCH 4/8] Update build.sh Signed-off-by: Lorenzo Mangani --- build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sh b/build.sh index dbba786..8d47071 100755 --- a/build.sh +++ b/build.sh @@ -3,3 +3,5 @@ export PKG_CONFIG_PATH=$(pwd) echo "Building fluxpipe-server ..." go build -a -ldflags '-extldflags "-static -w -ldl"' -o fluxpipe-server fluxpipe-server.go +echo "Building fluxpipe-lambda ..." +go build -a -ldflags '-extldflags "-static -w -ldl"' -o fluxpipe-lambda fluxpipe-lambda.go From 1e2984412ecbaf36fa80eac50ac85ff481b07efd Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:09:15 +0100 Subject: [PATCH 5/8] Update go.yml Signed-off-by: Lorenzo Mangani --- .github/workflows/go.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4f5cbac..78d9cb8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -74,6 +74,7 @@ jobs: - name: Compress Fluxpipe run: | strip fluxpipe-server + strip fluxpipe-lambda upx fluxpipe-server - id: tag_bump @@ -106,6 +107,7 @@ jobs: with: release_config: | fluxpipe-server + fluxpipe-lambda tag_name: ${{ env.VERSION }} release_name: fluxpipe_${{ env.VERSION }} draft: false @@ -120,7 +122,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker Build and push + - name: Docker Build and push (server) if: github.event_name != 'pull_request' uses: docker/build-push-action@v3.0.0 with: @@ -129,3 +131,15 @@ jobs: tags: | ghcr.io/metrico/fluxpipe:latest ghcr.io/metrico/fluxpipe:${{ env.VERSION }} + + - name: Docker Build and push (lambda) + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v3.0.0 + with: + context: . + file: Dockerfile.lambda + push: true + tags: | + ghcr.io/metrico/fluxpipe-lambda:latest + ghcr.io/metrico/fluxpipe-lambda:${{ env.VERSION }} + From 3e269d3db9e0a5522e89009e9421d200c3ad9f01 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:10:22 +0100 Subject: [PATCH 6/8] Create Dockerfile.lambda Signed-off-by: Lorenzo Mangani --- Dockerfile.lambda | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Dockerfile.lambda diff --git a/Dockerfile.lambda b/Dockerfile.lambda new file mode 100644 index 0000000..bf14b59 --- /dev/null +++ b/Dockerfile.lambda @@ -0,0 +1,9 @@ +FROM alpine as builder +RUN apk add --allow-untrusted --update --no-cache curl ca-certificates +WORKDIR / +RUN curl -fsSL github.com/metrico/fluxpipe/releases/latest/download/fluxpipe-lambda -O && chmod +x fluxpipe-lambda + +FROM scratch +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /fluxpipe-lambda /fluxpipe-lambda +CMD ["/fluxpipe-lambda"] From 0bec97b16aba3a2414f743de2532eea09bd9092c Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:22:55 +0100 Subject: [PATCH 7/8] Update fluxpipe-lambda.go Signed-off-by: Lorenzo Mangani --- fluxpipe-lambda.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fluxpipe-lambda.go b/fluxpipe-lambda.go index 1454b70..8b830d8 100644 --- a/fluxpipe-lambda.go +++ b/fluxpipe-lambda.go @@ -2,10 +2,9 @@ package main import ( "context" - "encoding/json" "fmt" - "strings" "time" + "bytes" "github.com/influxdata/flux" "github.com/influxdata/flux/csv" @@ -96,11 +95,12 @@ func exec(inputString string) (string, string) { } type FluxEvent struct { - Query string `json:"query"` + Flux string `json:"query"` } func HandleRequest(ctx context.Context, query FluxEvent) (string, error) { - return exec(name.Query) + buf, _ := exec(Flux.query) + return buf, nil } func main() { From e73dfc8d7c78a292fb8637d4c5caf5d64e11c64d Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 26 Mar 2023 01:34:52 +0100 Subject: [PATCH 8/8] Update fluxpipe-lambda.go Signed-off-by: Lorenzo Mangani --- fluxpipe-lambda.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fluxpipe-lambda.go b/fluxpipe-lambda.go index 8b830d8..023eff0 100644 --- a/fluxpipe-lambda.go +++ b/fluxpipe-lambda.go @@ -95,11 +95,11 @@ func exec(inputString string) (string, string) { } type FluxEvent struct { - Flux string `json:"query"` + Query string `json:"flux"` } -func HandleRequest(ctx context.Context, query FluxEvent) (string, error) { - buf, _ := exec(Flux.query) +func HandleRequest(ctx context.Context, flux FluxEvent) (string, error) { + buf, _ := exec(flux.Query) return buf, nil }