From 950eb8a04b3fa25888bebd59a67a01d0e17cd60d Mon Sep 17 00:00:00 2001 From: Jay D Date: Wed, 14 Jun 2023 18:43:25 +0800 Subject: [PATCH 1/6] feat: Add server-side support for `Airtable` --- go.mod | 9 +- go.sum | 7 - pkg/action/factory.go | 5 + pkg/action/service.go | 9 +- pkg/app/service.go | 4 +- pkg/plugins/airtable/base.go | 444 ++++++++++++++++++++++++++++++++ pkg/plugins/airtable/service.go | 114 ++++++++ pkg/plugins/airtable/types.go | 93 +++++++ pkg/resource/factory.go | 5 + pkg/resource/service.go | 9 +- 10 files changed, 676 insertions(+), 23 deletions(-) create mode 100644 pkg/plugins/airtable/base.go create mode 100644 pkg/plugins/airtable/service.go create mode 100644 pkg/plugins/airtable/types.go diff --git a/go.mod b/go.mod index 90ccd22e..247aaa97 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-resty/resty/v2 v2.7.0 github.com/go-sql-driver/mysql v1.7.0 github.com/golang-jwt/jwt/v4 v4.4.3 + github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 github.com/google/wire v0.5.0 github.com/gorilla/mux v1.8.0 @@ -38,7 +39,9 @@ require ( github.com/stretchr/testify v1.8.1 go.mongodb.org/mongo-driver v1.11.1 go.uber.org/zap v1.24.0 + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 google.golang.org/api v0.105.0 + google.golang.org/protobuf v1.30.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gorm.io/driver/postgres v1.4.5 gorm.io/gorm v1.24.2 @@ -94,11 +97,9 @@ require ( github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/subcommands v1.0.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect @@ -141,21 +142,17 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.4.0 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.5.0 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/term v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect google.golang.org/grpc v1.51.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 63196432..291abe19 100644 --- a/go.sum +++ b/go.sum @@ -315,7 +315,6 @@ github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1V github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -746,8 +745,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -896,8 +893,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -967,8 +962,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/pkg/action/factory.go b/pkg/action/factory.go index 7af54854..ba655975 100644 --- a/pkg/action/factory.go +++ b/pkg/action/factory.go @@ -15,6 +15,7 @@ package action import ( + "github.com/illacloud/builder-backend/pkg/plugins/airtable" "github.com/illacloud/builder-backend/pkg/plugins/appwrite" "github.com/illacloud/builder-backend/pkg/plugins/clickhouse" "github.com/illacloud/builder-backend/pkg/plugins/common" @@ -65,6 +66,7 @@ var ( GOOGLESHEETS_ACTION = "googlesheets" NEON_ACTION = "neon" UPSTASH_ACTION = "upstash" + AIRTABLE_ACTION = "airtable" ) type AbstractActionFactory interface { @@ -137,6 +139,9 @@ func (f *Factory) Build() common.DataConnector { case GOOGLESHEETS_ACTION: googlesheetsAction := &googlesheets.Connector{} return googlesheetsAction + case AIRTABLE_ACTION: + airtableAction := &airtable.Connector{} + return airtableAction default: return nil } diff --git a/pkg/action/service.go b/pkg/action/service.go index d720b383..8157b05d 100644 --- a/pkg/action/service.go +++ b/pkg/action/service.go @@ -25,9 +25,9 @@ import ( "go.uber.org/zap" ) -var type_array = [26]string{"transformer", "restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", +var type_array = [27]string{"transformer", "restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", "tidb", "elasticsearch", "s3", "smtp", "supabasedb", "firebase", "clickhouse", "mssql", "huggingface", "dynamodb", - "snowflake", "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash"} + "snowflake", "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash", "airtable"} var type_map = map[string]int{ "transformer": 0, "restapi": 1, @@ -55,6 +55,7 @@ var type_map = map[string]int{ "googlesheets": 23, "neon": 24, "upstash": 25, + "airtable": 26, } type ActionService interface { @@ -77,7 +78,7 @@ type ActionDto struct { Version int `json:"-"` Resource int `json:"resourceId,omitempty"` DisplayName string `json:"displayName" validate:"required"` - Type string `json:"actionType" validate:"oneof=transformer restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash"` + Type string `json:"actionType" validate:"oneof=transformer restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash airtable"` Template map[string]interface{} `json:"content" validate:"required"` Transformer map[string]interface{} `json:"transformer" validate:"required"` TriggerMode string `json:"triggerMode" validate:"oneof=manually automate"` @@ -96,7 +97,7 @@ type ActionDtoForExport struct { Version int `json:"-"` Resource string `json:"resourceId,omitempty"` DisplayName string `json:"displayName" validate:"required"` - Type string `json:"actionType" validate:"oneof=transformer restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash"` + Type string `json:"actionType" validate:"oneof=transformer restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash airtable"` Template map[string]interface{} `json:"content" validate:"required"` Transformer map[string]interface{} `json:"transformer" validate:"required"` TriggerMode string `json:"triggerMode" validate:"oneof=manually automate"` diff --git a/pkg/app/service.go b/pkg/app/service.go index 49c9959f..e2a318d7 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -49,9 +49,9 @@ type AppServiceImpl struct { actionRepository repository.ActionRepository } -var type_array = [26]string{"transformer", "restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", +var type_array = [27]string{"transformer", "restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", "tidb", "elasticsearch", "s3", "smtp", "supabasedb", "firebase", "clickhouse", "mssql", "huggingface", "dynamodb", - "snowflake", "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash"} + "snowflake", "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash", "airtable"} type AppDto struct { ID int `json:"appId"` // generated by database primary key serial diff --git a/pkg/plugins/airtable/base.go b/pkg/plugins/airtable/base.go new file mode 100644 index 00000000..e83d5dea --- /dev/null +++ b/pkg/plugins/airtable/base.go @@ -0,0 +1,444 @@ +// Copyright 2023 Illa Soft, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package airtable + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/go-playground/validator/v10" + "github.com/go-resty/resty/v2" + "github.com/illacloud/builder-backend/pkg/plugins/common" + "github.com/mitchellh/mapstructure" +) + +func (a *Connector) ListRecords() (common.RuntimeResult, error) { + // format `list` method config + var listConfig ListConfig + if err := mapstructure.Decode(a.Action.Config, &listConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `list` method config + if listConfig.PageSize <= 0 { + listConfig.PageSize = 100 + } + if listConfig.CellFormat == STRING_CELL_FORMAT { + if listConfig.TimeZone == "" && listConfig.UserLocale == "" { + return common.RuntimeResult{Success: false}, errors.New("missing timezone or user locale") + } + } else { + listConfig.CellFormat = JSON_CELL_FORMAT + } + + // build `List Records` request body + listReqBody := make(map[string]interface{}) + if len(listConfig.Fields) > 0 { + listReqBody["fields"] = listConfig.Fields + } + if listConfig.FilterByFormula != "" { + listReqBody["filterByFormula"] = listConfig.FilterByFormula + } + if listConfig.MaxRecords > 0 { + listReqBody["maxRecords"] = listConfig.MaxRecords + } + if listConfig.PageSize > 0 { + listReqBody["pageSize"] = listConfig.PageSize + } + sortObjs := make([]map[string]string, 0) + for _, sortObj := range listConfig.Sort { + if sortObj.Field != "" { + sortMap := map[string]string{ + "field": sortObj.Field, + "direction": sortObj.Direction, + } + sortObjs = append(sortObjs, sortMap) + } + } + listReqBody["sort"] = sortObjs + if listConfig.View != "" { + listReqBody["view"] = listConfig.View + } + if listConfig.CellFormat != "" { + listReqBody["cellFormat"] = listConfig.CellFormat + } + if listConfig.TimeZone != "" { + listReqBody["timeZone"] = listConfig.TimeZone + } + if listConfig.UserLocale != "" { + listReqBody["userLocale"] = listConfig.UserLocale + } + if listConfig.Offset != "" { + listReqBody["offset"] = listConfig.Offset + } + + // call `List Records` method + restyClient := resty.New() + listReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + listReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + listReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := listReq.SetBody(listReqBody). + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Post(AIRTABLE_API) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) GetRecord() (common.RuntimeResult, error) { + // format `get` method config + var getConfig GetConfig + if err := mapstructure.Decode(a.Action.Config, &getConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `get` method config + validate := validator.New() + if err := validate.Struct(getConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // call `Get Record` method + restyClient := resty.New() + getReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + getReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + getReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := getReq. + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Get(AIRTABLE_API + "/" + getConfig.RecordID) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) CreateRecords() (common.RuntimeResult, error) { + // format `create` method config + var createConfig CreateConfig + if err := mapstructure.Decode(a.Action.Config, &createConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `create` method config + validate := validator.New() + if err := validate.Struct(createConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // build `Create Records` request body + createReqBody := make(map[string]interface{}, 1) + if len(createConfig.Records) == 1 { + createReqBody = createConfig.Records[0] + } else if len(createConfig.Records) > 1 { + records := make([]map[string]interface{}, 0) + for _, record := range createConfig.Records { + records = append(records, record) + } + createReqBody["records"] = records + } + + // call `Create Records` method + restyClient := resty.New() + createReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + createReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + createReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := createReq.SetBody(createReqBody). + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Post(AIRTABLE_API) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) UpdateMultipleRecords() (common.RuntimeResult, error) { + // format `bulkUpdate` method config + var bulkUpdateConfig BulkUpdateConfig + if err := mapstructure.Decode(a.Action.Config, &bulkUpdateConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `bulkUpdate` method config + validate := validator.New() + if err := validate.Struct(bulkUpdateConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // build `Update Multiple Records` request body + bulkUpdateReqBody := make(map[string]interface{}, 1) + bulkUpdateReqBody["records"] = bulkUpdateConfig.Records + + // call `Update Multiple Records` method + restyClient := resty.New() + bulkUpdateReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + bulkUpdateReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + bulkUpdateReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := bulkUpdateReq.SetBody(bulkUpdateReqBody). + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Patch(AIRTABLE_API) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) UpdateRecord() (common.RuntimeResult, error) { + // format `update` method config + var updateConfig UpdateConfig + if err := mapstructure.Decode(a.Action.Config, &updateConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `update` method config + validate := validator.New() + if err := validate.Struct(updateConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // build `Update Multiple Records` request body + updateReqBody := make(map[string]interface{}) + updateReqBody = updateConfig.Record + + // call `Update Multiple Records` method + restyClient := resty.New() + updateReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + updateReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + updateReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := updateReq.SetBody(updateReqBody). + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Patch(AIRTABLE_API + "/" + updateConfig.RecordId) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) DeleteMultipleRecords() (common.RuntimeResult, error) { + // format `bulkDelete` method config + var bulkDeleteConfig BulkDeleteConfig + if err := mapstructure.Decode(a.Action.Config, &bulkDeleteConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `bulkDelete` method config + validate := validator.New() + if err := validate.Struct(bulkDeleteConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // call `Delete Multiple Records` method + deleteIds := make([]string, 0, len(bulkDeleteConfig.RecordIds)) + for _, ids := range bulkDeleteConfig.RecordIds { + deleteIds = append(deleteIds, fmt.Sprintf("records=%s", ids)) + } + deleteIdsQueryParams := "?" + strings.Join(deleteIds, "&") + restyClient := resty.New() + deleteReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + deleteReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + deleteReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := deleteReq. + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Delete(AIRTABLE_API + "/" + deleteIdsQueryParams) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func (a *Connector) DeleteRecord() (common.RuntimeResult, error) { + // format `delete` method config + var deleteConfig DeleteConfig + if err := mapstructure.Decode(a.Action.Config, &deleteConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // validate `delete` method config + validate := validator.New() + if err := validate.Struct(deleteConfig); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // call `Delete Record` method + restyClient := resty.New() + deleteReq := restyClient.R().SetHeader("Content-Type", "application/json") + if a.Resource.AuthenticationType == API_KEY_AUTHENTICATION { + deleteReq.SetAuthToken(a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]) + } else if a.Resource.AuthenticationType == PERSONAL_TOKEN_AUTHENTICATION { + deleteReq.SetAuthToken(a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]) + } + resp, errRun := deleteReq. + SetPathParams(map[string]string{ + "baseId": a.Action.BaseConfig.BaseID, + "tableName": a.Action.BaseConfig.TableName, + }). + Delete(AIRTABLE_API + "/" + deleteConfig.RecordId) + + // handle response + if resp.StatusCode() != http.StatusOK { + if errRun != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Request to Airtable failed: " + errRun.Error()}}}, nil + } + respMap, errParseError := parseAirtableResponse(resp.String()) + if errParseError != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseError.Error()}}}, nil + } + respMap["status"] = resp.Status() + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil + } + respMap, errParseResp := parseAirtableResponse(resp.String()) + if errParseResp != nil { + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"message": "Parse Airtable response error: " + errParseResp.Error()}}}, nil + } + + return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{respMap}}, nil +} + +func parseAirtableResponse(responseString string) (map[string]interface{}, error) { + var respMap map[string]interface{} + if err := json.Unmarshal([]byte(responseString), &respMap); err != nil { + return nil, err + } + return respMap, nil +} diff --git a/pkg/plugins/airtable/service.go b/pkg/plugins/airtable/service.go new file mode 100644 index 00000000..bf54f349 --- /dev/null +++ b/pkg/plugins/airtable/service.go @@ -0,0 +1,114 @@ +// Copyright 2023 Illa Soft, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package airtable + +import ( + "errors" + + "github.com/go-playground/validator/v10" + "github.com/illacloud/builder-backend/pkg/plugins/common" + "github.com/mitchellh/mapstructure" +) + +type Connector struct { + Resource Resource + Action Action +} + +func (a *Connector) ValidateResourceOptions(resourceOpts map[string]interface{}) (common.ValidateResult, error) { + // format resource options + if err := mapstructure.Decode(resourceOpts, &a.Resource); err != nil { + return common.ValidateResult{Valid: false}, err + } + + // validate Airtable options + validate := validator.New() + if err := validate.Struct(a.Resource); err != nil { + return common.ValidateResult{Valid: false}, err + } + switch a.Resource.AuthenticationType { + case API_KEY_AUTHENTICATION: + if _, ok := a.Resource.AuthenticationConfig[API_KEY_AUTHENTICATION]; !ok { + return common.ValidateResult{Valid: false}, errors.New("missing API key") + } + case PERSONAL_TOKEN_AUTHENTICATION: + if _, ok := a.Resource.AuthenticationConfig[TOKEN_AUTHENTICATION]; !ok { + return common.ValidateResult{Valid: false}, errors.New("missing Personal Access Token") + } + default: + return common.ValidateResult{Valid: false}, errors.New("invalid parameters") + } + + return common.ValidateResult{Valid: true}, nil +} + +func (a *Connector) ValidateActionOptions(actionOpts map[string]interface{}) (common.ValidateResult, error) { + // format action options + if err := mapstructure.Decode(actionOpts, &a.Action); err != nil { + return common.ValidateResult{Valid: false}, err + } + + // validate Airtable options + validate := validator.New() + if err := validate.Struct(a.Action); err != nil { + return common.ValidateResult{Valid: false}, err + } + + return common.ValidateResult{Valid: true}, nil +} + +func (a *Connector) TestConnection(resourceOpts map[string]interface{}) (common.ConnectionResult, error) { + return common.ConnectionResult{Success: true}, nil +} + +func (a *Connector) GetMetaInfo(resourceOpts map[string]interface{}) (common.MetaInfoResult, error) { + return common.MetaInfoResult{Success: true}, nil +} + +func (a *Connector) Run(resourceOpts map[string]interface{}, actionOpts map[string]interface{}) (common.RuntimeResult, error) { + // format resource options + if err := mapstructure.Decode(resourceOpts, &a.Resource); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // format action options + if err := mapstructure.Decode(actionOpts, &a.Action); err != nil { + return common.RuntimeResult{Success: false}, err + } + + // run action based on action method + var result common.RuntimeResult + var errRun error + switch a.Action.Method { + case LIST_METHOD: + result, errRun = a.ListRecords() + case GET_METHOD: + result, errRun = a.GetRecord() + case CREATE_METHOD: + result, errRun = a.CreateRecords() + case BULKUPDATE_METHOD: + result, errRun = a.UpdateMultipleRecords() + case UPDATE_METHOD: + result, errRun = a.UpdateRecord() + case BULKDELETE_METHOD: + result, errRun = a.DeleteMultipleRecords() + case DELETE_METHOD: + result, errRun = a.DeleteRecord() + default: + errRun = errors.New("invalid action method") + } + + return result, errRun +} diff --git a/pkg/plugins/airtable/types.go b/pkg/plugins/airtable/types.go new file mode 100644 index 00000000..09dfe7a6 --- /dev/null +++ b/pkg/plugins/airtable/types.go @@ -0,0 +1,93 @@ +// Copyright 2023 Illa Soft, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package airtable + +const ( + AIRTABLE_API = "https://api.airtable.com/v0/{baseId}/{tableName}" + + PERSONAL_TOKEN_AUTHENTICATION = "personalToken" + TOKEN_AUTHENTICATION = "token" + API_KEY_AUTHENTICATION = "apiKey" + + LIST_METHOD = "list" + GET_METHOD = "get" + CREATE_METHOD = "create" + UPDATE_METHOD = "update" + BULKUPDATE_METHOD = "bulkUpdate" + DELETE_METHOD = "delete" + BULKDELETE_METHOD = "bulkDelete" + + JSON_CELL_FORMAT = "json" + STRING_CELL_FORMAT = "string" +) + +type Resource struct { + AuthenticationType string `mapstructure:"authenticationType" validate:"oneof=personalToken apiKey"` + AuthenticationConfig map[string]string `mapstructure:"authenticationConfig" validate:"required"` +} + +type Action struct { + Method string `mapstructure:"method" validate:"oneof=list get create update bulkUpdate delete bulkDelete"` + BaseConfig BaseConfig `mapstructure:"baseConfig" validate:"required"` + Config map[string]interface{} `mapstructure:"config" validate:"required"` +} + +type BaseConfig struct { + BaseID string `mapstructure:"baseId" validate:"required"` + TableName string `mapstructure:"tableName" validate:"required"` +} + +type ListConfig struct { + Fields []string `mapstructure:"fields"` + FilterByFormula string `mapstructure:"filterByFormula"` + MaxRecords int `mapstructure:"maxRecords"` + PageSize int `mapstructure:"pageSize"` + Sort []SortObject `mapstructure:"sort"` + View string `mapstructure:"view"` + CellFormat string `mapstructure:"cellFormat"` + TimeZone string `mapstructure:"timeZone"` + UserLocale string `mapstructure:"userLocale"` + Offset string `mapstructure:"offset"` +} + +type SortObject struct { + Field string `mapstructure:"field"` + Direction string `mapstructure:"direction" validate:"oneof=asc desc"` +} + +type GetConfig struct { + RecordID string `mapstructure:"recordId" validate:"required"` +} + +type CreateConfig struct { + Records []map[string]interface{} `mapstructure:"records" validate:"required,gt=0,lt=11"` +} + +type BulkUpdateConfig struct { + Records []map[string]interface{} `mapstructure:"records" validate:"required,gt=0,lt=11"` +} + +type UpdateConfig struct { + RecordId string `mapstructure:"recordId" validate:"required"` + Record map[string]interface{} `mapstructure:"record" validate:"required"` +} + +type DeleteConfig struct { + RecordId string `mapstructure:"recordId" validate:"required"` +} + +type BulkDeleteConfig struct { + RecordIds []string `mapstructure:"recordIds" validate:"required,gt=0,lt=11"` +} diff --git a/pkg/resource/factory.go b/pkg/resource/factory.go index 0b8802ed..b7efd02e 100644 --- a/pkg/resource/factory.go +++ b/pkg/resource/factory.go @@ -15,6 +15,7 @@ package resource import ( + "github.com/illacloud/builder-backend/pkg/plugins/airtable" "github.com/illacloud/builder-backend/pkg/plugins/appwrite" "github.com/illacloud/builder-backend/pkg/plugins/clickhouse" "github.com/illacloud/builder-backend/pkg/plugins/common" @@ -64,6 +65,7 @@ var ( GOOGLESHEETS_RESOURCE = "googlesheets" NEON_RESOURCE = "neon" UPSTASH_RESOURCE = "upstash" + AIRTABLE_RESOURCE = "airtable" ) type AbstractResourceFactory interface { @@ -136,6 +138,9 @@ func (f *Factory) Generate() common.DataConnector { case GOOGLESHEETS_RESOURCE: googlesheetsRsc := &googlesheets.Connector{} return googlesheetsRsc + case AIRTABLE_RESOURCE: + airtableRsc := &airtable.Connector{} + return airtableRsc default: return nil } diff --git a/pkg/resource/service.go b/pkg/resource/service.go index 93eed2b4..ac987973 100644 --- a/pkg/resource/service.go +++ b/pkg/resource/service.go @@ -25,9 +25,9 @@ import ( "go.uber.org/zap" ) -var type_array = [25]string{"restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", "tidb", +var type_array = [26]string{"restapi", "graphql", "redis", "mysql", "mariadb", "postgresql", "mongodb", "tidb", "elasticsearch", "s3", "smtp", "supabasedb", "firebase", "clickhouse", "mssql", "huggingface", "dynamodb", "snowflake", - "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash"} + "couchdb", "hfendpoint", "oracle", "appwrite", "googlesheets", "neon", "upstash", "airtable"} var type_map = map[string]int{ "restapi": 1, "graphql": 2, @@ -54,6 +54,7 @@ var type_map = map[string]int{ "googlesheets": 23, "neon": 24, "upstash": 25, + "airtable": 26, } type ResourceService interface { @@ -72,7 +73,7 @@ type ResourceDto struct { UID uuid.UUID `json:"uid"` TeamID int `json:"teamID"` Name string `json:"resourceName" validate:"required,min=1,max=128"` - Type string `json:"resourceType" validate:"oneof=restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash"` + Type string `json:"resourceType" validate:"oneof=restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash airtable"` Options map[string]interface{} `json:"content" validate:"required"` CreatedAt time.Time `json:"createdAt,omitempty"` CreatedBy int `json:"createdBy,omitempty"` @@ -85,7 +86,7 @@ type ResourceDtoForExport struct { UID uuid.UUID `json:"uid"` TeamID string `json:"teamID"` Name string `json:"resourceName" validate:"required"` - Type string `json:"resourceType" validate:"oneof=restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash"` + Type string `json:"resourceType" validate:"oneof=restapi graphql redis mysql mariadb postgresql mongodb tidb elasticsearch s3 smtp supabasedb firebase clickhouse mssql huggingface dynamodb snowflake couchdb hfendpoint oracle appwrite googlesheets neon upstash airtable"` Options map[string]interface{} `json:"content" validate:"required"` CreatedAt time.Time `json:"createdAt,omitempty"` CreatedBy string `json:"createdBy,omitempty"` From 7dc35d8b011e152a1e39c35b749e2105e3f2cf07 Mon Sep 17 00:00:00 2001 From: Jay D Date: Wed, 14 Jun 2023 19:39:25 +0800 Subject: [PATCH 2/6] fix: Modify url in `Airtable` action --- pkg/plugins/airtable/base.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/airtable/base.go b/pkg/plugins/airtable/base.go index e83d5dea..3f47f6bb 100644 --- a/pkg/plugins/airtable/base.go +++ b/pkg/plugins/airtable/base.go @@ -100,7 +100,7 @@ func (a *Connector) ListRecords() (common.RuntimeResult, error) { "baseId": a.Action.BaseConfig.BaseID, "tableName": a.Action.BaseConfig.TableName, }). - Post(AIRTABLE_API) + Post(AIRTABLE_API + "/listRecords") // handle response if resp.StatusCode() != http.StatusOK { From c20cd4a8319c01c686d861c1507c6564b858eef1 Mon Sep 17 00:00:00 2001 From: Jay D Date: Fri, 16 Jun 2023 17:21:33 +0800 Subject: [PATCH 3/6] fix: Add `options` support --- pkg/plugins/mongodb/query.go | 127 +++++++++++++++++++++++++++++++++-- pkg/plugins/mongodb/types.go | 36 ++++++++++ 2 files changed, 158 insertions(+), 5 deletions(-) diff --git a/pkg/plugins/mongodb/query.go b/pkg/plugins/mongodb/query.go index bca69d23..2c0dc937 100644 --- a/pkg/plugins/mongodb/query.go +++ b/pkg/plugins/mongodb/query.go @@ -16,6 +16,7 @@ package mongodb import ( "context" + "encoding/json" "strconv" "github.com/illacloud/builder-backend/pkg/plugins/common" @@ -45,7 +46,27 @@ func (q *QueryRunner) aggregate() (common.RuntimeResult, error) { } } - cursor, err := coll.Aggregate(context.TODO(), aggregateStage) + // build `AggregateOptions` + var rawAggregateOptions map[string]interface{} + if err := json.Unmarshal([]byte(aggregateOptions.Options), &rawAggregateOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + var parsedAggregateOptions AggregateOptions + if err := mapstructure.Decode(rawAggregateOptions, &parsedAggregateOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + opts := options.Aggregate() + if _, ok := rawAggregateOptions["collation"]; ok { + opts = opts.SetCollation(parsedAggregateOptions.Collation) + } + if _, ok := rawAggregateOptions["hint"]; ok { + opts = opts.SetHint(parsedAggregateOptions.Hint) + } + if _, ok := rawAggregateOptions["batchSize"]; ok { + opts = opts.SetBatchSize(parsedAggregateOptions.BatchSize) + } + + cursor, err := coll.Aggregate(context.TODO(), aggregateStage, opts) if err != nil { return common.RuntimeResult{Success: false}, err } @@ -199,7 +220,21 @@ func (q *QueryRunner) distinct() (common.RuntimeResult, error) { } } - results, err := coll.Distinct(context.TODO(), distinctOptions.Field, filter) + // build `DistinctOptions` + var rawDistinctOptions map[string]interface{} + if err := json.Unmarshal([]byte(distinctOptions.Options), &rawDistinctOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + var parsedAggregateOptions DistinctOptions + if err := mapstructure.Decode(rawDistinctOptions, &parsedAggregateOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + opts := options.Distinct() + if _, ok := rawDistinctOptions["collation"]; ok { + opts = opts.SetCollation(parsedAggregateOptions.Collation) + } + + results, err := coll.Distinct(context.TODO(), distinctOptions.Field, filter, opts) if err != nil { return common.RuntimeResult{Success: false}, err } @@ -320,8 +355,44 @@ func (q *QueryRunner) findOneAndUpdate() (common.RuntimeResult, error) { } } + // build `FindOneAndUpdateOptions` + var rawFindOneAndUpdateOptions map[string]interface{} + if err := json.Unmarshal([]byte(fAUOptions.Options), &rawFindOneAndUpdateOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + var parsedFindOneAndUpdateOptions FindOneAndUpdateOptions + if err := mapstructure.Decode(rawFindOneAndUpdateOptions, &parsedFindOneAndUpdateOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + opts := options.FindOneAndUpdate() + if _, ok := rawFindOneAndUpdateOptions["collation"]; ok { + opts = opts.SetCollation(parsedFindOneAndUpdateOptions.Collation) + } + if _, ok := rawFindOneAndUpdateOptions["hint"]; ok { + opts = opts.SetHint(parsedFindOneAndUpdateOptions.Hint) + } + if _, ok := rawFindOneAndUpdateOptions["arrayFilters"]; ok { + opts = opts.SetArrayFilters(options.ArrayFilters{Filters: parsedFindOneAndUpdateOptions.ArrayFilters}) + } + if _, ok := rawFindOneAndUpdateOptions["upsert"]; ok { + opts = opts.SetUpsert(parsedFindOneAndUpdateOptions.Upsert) + } + if _, ok := rawFindOneAndUpdateOptions["projection"]; ok { + opts = opts.SetProjection(parsedFindOneAndUpdateOptions.Projection) + } + if _, ok := rawFindOneAndUpdateOptions["sort"]; ok { + opts = opts.SetSort(parsedFindOneAndUpdateOptions.Sort) + } + if _, ok := rawFindOneAndUpdateOptions["returnDocument"]; ok { + if parsedFindOneAndUpdateOptions.ReturnDocument == "after" { + opts = opts.SetReturnDocument(options.After) + } else if parsedFindOneAndUpdateOptions.ReturnDocument == "before" { + opts = opts.SetReturnDocument(options.Before) + } + } + var results bson.M - if err := coll.FindOneAndUpdate(context.TODO(), filter, update).Decode(&results); err != nil { + if err := coll.FindOneAndUpdate(context.TODO(), filter, update, opts).Decode(&results); err != nil { return common.RuntimeResult{Success: false}, err } return common.RuntimeResult{Success: true, Rows: []map[string]interface{}{{"result": results}}}, nil @@ -421,7 +492,30 @@ func (q *QueryRunner) updateMany() (common.RuntimeResult, error) { } } - results, err := coll.UpdateMany(context.TODO(), filter, update) + // build `UpdateManyOptions` + var rawUpdateManyOptions map[string]interface{} + if err := json.Unmarshal([]byte(uMOptions.Options), &rawUpdateManyOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + var parsedUpdateManyOptions UpdateManyOptions + if err := mapstructure.Decode(rawUpdateManyOptions, &parsedUpdateManyOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + opts := &options.UpdateOptions{} + if _, ok := rawUpdateManyOptions["collation"]; ok { + opts = opts.SetCollation(parsedUpdateManyOptions.Collation) + } + if _, ok := rawUpdateManyOptions["hint"]; ok { + opts = opts.SetHint(parsedUpdateManyOptions.Hint) + } + if _, ok := rawUpdateManyOptions["arrayFilters"]; ok { + opts = opts.SetArrayFilters(options.ArrayFilters{Filters: parsedUpdateManyOptions.ArrayFilters}) + } + if _, ok := rawUpdateManyOptions["upsert"]; ok { + opts = opts.SetUpsert(parsedUpdateManyOptions.Upsert) + } + + results, err := coll.UpdateMany(context.TODO(), filter, update, opts) if err != nil { return common.RuntimeResult{Success: false}, err } @@ -448,7 +542,30 @@ func (q *QueryRunner) updateOne() (common.RuntimeResult, error) { } } - results, err := coll.UpdateOne(context.TODO(), filter, update) + // build `UpdateManyOptions` + var rawUpdateOneOptions map[string]interface{} + if err := json.Unmarshal([]byte(uOOptions.Options), &rawUpdateOneOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + var parsedUpdateOneOptions UpdateOneOptions + if err := mapstructure.Decode(rawUpdateOneOptions, &parsedUpdateOneOptions); err != nil { + return common.RuntimeResult{Success: false}, err + } + opts := &options.UpdateOptions{} + if _, ok := rawUpdateOneOptions["collation"]; ok { + opts = opts.SetCollation(parsedUpdateOneOptions.Collation) + } + if _, ok := rawUpdateOneOptions["hint"]; ok { + opts = opts.SetHint(parsedUpdateOneOptions.Hint) + } + if _, ok := rawUpdateOneOptions["arrayFilters"]; ok { + opts = opts.SetArrayFilters(options.ArrayFilters{Filters: parsedUpdateOneOptions.ArrayFilters}) + } + if _, ok := rawUpdateOneOptions["upsert"]; ok { + opts = opts.SetUpsert(parsedUpdateOneOptions.Upsert) + } + + results, err := coll.UpdateOne(context.TODO(), filter, update, opts) if err != nil { return common.RuntimeResult{Success: false}, err } diff --git a/pkg/plugins/mongodb/types.go b/pkg/plugins/mongodb/types.go index a2a23919..b82a1081 100644 --- a/pkg/plugins/mongodb/types.go +++ b/pkg/plugins/mongodb/types.go @@ -14,6 +14,8 @@ package mongodb +import "go.mongodb.org/mongo-driver/mongo/options" + const ( STANDARD_FORMAT = "standard" DNSSEEDLIST_FORMAT = "mongodb+srv" @@ -131,3 +133,37 @@ type UpdateOneContent struct { type CommandContent struct { Document string } + +type AggregateOptions struct { + Collation *options.Collation + Hint interface{} + BatchSize int32 +} + +type DistinctOptions struct { + Collation *options.Collation +} + +type FindOneAndUpdateOptions struct { + Collation *options.Collation + Hint interface{} + ArrayFilters []interface{} + Upsert bool + Projection interface{} + Sort interface{} + ReturnDocument string +} + +type UpdateManyOptions struct { + Collation *options.Collation + Hint interface{} + ArrayFilters []interface{} + Upsert bool +} + +type UpdateOneOptions struct { + Collation *options.Collation + Hint interface{} + ArrayFilters []interface{} + Upsert bool +} From cded4271f4ead32d5ef90504c0f3d3aa6dee1670 Mon Sep 17 00:00:00 2001 From: Jay D Date: Fri, 16 Jun 2023 17:44:19 +0800 Subject: [PATCH 4/6] fix: Add corner case handler --- pkg/plugins/mongodb/query.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/plugins/mongodb/query.go b/pkg/plugins/mongodb/query.go index 2c0dc937..2cc46786 100644 --- a/pkg/plugins/mongodb/query.go +++ b/pkg/plugins/mongodb/query.go @@ -47,6 +47,9 @@ func (q *QueryRunner) aggregate() (common.RuntimeResult, error) { } // build `AggregateOptions` + if aggregateOptions.Options == "" { + aggregateOptions.Options = "{}" + } var rawAggregateOptions map[string]interface{} if err := json.Unmarshal([]byte(aggregateOptions.Options), &rawAggregateOptions); err != nil { return common.RuntimeResult{Success: false}, err @@ -221,6 +224,9 @@ func (q *QueryRunner) distinct() (common.RuntimeResult, error) { } // build `DistinctOptions` + if distinctOptions.Options == "" { + distinctOptions.Options = "{}" + } var rawDistinctOptions map[string]interface{} if err := json.Unmarshal([]byte(distinctOptions.Options), &rawDistinctOptions); err != nil { return common.RuntimeResult{Success: false}, err @@ -356,6 +362,9 @@ func (q *QueryRunner) findOneAndUpdate() (common.RuntimeResult, error) { } // build `FindOneAndUpdateOptions` + if fAUOptions.Options == "" { + fAUOptions.Options = "{}" + } var rawFindOneAndUpdateOptions map[string]interface{} if err := json.Unmarshal([]byte(fAUOptions.Options), &rawFindOneAndUpdateOptions); err != nil { return common.RuntimeResult{Success: false}, err @@ -493,6 +502,9 @@ func (q *QueryRunner) updateMany() (common.RuntimeResult, error) { } // build `UpdateManyOptions` + if uMOptions.Options == "" { + uMOptions.Options = "{}" + } var rawUpdateManyOptions map[string]interface{} if err := json.Unmarshal([]byte(uMOptions.Options), &rawUpdateManyOptions); err != nil { return common.RuntimeResult{Success: false}, err @@ -542,7 +554,10 @@ func (q *QueryRunner) updateOne() (common.RuntimeResult, error) { } } - // build `UpdateManyOptions` + // build `UpdateOneOptions` + if uOOptions.Options == "" { + uOOptions.Options = "{}" + } var rawUpdateOneOptions map[string]interface{} if err := json.Unmarshal([]byte(uOOptions.Options), &rawUpdateOneOptions); err != nil { return common.RuntimeResult{Success: false}, err From 699b6c760425657abf12c970434b9c2ab92dc437 Mon Sep 17 00:00:00 2001 From: Jay D Date: Wed, 21 Jun 2023 11:08:42 +0800 Subject: [PATCH 5/6] fix: remove `required` tag for some fileds of `airtable` action --- pkg/plugins/airtable/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/plugins/airtable/types.go b/pkg/plugins/airtable/types.go index 09dfe7a6..b8df86a1 100644 --- a/pkg/plugins/airtable/types.go +++ b/pkg/plugins/airtable/types.go @@ -45,8 +45,8 @@ type Action struct { } type BaseConfig struct { - BaseID string `mapstructure:"baseId" validate:"required"` - TableName string `mapstructure:"tableName" validate:"required"` + BaseID string `mapstructure:"baseId"` + TableName string `mapstructure:"tableName"` } type ListConfig struct { From 67f7ed68d637c8158ad9a898dc738cd2795b9215 Mon Sep 17 00:00:00 2001 From: Jay D Date: Wed, 21 Jun 2023 14:48:38 +0800 Subject: [PATCH 6/6] fix: Update `list` action fields in `airtable` --- pkg/plugins/airtable/base.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/plugins/airtable/base.go b/pkg/plugins/airtable/base.go index 3f47f6bb..c949c034 100644 --- a/pkg/plugins/airtable/base.go +++ b/pkg/plugins/airtable/base.go @@ -54,10 +54,10 @@ func (a *Connector) ListRecords() (common.RuntimeResult, error) { if listConfig.FilterByFormula != "" { listReqBody["filterByFormula"] = listConfig.FilterByFormula } - if listConfig.MaxRecords > 0 { + if listConfig.MaxRecords > -1 { listReqBody["maxRecords"] = listConfig.MaxRecords } - if listConfig.PageSize > 0 { + if listConfig.PageSize > -1 { listReqBody["pageSize"] = listConfig.PageSize } sortObjs := make([]map[string]string, 0)