Aarchitecture for QuackPipe

Init version for quackpipe architecture

controller/root/root.go

@@ -0,0 +1,50 @@
+package root
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/http"
+	"quackpipe/model"
+	"quackpipe/service/db"
+	"quackpipe/utils"
+)
+
+func QueryOperation(flagInformation *model.CommandLineFlags, query string, r *http.Request, default_path string, default_format string, default_params string) (string, error) {
+	// auth to hash based temp file storage
+	username, password, ok := r.BasicAuth()
+	hashdb := ""
+	if ok && len(password) > 0 {
+		hash := sha256.Sum256([]byte(username + password))
+		hashdb = fmt.Sprintf("%s/%x.db", default_path, hash)
+	}
+	// extract FORMAT from query and override the current `default_format`
+	cleanquery, format := utils.ExtractAndRemoveFormat(query)
+	if len(format) > 0 {
+		query = cleanquery
+		default_format = format
+	}
+
+	if len(format) > 0 {
+		query = cleanquery
+		default_format = format
+	}
+
+	if len(query) == 0 {
+		return "", errors.New("query length is empty")
+	} else {
+		rows, duration, err := db.Quack(*flagInformation, query, false, default_params, hashdb)
+		if err != nil {
+			return "", err
+		} else {
+
+			result, err := utils.ConversationOfRows(rows, default_format, duration)
+			if err != nil {
+				return "", err
+			}
+			return result, nil
+		}
+	}
+
+	return "", nil
+}

go.mod

@@ -1,20 +1,7 @@
 module quackpipe
 
-go 1.20
+go 1.21
 
-require github.com/marcboeker/go-duckdb v1.7.0
+toolchain go1.22.4
 
-require (
-	github.com/apache/arrow/go/v14 v14.0.2 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/google/flatbuffers v23.5.26+incompatible // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
-	github.com/zeebo/xxh3 v1.0.2 // indirect
-	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-)
+require github.com/gorilla/mux v1.8.1

go.sum

@@ -1,36 +1,2 @@
-github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw=
-github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
-github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
-github.com/marcboeker/go-duckdb v1.7.0 h1:c9DrS13ta+gqVgg9DiEW8I+PZBE85nBMLL/YMooYoUY=
-github.com/marcboeker/go-duckdb v1.7.0/go.mod h1:WtWeqqhZoTke/Nbd7V9lnBx7I2/A/q0SAq/urGzPCMs=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
-github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
-golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
-golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=

main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"quackpipe/model"
+	"quackpipe/route"
+	"quackpipe/utils"
+)
+
+// initFlags initializes the command line flags
+func initFlags() *model.CommandLineFlags {
+
+	appFlags := &model.CommandLineFlags{}
+	appFlags.Host = flag.String("host", "0.0.0.0", "API host. Default 0.0.0.0")
+	appFlags.Port = flag.String("port", "8123", "API port. Default 8123")
+	appFlags.Format = flag.String("format", "JSONCompact", "API port. Default JSONCompact")
+	appFlags.Params = flag.String("params", "", "DuckDB optional parameters. Default to none.")
+	appFlags.DBPath = flag.String("dbpath", "/tmp/", "DuckDB DB storage path. Default to /tmp/")
+	appFlags.Stdin = flag.Bool("stdin", false, "STDIN query. Default false")
+	appFlags.Alias = flag.Bool("alias", true, "Built-in CH Aliases. Default true")
+	flag.Parse()
+
+	return appFlags
+}
+
+var appFlags *model.CommandLineFlags
+
+func main() {
+	appFlags = initFlags()
+	if *appFlags.Stdin {
+		rows, duration, format, err := utils.ReadFromScanner(*appFlags)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		results, err := utils.ConversationOfRows(rows, format, duration)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		} else {
+			fmt.Println(results)
+		}
+
+	} else {
+		r := route.NewRouter(appFlags)
+		fmt.Printf("QuackPipe API Running: %s:%s\n", *appFlags.Host, *appFlags.Port)
+		if err := http.ListenAndServe(*appFlags.Host+":"+*appFlags.Port, r); err != nil {
+			panic(err)
+		}
+
+	}
+
+}

model/flags.go

@@ -0,0 +1,12 @@
+package model
+
+// params for Flags
+type CommandLineFlags struct {
+	Host   *string `json:"host"`
+	Port   *string `json:"port"`
+	Stdin  *bool   `json:"stdin"`
+	Alias  *bool   `json:"alias"`
+	Format *string `json:"format"`
+	Params *string `json:"params"`
+	DBPath *string `json:"dbpath"`
+}

model/helper.go

@@ -0,0 +1,23 @@
+package model
+
+// Metadata is the metadata for a column
+type Metadata struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
+// Statistics is the statistics for a query
+type Statistics struct {
+	Elapsed   float64 `json:"elapsed"`
+	RowsRead  int     `json:"rows_read"`
+	BytesRead int     `json:"bytes_read"`
+}
+
+// OutputJSON is the JSON output for a query
+type OutputJSON struct {
+	Meta                   []Metadata      `json:"meta"`
+	Data                   [][]interface{} `json:"data"`
+	Rows                   int             `json:"rows"`
+	RowsBeforeLimitAtLeast int             `json:"rows_before_limit_at_least"`
+	Statistics             Statistics      `json:"statistics"`
+}