diff --git a/Dockerfile b/Dockerfile index e36aade..53ea7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.20 AS builder WORKDIR / COPY . . -RUN CGO_ENABLED=1 go build -o quackpipe quackpipe.go +RUN CGO_ENABLED=1 go build -o quackpipe . RUN strip quackpipe RUN apt update && apt install -y libgrpc-dev diff --git a/controller/root/root.go b/controller/root/root.go index 169e265..c89c1d3 100644 --- a/controller/root/root.go +++ b/controller/root/root.go @@ -31,7 +31,7 @@ func QueryOperation(flagInformation *model.CommandLineFlags, query string, r *ht } if len(query) == 0 { - return "", errors.New("query length is empty") + return "", errors.New("") } else { rows, duration, err := db.Quack(*flagInformation, query, false, defaultParams, hashdb) if err != nil { diff --git a/go.mod b/go.mod index 4e1664f..38fe3be 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,23 @@ module quackpipe -go 1.21 +go 1.20 -toolchain go1.22.4 +require ( + github.com/gorilla/mux v1.8.1 + github.com/marcboeker/go-duckdb v1.7.0 +) -require github.com/gorilla/mux v1.8.1 +require ( + github.com/apache/arrow/go/v14 v14.0.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect +) diff --git a/go.sum b/go.sum index 7128337..3b1fb07 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,47 @@ +github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= +github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/marcboeker/go-duckdb v1.7.0 h1:c9DrS13ta+gqVgg9DiEW8I+PZBE85nBMLL/YMooYoUY= +github.com/marcboeker/go-duckdb v1.7.0/go.mod h1:WtWeqqhZoTke/Nbd7V9lnBx7I2/A/q0SAq/urGzPCMs= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handler/api_handler.go b/handler/api_handler.go index be3b6d7..4630255 100644 --- a/handler/api_handler.go +++ b/handler/api_handler.go @@ -57,7 +57,7 @@ func (u *Handler) Handlers(w http.ResponseWriter, r *http.Request) { } result, err := root.QueryOperation(u.FlagInformation, query, r, defaultPath, defaultFormat, defaultParams) - if err != nil && strings.Contains(err.Error(), "query length is empty") { + if err != nil && strings.Contains(err.Error(), "") { _, _ = w.Write([]byte(staticPlay)) } if err != nil { diff --git a/quackpipe.go b/quackpipe.go deleted file mode 100644 index 52a9e3b..0000000 --- a/quackpipe.go +++ /dev/null @@ -1,386 +0,0 @@ -package main - -// -//import ( -// "bufio" -// "context" -// "crypto/sha256" -// "database/sql" -// _ "embed" -// "encoding/json" -// "fmt" -// "io/ioutil" -// "log" -// "net/http" -// "os" -// "regexp" -// "strings" -// "time" -// -// _ "github.com/marcboeker/go-duckdb" // load duckdb driver -//) -// -////go:embed route/root/play.html -//var staticPlay string -// -//// params for Flags -//type CommandLineFlags struct { -// Host *string `json:"host"` -// Port *string `json:"port"` -// Stdin *bool `json:"stdin"` -// Alias *bool `json:"alias"` -// Format *string `json:"format"` -// Params *string `json:"params"` -// DBPath *string `json:"dbpath"` -//} -// -//var appFlags CommandLineFlags -// -//var db *sql.DB -// -//func check(args ...interface{}) { -// err := args[len(args)-1] -// if err != nil { -// panic(err) -// } -//} -// -//func quack(query string, stdin bool, format string, params string, hashdb string) (string, error) { -// var err error -// alias := *appFlags.Alias -// motherduck, md := os.LookupEnv("motherduck_token") -// -// if len(hashdb) > 0 { -// params = hashdb + "?" + params -// } -// -// db, err = sql.Open("duckdb", params) -// if err != nil { -// log.Fatal(err) -// } -// defer db.Close() -// -// if !stdin { -// check(db.ExecContext(context.Background(), "LOAD httpfs; LOAD json; LOAD parquet;")) -// check(db.ExecContext(context.Background(), "SET autoinstall_known_extensions=1;")) -// check(db.ExecContext(context.Background(), "SET autoload_known_extensions=1;")) -// } -// -// if alias { -// check(db.ExecContext(context.Background(), "LOAD chsql;")) -// } -// -// if (md) && (motherduck != "") { -// check(db.ExecContext(context.Background(), "LOAD motherduck; ATTACH 'md:';")) -// } -// -// startTime := time.Now() -// rows, err := db.QueryContext(context.Background(), query) -// if err != nil { -// return "", err -// } -// elapsedTime := time.Since(startTime) -// -// switch format { -// case "JSONCompact", "JSON": -// return rowsToJSON(rows, elapsedTime) -// case "CSVWithNames": -// return rowsToCSV(rows, true) -// case "TSVWithNames", "TabSeparatedWithNames": -// return rowsToTSV(rows, true) -// case "TSV", "TabSeparated": -// return rowsToTSV(rows, false) -// default: -// return rowsToTSV(rows, false) -// } -//} -// -//////initFlags initializes the command line flags -////func initFlags() { -//// appFlags.Host = flag.String("host", "0.0.0.0", "API host. Default 0.0.0.0") -//// appFlags.Port = flag.String("port", "8123", "API port. Default 8123") -//// appFlags.Format = flag.String("format", "JSONCompact", "API port. Default JSONCompact") -//// appFlags.Params = flag.String("params", "", "DuckDB optional parameters. Default to none.") -//// appFlags.DBPath = flag.String("dbpath", "/tmp/", "DuckDB DB storage path. Default to /tmp/") -//// appFlags.Stdin = flag.Bool("stdin", false, "STDIN query. Default false") -//// appFlags.Alias = flag.Bool("alias", true, "Built-in CH Aliases. Default true") -//// flag.Parse() -////} -// -//// extractAndRemoveFormat extracts the FORMAT clause from the query and returns the query without the FORMAT clause -//func extractAndRemoveFormat(input string) (string, string) { -// re := regexp.MustCompile(`(?i)\bFORMAT\s+(\w+)\b`) -// match := re.FindStringSubmatch(input) -// if len(match) != 2 { -// return input, "" -// } -// format := match[1] -// return re.ReplaceAllString(input, ""), format -//} -// -//// Metadata is the metadata for a column -//type Metadata struct { -// Name string `json:"name"` -// Type string `json:"type"` -//} -// -//// Statistics is the statistics for a query -//type Statistics struct { -// Elapsed float64 `json:"elapsed"` -// RowsRead int `json:"rows_read"` -// BytesRead int `json:"bytes_read"` -//} -// -//// OutputJSON is the JSON output for a query -//type OutputJSON struct { -// Meta []Metadata `json:"meta"` -// Data [][]interface{} `json:"data"` -// Rows int `json:"rows"` -// RowsBeforeLimitAtLeast int `json:"rows_before_limit_at_least"` -// Statistics Statistics `json:"statistics"` -//} -// -//// rowsToJSON converts the rows to JSON string -//func rowsToJSON(rows *sql.Rows, elapsedTime time.Duration) (string, error) { -// defer rows.Close() -// -// // Get column names -// columns, err := rows.Columns() -// if err != nil { -// return "", err -// } -// -// // Create a slice to store maps of column names and their corresponding values -// var results OutputJSON -// results.Meta = make([]Metadata, len(columns)) -// results.Data = make([][]interface{}, 0) -// -// for i, column := range columns { -// results.Meta[i].Name = column -// } -// -// for rows.Next() { -// // Create a slice to hold pointers to the values of the columns -// values := make([]interface{}, len(columns)) -// for i := range columns { -// values[i] = new(interface{}) -// } -// -// // Scan the values from the row into the pointers -// err := rows.Scan(values...) -// if err != nil { -// return "", err -// } -// -// // Create a slice to hold the row data -// rowData := make([]interface{}, len(columns)) -// for i, value := range values { -// // Convert the value to the appropriate Go type -// switch v := (*(value.(*interface{}))).(type) { -// case []byte: -// rowData[i] = string(v) -// default: -// rowData[i] = v -// } -// } -// results.Data = append(results.Data, rowData) -// } -// -// err = rows.Err() -// if err != nil { -// return "", err -// } -// -// results.Rows = len(results.Data) -// results.RowsBeforeLimitAtLeast = len(results.Data) -// -// // Populate the statistics object with number of rows, bytes, and elapsed time -// results.Statistics.Elapsed = elapsedTime.Seconds() -// results.Statistics.RowsRead = results.Rows -// // Note: bytes_read is an approximation, it's just the number of rows * number of columns -// // results.Statistics.BytesRead = results.Rows * len(columns) * 8 // Assuming each value takes 8 bytes -// jsonData, err := json.Marshal(results) -// if err != nil { -// return "", err -// } -// -// return string(jsonData), nil -//} -// -//// rowsToTSV converts the rows to TSV string -//func rowsToTSV(rows *sql.Rows, cols bool) (string, error) { -// var result []string -// columns, err := rows.Columns() -// if err != nil { -// return "", err -// } -// -// if cols { -// // Append column names as the first row -// result = append(result, strings.Join(columns, "\t")) -// } -// -// // Fetch rows and append their values as tab-delimited lines -// values := make([]interface{}, len(columns)) -// scanArgs := make([]interface{}, len(columns)) -// for i := range values { -// scanArgs[i] = &values[i] -// } -// for rows.Next() { -// err := rows.Scan(scanArgs...) -// if err != nil { -// return "", err -// } -// -// var lineParts []string -// for _, v := range values { -// lineParts = append(lineParts, fmt.Sprintf("%v", v)) -// } -// result = append(result, strings.Join(lineParts, "\t")) -// } -// -// if err := rows.Err(); err != nil { -// return "", err -// } -// -// return strings.Join(result, "\n"), nil -//} -// -//// rowsToCSV converts the rows to CSV string -//func rowsToCSV(rows *sql.Rows, cols bool) (string, error) { -// var result []string -// columns, err := rows.Columns() -// if err != nil { -// return "", err -// } -// -// if cols { -// // Append column names as the first row -// result = append(result, strings.Join(columns, ",")) -// } -// -// // Fetch rows and append their values as CSV rows -// values := make([]interface{}, len(columns)) -// scanArgs := make([]interface{}, len(columns)) -// for i := range values { -// scanArgs[i] = &values[i] -// } -// for rows.Next() { -// err := rows.Scan(scanArgs...) -// if err != nil { -// return "", err -// } -// -// var lineParts []string -// for _, v := range values { -// lineParts = append(lineParts, fmt.Sprintf("%v", v)) -// } -// result = append(result, strings.Join(lineParts, ",")) -// } -// -// if err := rows.Err(); err != nil { -// return "", err -// } -// -// return strings.Join(result, "\n"), nil -//} -// -//func main() { -// initFlags() -// default_format := *appFlags.Format -// default_params := *appFlags.Params -// default_path := *appFlags.DBPath -// -// if *appFlags.Stdin { -// scanner := bufio.NewScanner((os.Stdin)) -// query := "" -// for scanner.Scan() { -// query = query + "\n" + scanner.Text() -// } -// if err := scanner.Err(); err != nil { -// fmt.Fprintln(os.Stderr, "reading standard input:", err) -// } -// cleanquery, format := extractAndRemoveFormat(query) -// if len(format) > 0 { -// query = cleanquery -// default_format = format -// } -// result, err := quack(query, true, default_format, default_params, "") -// if err != nil { -// fmt.Println(err) -// os.Exit(1) -// } else { -// fmt.Println(result) -// } -// } else { -// http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { -// var bodyBytes []byte -// var query string -// var err error -// -// // handle query parameter -// if r.URL.Query().Get("query") != "" { -// // query = r.FormValue("query") -// query = r.URL.Query().Get("query") -// } else if r.Body != nil { -// bodyBytes, err = ioutil.ReadAll(r.Body) -// if err != nil { -// fmt.Printf("Body reading error: %v", err) -// return -// } -// defer r.Body.Close() -// query = string(bodyBytes) -// } -// -// switch r.Header.Get("Accept") { -// case "application/json": -// w.Header().Set("Content-Type", "application/json; charset=utf-8") -// case "application/xml": -// w.Header().Set("Content-Type", "application/xml; charset=utf-8") -// case "text/css": -// w.Header().Set("Content-Type", "text/css; charset=utf-8") -// default: -// w.Header().Set("Content-Type", "text/html; charset=utf-8") -// } -// -// // format handling -// if r.URL.Query().Get("default_format") != "" { -// default_format = r.URL.Query().Get("default_format") -// } -// // param handling -// if r.URL.Query().Get("default_params") != "" { -// default_params = r.URL.Query().Get("default_params") -// } -// // auth to hash based temp file storage -// username, password, ok := r.BasicAuth() -// hashdb := "" -// if ok && len(password) > 0 { -// hash := sha256.Sum256([]byte(username + password)) -// hashdb = fmt.Sprintf("%s/%x.db", default_path, hash) -// } -// -// // extract FORMAT from query and override the current `default_format` -// cleanquery, format := extractAndRemoveFormat(query) -// if len(format) > 0 { -// query = cleanquery -// default_format = format -// } -// -// if len(query) == 0 { -// _, _ = w.Write([]byte(staticPlay)) -// } else { -// result, err := quack(query, false, default_format, default_params, hashdb) -// if err != nil { -// _, _ = w.Write([]byte(err.Error())) -// } else { -// _, _ = w.Write([]byte(result)) -// } -// } -// }) -// -// fmt.Printf("QuackPipe API Running: %s:%s\n", *appFlags.Host, *appFlags.Port) -// if err := http.ListenAndServe(*appFlags.Host+":"+*appFlags.Port, nil); err != nil { -// panic(err) -// } -// } -//}