Add fuzz test and fix parser bugs

Author: blacktear23

.gitignore

@@ -17,3 +17,4 @@ bin/
 .idea/
 release
 *.pprof
+query/testdata

query/lexer.go

@@ -141,7 +141,7 @@ func (l *Lexer) Split() []*Token {
 				tokLen++
 				break
 			}
-			curr = l.Query[tokStart : tokStart+tokLen]
+			curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 			if token := buildToken(curr, tokStartPos); token != nil {
 				ret = append(ret, token)
 			}
@@ -156,7 +156,7 @@ func (l *Lexer) Split() []*Token {
 				tokStart = i + 1
 			} else if strStartChar == char {
 				strStart = false
-				curr = l.Query[tokStart : tokStart+tokLen]
+				curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 				token := &Token{
 					Tp:   STRING,
 					Data: curr,
@@ -175,7 +175,7 @@ func (l *Lexer) Split() []*Token {
 				tokStart = i + 1
 			} else if strStartChar == char {
 				strStart = false
-				curr = l.Query[tokStart : tokStart+tokLen]
+				curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 				token := &Token{
 					Tp:   NAME,
 					Data: curr,
@@ -191,7 +191,7 @@ func (l *Lexer) Split() []*Token {
 				tokLen++
 				break
 			}
-			curr = l.Query[tokStart : tokStart+tokLen]
+			curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 			if token := buildToken(curr, tokStartPos); token != nil {
 				ret = append(ret, token)
 			}
@@ -271,7 +271,7 @@ func (l *Lexer) Split() []*Token {
 				tokLen++
 				break
 			}
-			curr = l.Query[tokStart : tokStart+tokLen]
+			curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 			token := buildToken(curr, tokStartPos)
 			if token != nil {
 				ret = append(ret, token)
@@ -317,7 +317,7 @@ func (l *Lexer) Split() []*Token {
 				tokLen++
 				break
 			}
-			curr = l.Query[tokStart : tokStart+tokLen]
+			curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 			token := buildToken(curr, tokStartPos)
 			if token != nil {
 				ret = append(ret, token)
@@ -337,7 +337,7 @@ func (l *Lexer) Split() []*Token {
 		prev = char
 	}
 	if tokLen > 0 {
-		curr = l.Query[tokStart : tokStart+tokLen]
+		curr = l.Query[tokStart : tokStart+min(tokLen, l.Length-tokStart)]
 		if token := buildToken(curr, tokStartPos); token != nil {
 			ret = append(ret, token)
 		}
@@ -425,5 +425,4 @@ func buildToken(curr string, pos int) *Token {
 		token.Tp = NAME
 		return token
 	}
-	return nil
 }

query/optimizer.go

@@ -156,7 +156,7 @@ func (o *Optimizer) buildFinalPlan(t Txn, fp Plan) (FinalPlan, error) {
 	return ffp, nil
 }
 
-func (o *Optimizer) BuildPlan(t Txn) (FinalPlan, error) {
+func (o *Optimizer) buildPlan(t Txn) (FinalPlan, error) {
 	err := o.init()
 	if err != nil {
 		return nil, err
@@ -184,6 +184,14 @@ func (o *Optimizer) BuildPlan(t Txn) (FinalPlan, error) {
 	if err != nil {
 		return nil, err
 	}
+	return ret, nil
+}
+
+func (o *Optimizer) BuildPlan(t Txn) (FinalPlan, error) {
+	ret, err := o.buildPlan(t)
+	if err != nil {
+		return nil, err
+	}
 	err = ret.Init()
 	if err != nil {
 		return nil, err

query/parser.go

@@ -377,6 +377,9 @@ func (p *Parser) parseSelect() (*SelectStmt, error) {
 				return nil, NewSyntaxError(p.tok.Pos, "Invalid field expression")
 			}
 			if len(fields) > 0 {
+				if p.tok == nil {
+					return nil, NewSyntaxError(-1, "Invalid field expression")
+				}
 				return nil, NewSyntaxError(p.tok.Pos, "Invalid field expression")
 			}
 			break
@@ -389,7 +392,9 @@ func (p *Parser) parseSelect() (*SelectStmt, error) {
 		if p.tok != nil {
 			if p.tok.Tp == AS {
 				p.next()
-				if p.tok.Tp != NAME {
+				if p.tok == nil {
+					return nil, NewSyntaxError(-1, "Require field name")
+				} else if p.tok.Tp != NAME {
 					return nil, NewSyntaxError(p.tok.Pos, "Invalid field name")
 				}
 				fieldName = p.tok.Data
@@ -644,10 +649,12 @@ func (p *Parser) parseOrderBy(selStmt *SelectStmt) (*OrderStmt, error) {
 
 func (p *Parser) Parse() (*SelectStmt, error) {
 	if p.numToks == 0 {
-		return nil, NewSyntaxError(p.tok.Pos, "Expect select or where keyword")
+		return nil, NewSyntaxError(-1, "Expect select or where keyword")
 	}
 	p.next()
-	if p.tok == nil || (p.tok.Tp != WHERE && p.tok.Tp != SELECT) {
+	if p.tok == nil {
+		return nil, NewSyntaxError(-1, "Expect select or where keyword")
+	} else if p.tok.Tp != WHERE && p.tok.Tp != SELECT {
 		return nil, NewSyntaxError(p.tok.Pos, "Expect select or where keyword")
 	}
 	var (

query/parser_fuzz_test.go

@@ -0,0 +1,21 @@
+package query
+
+import "testing"
+
+func FuzzSQLParser(f *testing.F) {
+	tests := []string{
+		"select key, int(value) where int(key) + 1 >= 1 & (int(value) - 1 > 10 | int(value) <= 20)",
+		"select key, int(value) where key ^= 'key' order by key limit 20, 10",
+		"select * where key in ('k1', 'k2', 'k3')",
+		"select * where (key between 'k1' and 'k3') & int(value) between 1 and 10",
+		"select key, json(value)['test'] where key ^= 'k' & json(value)['test'][1] = 'v1'",
+	}
+
+	for _, t := range tests {
+		f.Add(t)
+	}
+	f.Fuzz(func(t *testing.T, query string) {
+		o := NewOptimizer(query)
+		o.buildPlan(nil)
+	})
+}