feat: micro gc for the proper GC

metrico · Mar 11, 2024 · 1d30e44 · 1d30e44
1 parent a10a0b7
commit 1d30e44
Show file tree

Hide file tree

Showing 9 changed files with 725 additions and 108 deletions.
diff --git a/promql/index.js b/promql/index.js
@@ -16,8 +16,9 @@ module.exports.PSQLError = PSQLError
  * @param stepMs {number}
  */
 module.exports.rangeQuery = async (query, startMs, endMs, stepMs) => {
+  let resp
   try {
-    const resp = await prometheus.pqlRangeQuery(query, startMs, endMs, stepMs, module.exports.getData)
+    resp = await prometheus.pqlRangeQuery(query, startMs, endMs, stepMs, module.exports.getData)
     return JSON.parse(resp)
   } catch (e) {
     if (e instanceof prometheus.WasmError) {

diff --git a/wasm_parts/go.mod b/wasm_parts/go.mod
@@ -1,5 +1,9 @@
 module wasm_parts
 
+go 1.21
+
+toolchain go1.21.3
+
 replace (
 	cloud.google.com/go v0.65.0 => cloud.google.com/go v0.102.1
 	github.com/InfluxCommunity/influxdb3-go v0.2.0 => github.com/akvlad/influxdb3-go v0.0.1
@@ -10,7 +14,7 @@ replace (
 
 require (
 	github.com/alecthomas/participle/v2 v2.1.0
-	github.com/metrico/micro-gc v0.0.0-20240229134304-977775afb420
+	github.com/metrico/micro-gc v0.0.4
 	github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7
 	github.com/prometheus/prometheus v1.8.2-0.20220714142409-b41e0750abf5
 )
@@ -57,5 +61,3 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-go 1.20
diff --git a/wasm_parts/go.sum b/wasm_parts/go.sum
diff --git a/wasm_parts/main.go b/wasm_parts/main.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	gcContext "github.com/metrico/micro-gc/context"
+	"sync"
+
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -19,6 +21,11 @@ import (
 	"wasm_parts/types"
 )
 
+//go:linkname get sync.(*Pool).Get
+func get(p *sync.Pool) any {
+	panic("GET POOL")
+}
+
 var maxSamples = 5000000
 
 type ctx struct {
@@ -111,6 +118,8 @@ func transpileTraceQL(id uint32) int {
 		options = append(options, sql.STRING_OPT_INLINE_WITH)
 	}
 	str, err := sel.String(request.Ctx.CHSqlCtx, options...)
+	print(str)
+	print("\n")
 	if err != nil {
 		data[id].response = []byte(err.Error())
 		return 1
@@ -119,24 +128,30 @@ func transpileTraceQL(id uint32) int {
 	return 0
 }
 
-var eng *promql.Engine = nil
-var engC = 0
+var eng *promql.Engine = promql.NewEngine(promql.EngineOpts{
+	Logger:                   TestLogger{},
+	MaxSamples:               maxSamples,
+	Timeout:                  time.Second * 30,
+	ActiveQueryTracker:       nil,
+	LookbackDelta:            0,
+	NoStepSubqueryIntervalFn: nil,
+	EnableAtModifier:         false,
+	EnableNegativeOffset:     false,
+})
+var engC = func() *promql.Engine {
+	return promql.NewEngine(promql.EngineOpts{
+		Logger:                   TestLogger{},
+		MaxSamples:               maxSamples,
+		Timeout:                  time.Second * 30,
+		ActiveQueryTracker:       nil,
+		LookbackDelta:            0,
+		NoStepSubqueryIntervalFn: nil,
+		EnableAtModifier:         false,
+		EnableNegativeOffset:     false,
+	})
+}()
 
 func getEng() *promql.Engine {
-	if eng == nil || engC > 5 {
-		eng = promql.NewEngine(promql.EngineOpts{
-			Logger:                   TestLogger{},
-			MaxSamples:               maxSamples,
-			Timeout:                  time.Second * 30,
-			ActiveQueryTracker:       nil,
-			LookbackDelta:            0,
-			NoStepSubqueryIntervalFn: nil,
-			EnableAtModifier:         false,
-			EnableNegativeOffset:     false,
-		})
-		engC = 0
-	}
-	engC++
 	return eng
 }
 
@@ -343,7 +358,11 @@ func writeVector(v promql.Vector) string {
 	return jsonBuilder.String()
 }
 
-func main() {}
+func main() {
+	p := sync.Pool{}
+	a := p.Get()
+	_ = a
+}
 
 type TestLogger struct{}
 

diff --git a/wasm_parts/main.js b/wasm_parts/main.js
@@ -7,7 +7,7 @@ const { gunzipSync } = require('zlib')
 class WasmError extends Error {}
 module.exports.WasmError = WasmError
 
-let counter = 0
+let counter = 1
 
 const getWasm = (() => {
   const _Go = Go
@@ -20,6 +20,9 @@ const getWasm = (() => {
     run = true
     const _wasm = await WebAssembly.instantiate(
       gunzipSync(fs.readFileSync(WASM_URL)), go.importObject)
+    setInterval(() => {
+      console.log(`WASM SIZE: ${Math.floor(wasm.exports.memory.buffer.byteLength / 1024 / 1024)} MB`)
+    }, 5000)
     go.run(_wasm.instance)
     wasm = _wasm.instance
     wasm.exports.setMaxSamples(process.env.ADVANCED_PROMETHEUS_MAX_SAMPLES || 5000000)
@@ -28,10 +31,6 @@ const getWasm = (() => {
   }
   init()
   return () => {
-    if (cnt >= 20 && !run) {
-      init()
-    }
-    cnt++
     return wasm
   }
 })()

diff --git a/wasm_parts/main.wasm.gz b/wasm_parts/main.wasm.gz
diff --git a/wasm_parts/traceql/parser/lexer_rules v2.go b/wasm_parts/traceql/parser/lexer_rules v2.go
@@ -38,4 +38,4 @@ var TraceQLLexerRulesV2 = []lexer.SimpleRule{
 	{"space", `\s+`},
 }
 
-var TraceQLLexerDefinition lexer.Definition = lexer.MustSimple(TraceQLLexerRulesV2)
+var TraceQLLexerDefinition = func() lexer.Definition { return lexer.MustSimple(TraceQLLexerRulesV2) }
diff --git a/wasm_parts/traceql/parser/parser.go b/wasm_parts/traceql/parser/parser.go
@@ -6,7 +6,7 @@ import (
 
 func Parse(str string) (*TraceQLScript, error) {
 	res := &TraceQLScript{}
-	parser, err := participle.Build[TraceQLScript](participle.Lexer(TraceQLLexerDefinition), participle.UseLookahead(2))
+	parser, err := participle.Build[TraceQLScript](participle.Lexer(TraceQLLexerDefinition()), participle.UseLookahead(2))
 	if err != nil {
 		return nil, err
 	}