From 70dd00da43695e589755f1f9deb65b8a9ff6c18b Mon Sep 17 00:00:00 2001 From: akvlad Date: Tue, 17 Oct 2023 13:31:47 +0300 Subject: [PATCH 1/8] #fix: distributed table usage --- .../smart_optimizations/optimization_v3_2.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/parser/registry/smart_optimizations/optimization_v3_2.js b/parser/registry/smart_optimizations/optimization_v3_2.js index e44670e2..ede56134 100644 --- a/parser/registry/smart_optimizations/optimization_v3_2.js +++ b/parser/registry/smart_optimizations/optimization_v3_2.js @@ -1,9 +1,12 @@ -const { getDuration, dist, Aliased } = require('../common') +const { getDuration, Aliased } = require('../common') const reg = require('./log_range_agg_reg_v3_2') const Sql = require('@cloki/clickhouse-sql') const { DATABASE_NAME, checkVersion } = require('../../../lib/utils') const streamSelectorReg = require('../stream_selector_operator_registry') const aggOpReg = require('../high_level_aggregation_registry') +const { clusterName } = require('../../../common') +const logger = require('../../../lib/logger') +const _dist = clusterName ? '_dist' : '' /** * @@ -46,14 +49,15 @@ module.exports.apply = (token, fromNS, toNS, stepNS) => { : Sql.Gt('samples.timestamp_ns', fromNS) let q = (new Sql.Select()) .select(['samples.fingerprint', 'fingerprint']) - .from([`${DATABASE_NAME()}.metrics_15s`, 'samples']) + .from([`${DATABASE_NAME()}.metrics_15s${_dist}`, 'samples']) .where(tsClause) - q.join(new Aliased(`${DATABASE_NAME()}.time_series${dist}`, 'time_series'), 'left any', + q.join(new Aliased(`${DATABASE_NAME()}.time_series`, 'time_series'), 'left any', Sql.Eq('samples.fingerprint', new Sql.Raw('time_series.fingerprint'))) q.select([new Sql.Raw('any(JSONExtractKeysAndValues(time_series.labels, \'String\'))'), 'labels']) q.ctx = { - step: stepNS / 1000000000 + step: stepNS / 1000000000, + inline: !!clusterName } for (const streamSelectorRule of token.Children('log_stream_selector_rule')) { @@ -68,5 +72,7 @@ module.exports.apply = (token, fromNS, toNS, stepNS) => { q = aggOpReg[aggOp.Child('aggregation_operator_fn').value](aggOp, q) } + logger.debug(q.toString()) + return q } From eca14c93016859bc94c4ae14135e66dc2594e13b Mon Sep 17 00:00:00 2001 From: Automated Version Bump Date: Tue, 17 Oct 2023 10:37:18 +0000 Subject: [PATCH 2/8] ci: version bump to v2.4.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1011fe64..b3a8b290 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "qryn", - "version": "2.4.2", + "version": "2.4.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "qryn", - "version": "2.4.2", + "version": "2.4.3", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/package.json b/package.json index 3bdfb8ac..8ac48b90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "qryn", - "version": "2.4.2", + "version": "2.4.3", "description": "Polyglot Observability Stack with ClickHouse storage", "main": "qryn.js", "bin": { From e581bb0ee97e4e9a32c5b5b4d6eb4a434009f831 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 17 Oct 2023 20:51:21 +0200 Subject: [PATCH 3/8] Update README.md Signed-off-by: Lorenzo Mangani --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 88663a82..c9f323e1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - **Logs, Metrics and Traces** living happily together. Drop-in compatible with multiple vendors formats. - Native [LogQL/PromQL/TempoQL APIs](https://qryn.cloud) support for [querying](https://github.com/lmangani/qryn/wiki/LogQL-for-Beginners), [processing](https://github.com/lmangani/qryn/wiki/LogQL-Supported-Queries), [tracing](https://github.com/lmangani/qryn/wiki/Tempo-Tracing) and [alerting](https://github.com/lmangani/qryn/wiki/Ruler---Alerts) [^2] in [Grafana](http://docs.grafana.org/features/explore/) [^3] - Ingestion [APIs](https://qryn.metrico.in/#/support) transparently compatible with [Opentelemetry, Loki, Prometheus, InfluxDB, Elastic](https://qryn.dev) _and [more](https://github.com/metrico/otel-collector)_ -- Dynamically search, filter and extract metrics from _logs, events, spans and traces_. _NO SQL required_. +- Search, filter and extract metrics from _logs, events, spans and traces_ using familiar languages. _NO SQL required_. - Ready to use with popular Agents such as [Promtail, Grafana-Agent, Vector, Logstash, Telegraf](https://qryn.metrico.in/#/ingestion) _and more_ - Built in [Explore UI](https://github.com/metrico/cloki-view) and [CLI](https://github.com/lmangani/vLogQL) for querying supported datasources - Designed for edge _(js/wasm)_ and core/backend deployments _(go/rust)_. From ede52a51a0a747919e63c69206f757d8cd8c0e24 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 17 Oct 2023 20:52:08 +0200 Subject: [PATCH 4/8] Update README.md Signed-off-by: Lorenzo Mangani --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9f323e1..0335e87d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - **Logs, Metrics and Traces** living happily together. Drop-in compatible with multiple vendors formats. - Native [LogQL/PromQL/TempoQL APIs](https://qryn.cloud) support for [querying](https://github.com/lmangani/qryn/wiki/LogQL-for-Beginners), [processing](https://github.com/lmangani/qryn/wiki/LogQL-Supported-Queries), [tracing](https://github.com/lmangani/qryn/wiki/Tempo-Tracing) and [alerting](https://github.com/lmangani/qryn/wiki/Ruler---Alerts) [^2] in [Grafana](http://docs.grafana.org/features/explore/) [^3] - Ingestion [APIs](https://qryn.metrico.in/#/support) transparently compatible with [Opentelemetry, Loki, Prometheus, InfluxDB, Elastic](https://qryn.dev) _and [more](https://github.com/metrico/otel-collector)_ -- Search, filter and extract metrics from _logs, events, spans and traces_ using familiar languages. _NO SQL required_. +- Search, filter and extract metrics from _logs, events, spans and traces_ using familiar languages. _SQL Optional_. - Ready to use with popular Agents such as [Promtail, Grafana-Agent, Vector, Logstash, Telegraf](https://qryn.metrico.in/#/ingestion) _and more_ - Built in [Explore UI](https://github.com/metrico/cloki-view) and [CLI](https://github.com/lmangani/vLogQL) for querying supported datasources - Designed for edge _(js/wasm)_ and core/backend deployments _(go/rust)_. From 0e898f58fddd33e0ac6097dfc48616c0eb78843f Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 17 Oct 2023 20:52:54 +0200 Subject: [PATCH 5/8] Update README.md Signed-off-by: Lorenzo Mangani --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0335e87d..c555f51d 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ :rocket: **qryn** is a _drop-in Grafana compatible_ **polyglot observability** framework
- **Logs, Metrics and Traces** living happily together. Drop-in compatible with multiple vendors formats. - Native [LogQL/PromQL/TempoQL APIs](https://qryn.cloud) support for [querying](https://github.com/lmangani/qryn/wiki/LogQL-for-Beginners), [processing](https://github.com/lmangani/qryn/wiki/LogQL-Supported-Queries), [tracing](https://github.com/lmangani/qryn/wiki/Tempo-Tracing) and [alerting](https://github.com/lmangani/qryn/wiki/Ruler---Alerts) [^2] in [Grafana](http://docs.grafana.org/features/explore/) [^3] -- Ingestion [APIs](https://qryn.metrico.in/#/support) transparently compatible with [Opentelemetry, Loki, Prometheus, InfluxDB, Elastic](https://qryn.dev) _and [more](https://github.com/metrico/otel-collector)_ - Search, filter and extract metrics from _logs, events, spans and traces_ using familiar languages. _SQL Optional_. +- Ingestion [APIs](https://qryn.metrico.in/#/support) transparently compatible with [Opentelemetry, Loki, Prometheus, InfluxDB, Elastic](https://qryn.dev) _and [more](https://github.com/metrico/otel-collector)_ - Ready to use with popular Agents such as [Promtail, Grafana-Agent, Vector, Logstash, Telegraf](https://qryn.metrico.in/#/ingestion) _and more_ - Built in [Explore UI](https://github.com/metrico/cloki-view) and [CLI](https://github.com/lmangani/vLogQL) for querying supported datasources -- Designed for edge _(js/wasm)_ and core/backend deployments _(go/rust)_. +- Designed for edge _(js/bun/wasm)_ and core/backend deployments _(golang/rust)_. - Total data control. Compatible with [ClickHouse](https://clickhouse.com/) or [InfluxDB IOx](https://influxdata.com) with S3 object storage. :rocket: **qryn.cloud** is the _supercharged_ **qryn** version developed in _go_ with additional _functionality, speed and features!_
From c90d5c5860ad63fa886d7d8748caff7acbed9552 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:10:29 +0200 Subject: [PATCH 6/8] build(deps-dev): bump undici from 5.20.0 to 5.26.3 (#354) Bumps [undici](https://github.com/nodejs/undici) from 5.20.0 to 5.26.3. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v5.20.0...v5.26.3) --- updated-dependencies: - dependency-name: undici dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lorenzo Mangani --- package-lock.json | 69 +++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index b3a8b290..f846c24e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -823,6 +823,15 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, + "node_modules/@fastify/busboy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", + "integrity": "sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==", + "dev": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@fastify/compress": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/@fastify/compress/-/compress-6.2.0.tgz", @@ -3840,18 +3849,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dev": true, - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", @@ -12352,15 +12349,6 @@ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "dev": true, - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -13053,15 +13041,15 @@ } }, "node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", + "version": "5.26.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", + "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", "dev": true, "dependencies": { - "busboy": "^1.6.0" + "@fastify/busboy": "^2.0.0" }, "engines": { - "node": ">=12.18" + "node": ">=14.0" } }, "node_modules/union-value": { @@ -14103,6 +14091,12 @@ } } }, + "@fastify/busboy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", + "integrity": "sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==", + "dev": true + }, "@fastify/compress": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/@fastify/compress/-/compress-6.2.0.tgz", @@ -16411,15 +16405,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, - "busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dev": true, - "requires": { - "streamsearch": "^1.1.0" - } - }, "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", @@ -22968,12 +22953,6 @@ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" }, - "streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "dev": true - }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -23504,12 +23483,12 @@ "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=" }, "undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", + "version": "5.26.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", + "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", "dev": true, "requires": { - "busboy": "^1.6.0" + "@fastify/busboy": "^2.0.0" } }, "union-value": { From 6573c7d427b7f525f0e5ed98d3dae8194b94b641 Mon Sep 17 00:00:00 2001 From: Automated Version Bump Date: Mon, 23 Oct 2023 11:10:52 +0000 Subject: [PATCH 7/8] ci: version bump to v2.4.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f846c24e..68960f27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "qryn", - "version": "2.4.3", + "version": "2.4.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "qryn", - "version": "2.4.3", + "version": "2.4.4", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/package.json b/package.json index 8ac48b90..36c565f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "qryn", - "version": "2.4.3", + "version": "2.4.4", "description": "Polyglot Observability Stack with ClickHouse storage", "main": "qryn.js", "bin": { From 0fa66cd34eb0591a395037ded6cfba6588490864 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 24 Oct 2023 22:07:53 +0200 Subject: [PATCH 8/8] Update bump_version.yml Signed-off-by: Lorenzo Mangani --- .github/workflows/bump_version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bump_version.yml b/.github/workflows/bump_version.yml index fa367728..7fa0222e 100644 --- a/.github/workflows/bump_version.yml +++ b/.github/workflows/bump_version.yml @@ -1,4 +1,4 @@ -name: 'Bump & Publish Multi-Arch' +name: 'Build Multi-Arch' on: push: