diff --git a/.github/workflows/node-clickhouse-cluster.js.yml b/.github/workflows/node-clickhouse-cluster.js.yml index 77db472a..c2b323fd 100644 --- a/.github/workflows/node-clickhouse-cluster.js.yml +++ b/.github/workflows/node-clickhouse-cluster.js.yml @@ -1,6 +1,6 @@ # This workflow will validate qryn using nodejs + clickhouse -name: QRYN CI +name: QRYN CI CLUSTER on: push: @@ -44,14 +44,9 @@ jobs: - name: Workflow Telemetry uses: runforesight/workflow-telemetry-action@v1.8.7 if: github.event_name != 'pull_request' - - env: - CLUSTER_NAME: test_cluster_two_shards - CLICKHOUSE_DB: qryn - CLICKHOUSE_TSDB: qryn - run: node qryn.mjs >/dev/stdout & - env: CLICKHOUSE_DB: qryn CLICKHOUSE_TSDB: qryn INTEGRATION_E2E: 1 CLOKI_EXT_URL: 127.0.0.1:3100 - run: npm run test --forceExit + run: CLUSTER_NAME=test_cluster_two_shards node qryn.mjs >/dev/stdout & npm run test --forceExit diff --git a/common.js b/common.js index 5936707d..e16812bf 100644 --- a/common.js +++ b/common.js @@ -135,3 +135,9 @@ module.exports.bun = () => { return false } } + +module.exports.logType = process.env.DISTINGUISH_LOGS_METRICS ? 1 : 0 + +module.exports.metricType = process.env.DISTINGUISH_LOGS_METRICS ? 2 : 0 + +module.exports.bothType = 0 diff --git a/lib/db/clickhouse.js b/lib/db/clickhouse.js index 9939adbf..88163cd3 100644 --- a/lib/db/clickhouse.js +++ b/lib/db/clickhouse.js @@ -131,7 +131,8 @@ const bulk = { fingerprint: r[0], timestamp_ns: r[1], value: r[2], - string: r[3] + string: r[3], + type: r[4] }, jsonSerializer)).join('\n'), id: id }) @@ -151,7 +152,8 @@ const bulkLabels = { date: r[0], fingerprint: r[1], labels: r[2], - name: r[3] + name: r[3], + type: r[4] }, jsonSerializer)).join('\n'), id: id }) diff --git a/lib/db/maintain/scripts.js b/lib/db/maintain/scripts.js index 823fb468..ca53735d 100644 --- a/lib/db/maintain/scripts.js +++ b/lib/db/maintain/scripts.js @@ -69,7 +69,47 @@ FROM samples_v3 as samples GROUP BY fingerprint, timestamp_ns;`, "INSERT INTO settings (fingerprint, type, name, value, inserted_at) VALUES (cityHash64('update_v3_2'), 'update', " + - "'v3_2', toString(toUnixTimestamp(NOW())), NOW())" + "'v3_2', toString(toUnixTimestamp(NOW())), NOW())", + + `ALTER TABLE {{DB}}.time_series {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS type UInt8, + MODIFY ORDER BY (fingerprint, type)`, + + `ALTER TABLE {{DB}}.samples_v3 {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS type UInt8`, + + `ALTER TABLE {{DB}}.time_series_gin {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS type UInt8, + MODIFY ORDER BY (key, val, fingerprint, type)`, + + `ALTER TABLE {{DB}}.metrics_15s {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS type UInt8, + MODIFY ORDER BY (fingerprint, timestamp_ns, type)`, + + 'RENAME TABLE {{DB}}.time_series_gin_view TO time_series_gin_view_bak {{{OnCluster}}}', + + `CREATE MATERIALIZED VIEW IF NOT EXISTS {{DB}}.time_series_gin_view {{{OnCluster}}} TO time_series_gin + AS SELECT date, pairs.1 as key, pairs.2 as val, fingerprint, type + FROM time_series ARRAY JOIN JSONExtractKeysAndValues(time_series.labels, 'String') as pairs`, + + 'DROP TABLE IF EXISTS {{DB}}.time_series_gin_view_bak {{{OnCluster}}}', + + 'RENAME TABLE {{DB}}.metrics_15s_mv TO metrics_15s_mv_bak {{{OnCluster}}}', + + `CREATE MATERIALIZED VIEW IF NOT EXISTS {{DB}}.metrics_15s_mv {{{OnCluster}}} TO metrics_15s AS +SELECT fingerprint, + intDiv(samples.timestamp_ns, 15000000000) * 15000000000 as timestamp_ns, + argMaxState(value, samples.timestamp_ns) as last, + maxSimpleState(value) as max, + minSimpleState(value) as min, + countState() as count, + sumSimpleState(value) as sum, + sumSimpleState(length(string)) as bytes, + type +FROM samples_v3 as samples +GROUP BY fingerprint, timestamp_ns, type;`, + + 'DROP TABLE IF EXISTS {{DB}}.metrics_15s_mv_bak {{{OnCluster}}}' ] module.exports.traces = [ @@ -195,6 +235,14 @@ module.exports.overall_dist = [ val String, fingerprint UInt64 ) ENGINE = Distributed('{{CLUSTER}}','{{DB}}', 'time_series_gin', rand());`, + + 'ALTER TABLE {{DB}}.metrics_15s_dist {{{OnCluster}}} ADD COLUMN IF NOT EXISTS `type` UInt8;', + + 'ALTER TABLE {{DB}}.samples_v3_dist {{{OnCluster}}} ADD COLUMN IF NOT EXISTS `type` UInt8', + + 'ALTER TABLE {{DB}}.time_series_dist {{{OnCluster}}} ADD COLUMN IF NOT EXISTS `type` UInt8;', + + 'ALTER TABLE {{DB}}.time_series_gin_dist {{{OnCluster}}} ADD COLUMN IF NOT EXISTS `type` UInt8;' ] module.exports.traces_dist = [ diff --git a/lib/db/throttler.js b/lib/db/throttler.js index d1da0529..c4ad3cf5 100644 --- a/lib/db/throttler.js +++ b/lib/db/throttler.js @@ -68,9 +68,9 @@ class TimeoutThrottler { } const samplesThrottler = new TimeoutThrottler( - `INSERT INTO ${clickhouseOptions.queryOptions.database}.${samplesTableName}${dist}(fingerprint, timestamp_ns, value, string) FORMAT JSONEachRow`) + `INSERT INTO ${clickhouseOptions.queryOptions.database}.${samplesTableName}${dist}(fingerprint, timestamp_ns, value, string, type) FORMAT JSONEachRow`) const timeSeriesThrottler = new TimeoutThrottler( - `INSERT INTO ${clickhouseOptions.queryOptions.database}.time_series${dist}(date, fingerprint, labels, name) FORMAT JSONEachRow`) + `INSERT INTO ${clickhouseOptions.queryOptions.database}.time_series${dist}(date, fingerprint, labels, name, type) FORMAT JSONEachRow`) const tracesThottler = new TimeoutThrottler( `INSERT INTO ${clickhouseOptions.queryOptions.database}.traces_input (trace_id, span_id, parent_id, name, timestamp_ns, duration_ns, service_name, payload_type, payload, tags) diff --git a/lib/handlers/datadog_log_push.js b/lib/handlers/datadog_log_push.js index a1cd8677..3417a419 100644 --- a/lib/handlers/datadog_log_push.js +++ b/lib/handlers/datadog_log_push.js @@ -21,7 +21,7 @@ const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache const { fingerPrint } = require('../utils') -const { readonly } = require('../../common') +const { readonly, logType } = require('../../common') const tagsToObject = (data, delimiter = ',') => Object.fromEntries(data.split(',').map(v => { @@ -79,7 +79,8 @@ async function handler (req, res) { new Date().toISOString().split('T')[0], finger, strJson, - JSONLabels.target || '' + JSONLabels.target || '', + logType ]])) for (const key in JSONLabels) { req.log.debug({ key, data: JSONLabels[key] }, 'Storing label') @@ -95,7 +96,8 @@ async function handler (req, res) { finger, BigInt((new Date().getTime() * 1000) + '000'), null, - stream.message + stream.message, + logType ] req.log.debug({ finger, values }, 'store') promises.push(bulk.add([values])) diff --git a/lib/handlers/datadog_series_push.js b/lib/handlers/datadog_series_push.js index 58cf1863..2eae1421 100644 --- a/lib/handlers/datadog_series_push.js +++ b/lib/handlers/datadog_series_push.js @@ -28,7 +28,7 @@ const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache const { fingerPrint } = require('../utils') -const { readonly } = require('../../common') +const { readonly, metricType } = require('../../common') async function handler (req, res) { req.log.debug('Datadog Series Index Request') @@ -73,7 +73,8 @@ async function handler (req, res) { new Date().toISOString().split('T')[0], finger, strJson, - JSONLabels.__name__ || 'undefined' + JSONLabels.__name__ || 'undefined', + metricType ]])) for (const key in JSONLabels) { labels.add('_LABELS_', key) @@ -98,7 +99,8 @@ async function handler (req, res) { finger, BigInt(pad('0000000000000000000', entry.timestamp)), entry.value, - JSONLabels.__name__ || 'undefined' + JSONLabels.__name__ || 'undefined', + metricType ] promises.push(bulk.add([values])) }) diff --git a/lib/handlers/elastic_bulk.js b/lib/handlers/elastic_bulk.js index f7668539..3d3bc8f8 100644 --- a/lib/handlers/elastic_bulk.js +++ b/lib/handlers/elastic_bulk.js @@ -6,7 +6,7 @@ */ -const { asyncLogError } = require('../../common') +const { asyncLogError, logType } = require('../../common') const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache @@ -80,7 +80,8 @@ async function handler (req, res) { new Date().toISOString().split('T')[0], finger, strJson, - JSONLabels.target || '' + JSONLabels.target || '', + logType ]])) for (const key in JSONLabels) { req.log.debug({ key, data: JSONLabels[key] }, 'Storing label') @@ -96,7 +97,8 @@ async function handler (req, res) { finger, BigInt((new Date().getTime() * 1000) + '000'), null, - JSON.stringify(stream) || stream + JSON.stringify(stream) || stream, + logType ] req.log.debug({ finger, values }, 'store') promises.push(bulk.add([values])) diff --git a/lib/handlers/elastic_index.js b/lib/handlers/elastic_index.js index ee314c45..047d3a3d 100644 --- a/lib/handlers/elastic_index.js +++ b/lib/handlers/elastic_index.js @@ -9,7 +9,7 @@ */ -const { asyncLogError } = require('../../common') +const { asyncLogError, logType } = require('../../common') const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache @@ -67,7 +67,8 @@ async function handler (req, res) { new Date().toISOString().split('T')[0], finger, strJson, - JSONLabels.target || '' + JSONLabels.target || '', + logType ]])) for (const key in JSONLabels) { req.log.debug({ key, data: JSONLabels[key] }, 'Storing label') @@ -88,7 +89,8 @@ async function handler (req, res) { finger, BigInt((new Date().getTime() * 1000) + '000'), null, - JSON.stringify(stream) || stream + JSON.stringify(stream) || stream, + logType ] req.log.debug({ finger, values }, 'store') promises.push(bulk.add([values])) diff --git a/lib/handlers/influx_write.js b/lib/handlers/influx_write.js index 571d2a89..48361946 100644 --- a/lib/handlers/influx_write.js +++ b/lib/handlers/influx_write.js @@ -38,7 +38,7 @@ const stringify = require('../utils').stringify const influxParser = require('../influx') -const { asyncLogError, errors } = require('../../common') +const { asyncLogError, errors, bothType, logType, metricType } = require('../../common') const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache const { fingerPrint } = require('../utils') @@ -69,6 +69,7 @@ async function handler (req, res) { let JSONLabels = {} let JSONFields = {} let finger = null + let strLabels = '' try { if (stream.tags) { JSONLabels = stream.tags @@ -80,16 +81,10 @@ async function handler (req, res) { JSONLabels.__name__ = stream.measurement || 'null' } // Calculate Fingerprint - const strLabels = stringify(Object.fromEntries(Object.entries(JSONLabels).sort())) + strLabels = stringify(Object.fromEntries(Object.entries(JSONLabels).sort())) finger = fingerPrint(strLabels) labels.add(finger.toString(), stream.labels) // Store Fingerprint - bulk_labels.add([[ - new Date().toISOString().split('T')[0], - finger, - strLabels, - stream.measurement || '' - ]]) for (const key in JSONLabels) { // req.log.debug({ key, data: JSONLabels[key] }, 'Storing label'); labels.add('_LABELS_', key) @@ -98,6 +93,7 @@ async function handler (req, res) { } catch (err) { asyncLogError(err, req.log) } + let type = bothType const timestamp = stream.timestamp || JSONFields.timestamp /* metrics */ if (stream.fields && stream.measurement !== 'syslog' && !JSONFields.message) { @@ -123,6 +119,7 @@ async function handler (req, res) { ] bulk.add([values]) } + type = metricType /* logs or syslog */ } else if (stream.measurement === 'syslog' || JSONFields.message) { // Send fields as a JSON object for qryn to parse @@ -134,7 +131,16 @@ async function handler (req, res) { JSONFields.message ] bulk.add([values]) + type = logType } + + bulk_labels.add([[ + new Date().toISOString().split('T')[0], + finger, + strLabels, + stream.measurement || '', + type + ]]) }) } await Promise.all(promises) @@ -156,13 +162,15 @@ function telegrafPrometheusV1 (stream) { new Date().toISOString().split('T')[0], fp, strLabels, - entry.measurement || '' + entry.measurement || '', + logType ]])) const values = [ fp, timestamp, 0, - entry.fields.message || '' + entry.fields.message || '', + logType ] promises.push(bulk.add([values])) } @@ -182,13 +190,15 @@ function telegrafPrometheusV1 (stream) { new Date().toISOString().split('T')[0], fp, strLabels, - entry.measurement || '' + entry.measurement || '', + metricType ]])) const values = [ fp, timestamp, iValue || 0, - key || '' + key || '', + metricType ] promises.push(bulk.add([values])) } diff --git a/lib/handlers/label.js b/lib/handlers/label.js index 375c62ff..6064263f 100644 --- a/lib/handlers/label.js +++ b/lib/handlers/label.js @@ -13,14 +13,16 @@ const clickhouse = require('../db/clickhouse') const utils = require('../utils') -const { clusterName } = require('../../common') +const { clusterName, bothType, logType } = require('../../common') const dist = clusterName ? '_dist' : '' async function handler (req, res) { req.log.debug('GET /loki/api/v1/label') + const types = req.types || [bothType, logType] let where = [ req.query.start && !isNaN(parseInt(req.query.start)) ? `date >= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.start)}, 1000000000)))` : null, - req.query.end && !isNaN(parseInt(req.query.end)) ? `date <= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.end)}, 1000000000)))` : null + req.query.end && !isNaN(parseInt(req.query.end)) ? `date <= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.end)}, 1000000000)))` : null, + `type IN (${types.map(t => `${t}`).join(',')})` ].filter(w => w) where = where.length ? `WHERE ${where.join(' AND ')}` : '' const q = `SELECT DISTINCT key FROM time_series_gin${dist} ${where} FORMAT JSON` diff --git a/lib/handlers/label_values.js b/lib/handlers/label_values.js index b6f51965..280c8512 100644 --- a/lib/handlers/label_values.js +++ b/lib/handlers/label_values.js @@ -14,15 +14,17 @@ const clickhouse = require('../db/clickhouse') const Sql = require('@cloki/clickhouse-sql') const utils = require('../utils') -const { clusterName } = require('../../common') +const { clusterName, bothType, logType } = require('../../common') const dist = clusterName ? '_dist' : '' async function handler (req, res) { req.log.debug(`GET /api/prom/label/${req.params.name}/values`) + const types = req.types || [bothType, logType] let where = [ `key = ${Sql.val(req.params.name)}`, req.query.start && !isNaN(parseInt(req.query.start)) ? `date >= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.start)}, 1000000000)))` : null, - req.query.end && !isNaN(parseInt(req.query.end)) ? `date <= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.end)}, 1000000000)))` : null + req.query.end && !isNaN(parseInt(req.query.end)) ? `date <= toDate(FROM_UNIXTIME(intDiv(${parseInt(req.query.end)}, 1000000000)))` : null, + `type IN (${types.map(t => `${t}`).join(',')})` ].filter(w => w) where = `WHERE ${where.join(' AND ')}` const q = `SELECT DISTINCT val FROM time_series_gin${dist} ${where} FORMAT JSON` diff --git a/lib/handlers/newrelic_log_push.js b/lib/handlers/newrelic_log_push.js index dda46c96..1396be24 100644 --- a/lib/handlers/newrelic_log_push.js +++ b/lib/handlers/newrelic_log_push.js @@ -34,7 +34,7 @@ const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache const { fingerPrint } = require('../utils') -const { readonly } = require('../../common') +const { readonly, logType } = require('../../common') async function handler (req, res) { req.log.debug('NewRelic Log Index Request') @@ -100,7 +100,8 @@ async function handler (req, res) { finger, ts, null, - log.message + log.message, + logType ] promises.push(bulk.add([values])) }) @@ -110,7 +111,8 @@ async function handler (req, res) { d, finger, strJson, - JSONLabels.target || '' + JSONLabels.target || '', + logType ]])) } } catch (err) { diff --git a/lib/handlers/prom_push.js b/lib/handlers/prom_push.js index 6caab169..a6de10df 100644 --- a/lib/handlers/prom_push.js +++ b/lib/handlers/prom_push.js @@ -11,7 +11,7 @@ } */ -const { asyncLogError } = require('../../common') +const { asyncLogError, metricType } = require('../../common') const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache @@ -69,7 +69,8 @@ async function handler (req, res) { finger, ts, value, - JSONLabels.__name__ || 'undefined' + JSONLabels.__name__ || 'undefined', + metricType ] dates[ new Date(parseInt((ts / BigInt('1000000')).toString())).toISOString().split('T')[0] @@ -83,7 +84,8 @@ async function handler (req, res) { d, finger, strJson, - JSONLabels.__name__ || 'undefined' + JSONLabels.__name__ || 'undefined', + metricType ]])) for (const key in JSONLabels) { labels.add('_LABELS_', key) diff --git a/lib/handlers/promlabel.js b/lib/handlers/promlabel.js index 6a296a93..6c7bf842 100644 --- a/lib/handlers/promlabel.js +++ b/lib/handlers/promlabel.js @@ -11,9 +11,12 @@ } */ +const { bothType, metricType } = require('../../common') + async function handler (req, res) { await require('./label.js')({ ...req, + types: [bothType, metricType], query: { ...req.query, start: req.query.start ? parseInt(req.query.start) * 1e9 : undefined, diff --git a/lib/handlers/promlabel_values.js b/lib/handlers/promlabel_values.js index e20622ff..14db8a48 100644 --- a/lib/handlers/promlabel_values.js +++ b/lib/handlers/promlabel_values.js @@ -11,9 +11,12 @@ } */ +const { bothType, metricType } = require('../../common') + async function handler (req, res) { await require('./label_values.js')({ ...req, + types: [bothType, metricType], query: { ...req.query, start: req.query.start ? parseInt(req.query.start) * 1e9 : undefined, diff --git a/lib/handlers/push.js b/lib/handlers/push.js index 12b27e4a..8437c9f9 100644 --- a/lib/handlers/push.js +++ b/lib/handlers/push.js @@ -22,7 +22,7 @@ const StreamValues = require('stream-json/streamers/StreamValues') const logger = require('../logger') const UTILS = require('../utils') const DATABASE = require('../db/clickhouse') -const { asyncLogError } = require('../../common') +const { asyncLogError, logType, metricType, bothType } = require('../../common') const stringify = UTILS.stringify const fingerPrint = UTILS.fingerPrint const { bulk_labels, bulk, labels } = DATABASE.cache @@ -42,6 +42,7 @@ function processStream (stream, labels, bulkLabels, bulk, toJSON, fingerPrint) { } // Calculate Fingerprint const strJson = stringify(JSONLabels) + let type = 3 finger = fingerPrint(strJson) labels.add(finger.toString(), finger.toString()) for (const key in JSONLabels) { @@ -60,12 +61,19 @@ function processStream (stream, labels, bulkLabels, bulk, toJSON, fingerPrint) { console.error('no bulkable data', entry) return } + if (!entry.value) { + type &= logType + } + if (!entry.line || entry.line === '') { + type &= metricType + } const ts = UTILS.parseStringifiedNanosOrRFC3339(entry.timestamp || entry.ts) values.push([ finger, ts, (typeof entry.value === 'undefined') ? null : entry.value, - entry.line || '' + entry.line || '', + type === 3 ? bothType : type ]) dates[new Date(Number(ts / BigInt(1000000))).toISOString().split('T')[0]] = true }) @@ -78,12 +86,21 @@ function processStream (stream, labels, bulkLabels, bulk, toJSON, fingerPrint) { console.error('no bulkable data', value) return } + + if (typeof value[2] === 'undefined') { + type &= logType + } + if (!value[1]) { + type &= metricType + } + const ts = BigInt(value[0]) values.push([ finger, BigInt(value[0]), (typeof value[2] === 'undefined') ? null : value[2], - value[1] || '' + value[1] || '', + type === 3 ? bothType : type ]) dates[new Date(Number(ts / BigInt(1000000))).toISOString().split('T')[0]] = true }) @@ -95,7 +112,8 @@ function processStream (stream, labels, bulkLabels, bulk, toJSON, fingerPrint) { date, finger, strJson, - JSONLabels.name || '' + JSONLabels.name || '', + type === 3 ? bothType : type ]])) } return Promise.all(promises) diff --git a/lib/handlers/telegraf.js b/lib/handlers/telegraf.js index a21e1b20..a564a05a 100644 --- a/lib/handlers/telegraf.js +++ b/lib/handlers/telegraf.js @@ -10,7 +10,7 @@ // CANDIDATE TO DELETE DUE TO REPLACE WITH INFLUX_WRITE -const { asyncLogError } = require('../../common') +const { asyncLogError, bothType } = require('../../common') const stringify = require('../utils').stringify const DATABASE = require('../db/clickhouse') const { bulk_labels, bulk, labels } = DATABASE.cache @@ -47,7 +47,8 @@ function handler (req, res) { new Date().toISOString().split('T')[0], finger, strLabels, - stream.name || '' + stream.name || '', + bothType ]) for (const key in JSONLabels) { // req.log.debug({ key, data: JSONLabels[key] }, 'Storing label'); @@ -72,7 +73,8 @@ function handler (req, res) { finger, BigInt(stream.timestamp + '000000000'), stream.fields[entry] || 0, - stream.fields[entry].toString() || '' + stream.fields[entry].toString() || '', + bothType ] bulk.add(values) }) diff --git a/parser/transpiler.js b/parser/transpiler.js index a257cfe2..1fea3806 100644 --- a/parser/transpiler.js +++ b/parser/transpiler.js @@ -29,12 +29,15 @@ const optimizations = require('./registry/smart_optimizations') const clusterName = require('../common').clusterName const dist = clusterName ? '_dist' : '' const wasm = require('../wasm_parts/main') +const { bothType, logType, metricType } = require('../common') /** * @param joinLabels {boolean} + * @param types {[number]} * @returns {Select} */ -module.exports.initQuery = (joinLabels) => { +module.exports.initQuery = (joinLabels, types) => { + types = types || [bothType, logType] const samplesTable = new Sql.Parameter(sharedParamNames.samplesTable) const timeSeriesTable = new Sql.Parameter(sharedParamNames.timeSeriesTable) const from = new Sql.Parameter(sharedParamNames.from) @@ -62,7 +65,9 @@ module.exports.initQuery = (joinLabels) => { ['samples.fingerprint', 'fingerprint'], [tsGetter, 'timestamp_ns']) .from([samplesTable, 'samples']) .orderBy(['timestamp_ns', 'desc']) - .where(tsClause) + .where(Sql.And( + tsClause, + new Sql.In('samples.type', 'in', types))) .limit(limit) .addParam(samplesTable) .addParam(timeSeriesTable) @@ -80,9 +85,11 @@ module.exports.initQuery = (joinLabels) => { /** * @param joinLabels {boolean} + * @param types {[number] || undefined} * @returns {Select} */ -module.exports.initQueryV3_2 = (joinLabels) => { +module.exports.initQueryV3_2 = (joinLabels, types) => { + types = types || [bothType, logType] const from = new Sql.Parameter(sharedParamNames.from) const to = new Sql.Parameter(sharedParamNames.to) const tsClause = new Sql.Raw('') @@ -95,7 +102,9 @@ module.exports.initQueryV3_2 = (joinLabels) => { const q = (new Sql.Select()) .select(['samples.fingerprint', 'fingerprint']) .from(['metrics_15s', 'samples']) - .where(tsClause) + .where(Sql.And( + tsClause, + new Sql.In('samples.type', 'in', types))) .addParam(from) .addParam(to) if (joinLabels) { @@ -156,7 +165,8 @@ module.exports.transpile = (request) => { const joinLabels = ['unwrap_function', 'log_range_aggregation', 'aggregation_operator', 'agg_statement', 'user_macro', 'parser_expression', 'label_filter_pipeline', 'line_format_expression', 'labels_format_expression', 'summary'].some(t => token.Child(t)) - let query = module.exports.initQuery(joinLabels) + let query = module.exports.initQuery(joinLabels, + token.Child('unwrap_value_statement') ? [bothType, metricType] : undefined) let limit = request.limit ? request.limit : 2000 const order = request.direction === 'forward' ? 'asc' : 'desc' query.orderBy(...query.orderBy().map(o => [o[0], order])) diff --git a/promql/index.js b/promql/index.js index cf61d741..8d5e1932 100644 --- a/promql/index.js +++ b/promql/index.js @@ -2,7 +2,7 @@ const Sql = require('@cloki/clickhouse-sql') const prometheus = require('../wasm_parts/main') const { rawRequest } = require('../lib/db/clickhouse') const { DATABASE_NAME } = require('../lib/utils') -const { clusterName } = require('../common') +const { clusterName, metricType, bothType } = require('../common') const _dist = clusterName ? '_dist' : '' class PSQLError extends Error {} @@ -54,7 +54,8 @@ module.exports.series = async (query, fromMs, toMs) => { .where(Sql.And( Sql.Gte('date', new Sql.Raw(`toDate(fromUnixTimestamp(${fromS}))`)), Sql.Lte('date', new Sql.Raw(`toDate(fromUnixTimestamp(${toS}))`)), - new Sql.In('fingerprint', 'in', new Sql.WithReference(withIdx)))) + new Sql.In('fingerprint', 'in', new Sql.WithReference(withIdx)), + new Sql.In('type', 'in', [bothType,metricType]))) .groupBy(new Sql.Raw('fingerprint')) const data = await rawRequest(req.toString() + ' FORMAT JSON', null, @@ -105,7 +106,8 @@ const getIdxSubquery = (conds, fromMs, toMs) => { .where(Sql.And( Sql.Or(...conds), Sql.Gte('date', new Sql.Raw(`toDate(fromUnixTimestamp(${fromS}))`)), - Sql.Lte('date', new Sql.Raw(`toDate(fromUnixTimestamp(${toS}))`)))) + Sql.Lte('date', new Sql.Raw(`toDate(fromUnixTimestamp(${toS}))`)), + new Sql.In('type', 'in', [bothType, metricType]))) .having( Sql.Eq( new Sql.Raw('groupBitOr(' + conds.map( @@ -124,7 +126,9 @@ module.exports.getData = async (matchers, fromMs, toMs) => { 'fingerprint', [new Sql.Raw('arraySort(JSONExtractKeysAndValues(labels, \'String\'))'), 'labels'] ).from(DATABASE_NAME() + '.time_series') - .where(new Sql.In('fingerprint', 'in', new Sql.WithReference(withIdx))) + .where(Sql.And( + new Sql.In('fingerprint', 'in', new Sql.WithReference(withIdx)), + new Sql.In('type', 'in', [bothType,metricType]))) const withTimeSeries = new Sql.With('timeSeries', timeSeries, !!clusterName) const raw = (new Sql.Select()) .with(withIdx) @@ -137,8 +141,8 @@ module.exports.getData = async (matchers, fromMs, toMs) => { new Sql.And( new Sql.In('fingerprint', 'in', new Sql.WithReference(withIdx)), Sql.Gte('timestamp_ns', new Sql.Raw(`${fromMs}000000`)), - Sql.Lte('timestamp_ns', new Sql.Raw(`${toMs}000000`)) - ) + Sql.Lte('timestamp_ns', new Sql.Raw(`${toMs}000000`)), + new Sql.In('type', 'in', [bothType, metricType])) ).groupBy('fingerprint', 'timestamp_ms') .orderBy('fingerprint', 'timestamp_ms') if (clusterName) { diff --git a/test/__snapshots__/transpiler.test.js.snap b/test/__snapshots__/transpiler.test.js.snap index 17978b19..96261a19 100644 --- a/test/__snapshots__/transpiler.test.js.snap +++ b/test/__snapshots__/transpiler.test.js.snap @@ -100,9 +100,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -423,9 +439,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -637,7 +669,7 @@ Select { } `; -exports[`log_range_aggregation 1 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; +exports[`log_range_aggregation 1 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; exports[`log_range_aggregation 2 1`] = ` Select { @@ -739,9 +771,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -1079,9 +1127,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -1306,7 +1370,7 @@ Select { } `; -exports[`log_range_aggregation 2 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; +exports[`log_range_aggregation 2 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; exports[`log_range_aggregation 3 1`] = ` Select { @@ -1408,9 +1472,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -1740,9 +1820,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -1963,7 +2059,7 @@ Select { } `; -exports[`log_range_aggregation 3 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (match(string, '^mol[eE][^ ]+e +voluptatibus') = 1) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; +exports[`log_range_aggregation 3 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (match(string, '^mol[eE][^ ]+e +voluptatibus') = 1) order by \`timestamp_ns\` desc) select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; exports[`log_range_aggregation 4 1`] = ` Select { @@ -2169,9 +2265,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -2460,9 +2572,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -2693,7 +2821,7 @@ Select { } `; -exports[`log_range_aggregation 4 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (match(string, 'cons[eE][^ ]+r nam soluta') = 0) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 1000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select \`labels\`,intDiv(timestamp_ns, 5000) * 5000 as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; +exports[`log_range_aggregation 4 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (match(string, 'cons[eE][^ ]+r nam soluta') = 0) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 1000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select \`labels\`,intDiv(timestamp_ns, 5000) * 5000 as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc"`; exports[`shoud transpile unwrap 1`] = ` Select { @@ -2843,9 +2971,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -3206,9 +3350,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -3512,9 +3672,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -3914,9 +4090,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -4305,9 +4497,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -4639,9 +4847,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -5069,9 +5293,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -5460,9 +5700,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -5794,9 +6050,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -6462,9 +6734,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -6741,9 +7029,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -6962,7 +7266,7 @@ Select { } `; -exports[`should transpile aggregation_operator 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; +exports[`should transpile aggregation_operator 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; exports[`should transpile aggregation_operator 3`] = ` Select { @@ -7354,9 +7658,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -7760,9 +8080,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -8108,7 +8444,7 @@ Select { } `; -exports[`should transpile aggregation_operator 4`] = `"WITH idx_sel AS (select \`sel_2\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_2\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; +exports[`should transpile aggregation_operator 4`] = `"WITH idx_sel AS (select \`sel_2\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_2\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; exports[`should transpile aggregation_operator 5`] = ` Select { @@ -8496,9 +8832,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -8775,9 +9127,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -8996,13 +9364,13 @@ Select { } `; -exports[`should transpile aggregation_operator 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; +exports[`should transpile aggregation_operator 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'minus_nam') and (\`val\` = 'aut illo'))) as \`sel_1\`), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 300000) * 300000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 300000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc), agg_a AS (select \`labels\`,intDiv(timestamp_ns, undefined) * undefined as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select arrayFilter(x -> x.1 IN ('label_1'), labels) as \`labels\`,\`timestamp_ns\`,sum(value) as \`value\` from agg_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`"`; exports[`should transpile complex pipelines 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '\${testID}')) and ((JSONHas(labels, 'freq') = 1) and (toFloat64OrNull(JSONExtractString(labels, 'freq')) >= '4'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 0000000 and 100000000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` asc limit 1000) select * from sel_a order by \`labels\` asc,\`timestamp_ns\` asc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '\${testID}')) and ((JSONHas(labels, 'freq') = 1) and (toFloat64OrNull(JSONExtractString(labels, 'freq')) >= '4'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 0000000 and 100000000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` asc limit 1000) select * from sel_a order by \`labels\` asc,\`timestamp_ns\` asc", "stream": Array [], } `; @@ -9013,9 +9381,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -9404,9 +9788,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -9880,7 +10280,7 @@ Select { } `; -exports[`should transpile log_stream_selector 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` = 'nemo doloremque'))) as \`sel_1\` inner any join (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'quia') and (\`val\` = 'eum voluptatem non eligendi'))) as \`sel_2\` on \`sel_1\`.\`fingerprint\` = \`sel_2\`.\`fingerprint\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` = 'nemo doloremque'))) as \`sel_1\` inner any join (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'quia') and (\`val\` = 'eum voluptatem non eligendi'))) as \`sel_2\` on \`sel_1\`.\`fingerprint\` = \`sel_2\`.\`fingerprint\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector 3`] = ` Select { @@ -9888,9 +10288,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -10208,7 +10624,7 @@ Select { } `; -exports[`should transpile log_stream_selector 4`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector 4`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector 5`] = ` Select { @@ -10216,9 +10632,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -10528,7 +10960,7 @@ Select { } `; -exports[`should transpile log_stream_selector 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector 7`] = ` Select { @@ -10536,9 +10968,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -10856,7 +11304,7 @@ Select { } `; -exports[`should transpile log_stream_selector 8`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector 8`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector with stream filter 1`] = ` Select { @@ -10864,9 +11312,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -11349,7 +11813,7 @@ Select { } `; -exports[`should transpile log_stream_selector with stream filter 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` = 'nemo doloremque'))) as \`sel_1\` inner any join (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'quia') and (\`val\` = 'eum voluptatem non eligendi'))) as \`sel_2\` on \`sel_1\`.\`fingerprint\` = \`sel_2\`.\`fingerprint\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (like(string, '%at et%') != 0) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector with stream filter 2`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` = 'nemo doloremque'))) as \`sel_1\` inner any join (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'quia') and (\`val\` = 'eum voluptatem non eligendi'))) as \`sel_2\` on \`sel_1\`.\`fingerprint\` = \`sel_2\`.\`fingerprint\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (like(string, '%at et%') != 0) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector with stream filter 3`] = ` Select { @@ -11357,9 +11821,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -11686,7 +12166,7 @@ Select { } `; -exports[`should transpile log_stream_selector with stream filter 4`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector with stream filter 4`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 1))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (notLike(string, '%consequatur nam soluta%') = 1) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector with stream filter 5`] = ` Select { @@ -11694,9 +12174,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -12015,7 +12511,7 @@ Select { } `; -exports[`should transpile log_stream_selector with stream filter 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (match(string, '^mol[eE][^ ]+e +voluptatibus') = 1) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector with stream filter 6`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'et_dolorem') and (\`val\` != 'nemo doloremque'))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (match(string, '^mol[eE][^ ]+e +voluptatibus') = 1) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile log_stream_selector with stream filter 7`] = ` Select { @@ -12023,9 +12519,25 @@ Select { "conditions": Conjunction { "args": Array [ Condition { - "column": Raw { - "raw": "", - "toString": [Function], + "column": Conjunction { + "args": Array [ + Raw { + "raw": "", + "toString": [Function], + }, + In { + "column": Term { + "term": "samples.type", + }, + "operator": "in", + "value": Value { + "value": Array [ + 0, + 0, + ], + }, + }, + ], }, "operator": undefined, "value": Value { @@ -12352,7 +12864,7 @@ Select { } `; -exports[`should transpile log_stream_selector with stream filter 8`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1 and 2) and (samples.fingerprint IN idx_sel) and (match(string, 'cons[eE][^ ]+r nam soluta') = 0) order by \`timestamp_ns\` desc limit 3"`; +exports[`should transpile log_stream_selector with stream filter 8`] = `"WITH idx_sel AS (select \`sel_1\`.\`fingerprint\` from (select \`fingerprint\` from \`loki\`.\`time_series_gin\` where ((\`key\` = 'rerum_laborum') and (match(val, '^con.+q.at[a-z]r') = 0))) as \`sel_1\`) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1 and 2) and (\`samples\`.\`type\` in (0,0))) and (samples.fingerprint IN idx_sel) and (match(string, 'cons[eE][^ ]+r nam soluta') = 0) order by \`timestamp_ns\` desc limit 3"`; exports[`should transpile logfmt requests 1`] = ` Array [ @@ -12372,7 +12884,7 @@ exports[`should transpile new style 1 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7387779420506657'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,sel_a.* from sel_a left any join \`loki\`.\`time_series\` AS time_series on \`sel_a\`.\`fingerprint\` = time_series.fingerprint order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7387779420506657'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,sel_a.* from sel_a left any join \`loki\`.\`time_series\` AS time_series on \`sel_a\`.\`fingerprint\` = time_series.fingerprint order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [], } `; @@ -12381,7 +12893,7 @@ exports[`should transpile new style 2 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.2119268970232')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) order by \`timestamp_ns\` desc limit 2000) select JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,sel_a.* from sel_a left any join \`loki\`.\`time_series\` AS time_series on \`sel_a\`.\`fingerprint\` = time_series.fingerprint order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.2119268970232')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\` from loki.samples_vX as \`samples\` where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) order by \`timestamp_ns\` desc limit 2000) select JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,sel_a.* from sel_a left any join \`loki\`.\`time_series\` AS time_series on \`sel_a\`.\`fingerprint\` = time_series.fingerprint order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [], } `; @@ -12390,7 +12902,7 @@ exports[`should transpile new style 3 1`] = ` Object { "duration": 1000, "matrix": true, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7026038163617259')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 1000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select \`labels\`,intDiv(timestamp_ns, 2000) * 2000 as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7026038163617259')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))), rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) order by \`timestamp_ns\` desc), rate_b AS (select labels as \`labels\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\`,toFloat64(count(1)) * 1000 / 1000 as \`value\` from rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc) select \`labels\`,intDiv(timestamp_ns, 2000) * 2000 as \`timestamp_ns\`,argMin(rate_b.value, rate_b.timestamp_ns) as \`value\` from \`rate_b\` group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", "stream": Array [], } `; @@ -12399,7 +12911,7 @@ exports[`should transpile new style 4 1`] = ` Object { "duration": 1000, "matrix": true, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7026038163617259')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))) select \`labels\`,toUInt64(intDiv(timestamp_ns, 1000000000) * 1000) as \`timestamp_ns\`,toFloat64(0) as \`value\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7026038163617259')) and ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '2'))) select \`labels\`,toUInt64(intDiv(timestamp_ns, 1000000000) * 1000) as \`timestamp_ns\`,toFloat64(0) as \`value\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (match(string, '2[0-9]$') = 1) group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", "stream": Array [ [Function], ], @@ -12410,7 +12922,7 @@ exports[`should transpile new style 5 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.000341166036469831_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.000341166036469831_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [ [Function], ], @@ -12421,7 +12933,7 @@ exports[`should transpile new style 6 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.2053747382122484_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter((x) -> x.2 != '', [('lbl_repl', if(JSONType(samples.string, 'new_lbl') == 'String', JSONExtractString(samples.string, 'new_lbl'), JSONExtractRaw(samples.string, 'new_lbl')))]) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (isValidJSON(samples.string) = 1) and ((indexOf(extra_labels, ('lbl_repl', 'new_val')) > 0) or ((arrayExists(x -> x.1 == 'lbl_repl', extra_labels) = 0) and ((arrayExists(x -> x.1 == 'lbl_repl', labels) = 1) and (arrayFirst(x -> x.1 == 'lbl_repl', labels).2 = 'new_val')))) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.2053747382122484_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter((x) -> x.2 != '', [('lbl_repl', if(JSONType(samples.string, 'new_lbl') == 'String', JSONExtractString(samples.string, 'new_lbl'), JSONExtractRaw(samples.string, 'new_lbl')))]) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (isValidJSON(samples.string) = 1) and ((indexOf(extra_labels, ('lbl_repl', 'new_val')) > 0) or ((arrayExists(x -> x.1 == 'lbl_repl', extra_labels) = 0) and ((arrayExists(x -> x.1 == 'lbl_repl', labels) = 1) and (arrayFirst(x -> x.1 == 'lbl_repl', labels).2 = 'new_val')))) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [], } `; @@ -12430,7 +12942,7 @@ exports[`should transpile new style 7 1`] = ` Object { "duration": 3000, "matrix": true, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.1547558751138609_json'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.1547558751138609_json'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc", "stream": Array [ [Function], [Function], @@ -12446,7 +12958,7 @@ exports[`should transpile new style 8 1`] = ` Object { "duration": 1000, "matrix": true, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.4075242197275857'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.4075242197275857'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc", "stream": Array [ [Function], [Function], @@ -12462,7 +12974,7 @@ exports[`should transpile new style 9 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7186063017626447_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter((x) -> x.2 != '', [('sid', if(JSONType(samples.string, 'str_id') == 'String', JSONExtractString(samples.string, 'str_id'), JSONExtractRaw(samples.string, 'str_id')))]) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (isValidJSON(samples.string) = 1) and ((arrayExists(x -> x.1 == 'sid' AND (coalesce(toFloat64OrNull(x.2) >= '598', 0)), extra_labels) != 0) or ((arrayExists(x -> x.1 == 'sid', extra_labels) = 0) and (arrayExists(x -> x.1 == 'sid', labels) = 1) and (toFloat64OrNull(arrayFirst(x -> x.1 == 'sid', labels).2) >= '598'))) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.7186063017626447_json'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter((x) -> x.2 != '', [('sid', if(JSONType(samples.string, 'str_id') == 'String', JSONExtractString(samples.string, 'str_id'), JSONExtractRaw(samples.string, 'str_id')))]) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and (isValidJSON(samples.string) = 1) and ((arrayExists(x -> x.1 == 'sid' AND (coalesce(toFloat64OrNull(x.2) >= '598', 0)), extra_labels) != 0) or ((arrayExists(x -> x.1 == 'sid', extra_labels) = 0) and (arrayExists(x -> x.1 == 'sid', labels) = 1) and (toFloat64OrNull(arrayFirst(x -> x.1 == 'sid', labels).2) >= '598'))) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [], } `; @@ -12471,7 +12983,7 @@ exports[`should transpile new style 10 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.5505504081219323'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(['e'], arrayMap(x -> x[length(x)], extractAllGroupsHorizontal(string, '^([^0-9]+)[0-9]+$')))) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (JSONExtractString(labels, 'test_id') = '0.5505504081219323'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(['e'], arrayMap(x -> x[length(x)], extractAllGroupsHorizontal(string, '^([^0-9]+)[0-9]+$')))) as \`extra_labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [], } `; @@ -12480,7 +12992,7 @@ exports[`should transpile new style 11 1`] = ` Object { "duration": 1000, "matrix": true, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'label') = 1) and (JSONExtractString(labels, 'label') = 'val'))), uw_rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,toFloat64OrNull(arrayFirst(x -> x.1 == 'b', labels).2) as \`unwrapped\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and ((arrayExists(x -> x.1 == 'b', labels) = 1) and (isNotNull(unwrapped) = 1)) order by \`timestamp_ns\` desc), uw_rate_b AS (select labels as \`labels\`,SUM(unwrapped) / 1 as \`value\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\` from uw_rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`) select \`labels\`,intDiv(timestamp_ns, 2000) * 2000 as \`timestamp_ns\`,argMin(uw_rate_b.value, uw_rate_b.timestamp_ns) as \`value\` from uw_rate_b group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'label') = 1) and (JSONExtractString(labels, 'label') = 'val'))), uw_rate_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,intDiv(samples.timestamp_ns, 1000000) as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\`,toFloat64OrNull(arrayFirst(x -> x.1 == 'b', labels).2) as \`unwrapped\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) and ((arrayExists(x -> x.1 == 'b', labels) = 1) and (isNotNull(unwrapped) = 1)) order by \`timestamp_ns\` desc), uw_rate_b AS (select labels as \`labels\`,SUM(unwrapped) / 1 as \`value\`,intDiv(timestamp_ns, 1000) * 1000 as \`timestamp_ns\` from uw_rate_a group by \`labels\`,\`timestamp_ns\` order by \`labels\`,\`timestamp_ns\`) select \`labels\`,intDiv(timestamp_ns, 2000) * 2000 as \`timestamp_ns\`,argMin(uw_rate_b.value, uw_rate_b.timestamp_ns) as \`value\` from uw_rate_b group by \`labels\`,\`timestamp_ns\` order by \`labels\` asc,\`timestamp_ns\` asc", "stream": Array [], } `; @@ -12489,7 +13001,7 @@ exports[`should transpile new style 12 1`] = ` Object { "duration": 1000, "matrix": false, - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '1'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'freq') = 1) and (JSONExtractString(labels, 'freq') = '1'))), sel_a AS (select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_vX as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` between 1638802620000000000 and 1638803220000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` desc limit 2000) select * from sel_a order by \`labels\` desc,\`timestamp_ns\` desc", "stream": Array [ [Function], [Function], @@ -12516,7 +13028,7 @@ exports[`should transpile series 1`] = `"WITH idx_sel AS ((select \`sel_1\`.\`fi exports[`should transpile tail 1`] = ` Object { - "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (extractAllGroups(JSONExtractString(labels, 'test_id'), '(_ws)') != '[]'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_v3 as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where (\`samples\`.\`timestamp_ns\` > (toUnixTimestamp(now()) - 5) * 1000000000) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` asc", + "query": "WITH str_sel AS (select DISTINCT \`fingerprint\` from loki.time_series where ((JSONHas(labels, 'test_id') = 1) and (extractAllGroups(JSONExtractString(labels, 'test_id'), '(_ws)') != '[]'))) select \`samples\`.\`string\` as \`string\`,\`samples\`.\`fingerprint\` as \`fingerprint\`,samples.timestamp_ns as \`timestamp_ns\`,JSONExtractKeysAndValues(time_series.labels, 'String') as \`labels\` from loki.samples_v3 as \`samples\` left any join \`loki\`.\`time_series\` AS time_series on \`samples\`.\`fingerprint\` = time_series.fingerprint where ((\`samples\`.\`timestamp_ns\` > (toUnixTimestamp(now()) - 5) * 1000000000) and (\`samples\`.\`type\` in (0,0))) and (\`samples\`.\`fingerprint\` in (select \`fingerprint\` from str_sel)) order by \`timestamp_ns\` asc", "stream": Array [], } `; diff --git a/test/e2e b/test/e2e index fc6ab17e..81b163a5 160000 --- a/test/e2e +++ b/test/e2e @@ -1 +1 @@ -Subproject commit fc6ab17eb3946cb84c7a5d2114c51c67e272e6be +Subproject commit 81b163a554f8e92b7312845211eec9fba8931f0a