From 50f790f660dcb9d91e7c224c411dd8cbbe7055b2 Mon Sep 17 00:00:00 2001 From: Serge Klochkov <3175289+slvrtrn@users.noreply.github.com> Date: Fri, 1 Sep 2023 19:16:55 +0200 Subject: [PATCH] Add default_format setting (#200) --- .docker/clickhouse/single_node_tls/Dockerfile | 2 +- CHANGELOG.md | 6 ++++++ docker-compose.cluster.yml | 4 ++-- docker-compose.yml | 2 +- examples/default_format_setting.ts | 17 +++++++++++++++++ karma.config.cjs | 6 +++++- .../__tests__/integration/error_parsing.test.ts | 6 +++++- .../__tests__/integration/insert.test.ts | 5 ++++- .../__tests__/integration/select.test.ts | 5 ++++- .../integration/select_query_binding.test.ts | 7 +++++-- .../client-common/__tests__/utils/client.ts | 3 ++- packages/client-common/src/settings.ts | 2 ++ packages/client-common/src/version.ts | 2 +- .../__tests__/integration/node_exec.test.ts | 13 +++++++++++++ packages/client-node/src/version.ts | 2 +- .../__tests__/integration/web_exec.test.ts | 13 +++++++++++++ packages/client-web/src/version.ts | 2 +- 17 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 examples/default_format_setting.ts diff --git a/.docker/clickhouse/single_node_tls/Dockerfile b/.docker/clickhouse/single_node_tls/Dockerfile index a9cc9132..12641cd9 100644 --- a/.docker/clickhouse/single_node_tls/Dockerfile +++ b/.docker/clickhouse/single_node_tls/Dockerfile @@ -1,4 +1,4 @@ -FROM clickhouse/clickhouse-server:23.5-alpine +FROM clickhouse/clickhouse-server:23.8-alpine COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ && chmod 600 /etc/clickhouse-server/certs/* \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d86e73..48b2f918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.2.2 (Common, Node.js & Web) + +### New features + +- Added `default_format` setting, which allows to perform `exec` calls without `FORMAT` clause. + ## 0.2.1 (Common, Node.js & Web) ### Breaking changes diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml index dfac0458..0b6555aa 100644 --- a/docker-compose.cluster.yml +++ b/docker-compose.cluster.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse1: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.5-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.8-alpine}' ulimits: nofile: soft: 262144 @@ -19,7 +19,7 @@ services: - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' clickhouse2: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.5-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.8-alpine}' ulimits: nofile: soft: 262144 diff --git a/docker-compose.yml b/docker-compose.yml index d3a0fb7c..d70e6841 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: clickhouse: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.5-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.8-alpine}' container_name: 'clickhouse-js-clickhouse-server' ports: - '8123:8123' diff --git a/examples/default_format_setting.ts b/examples/default_format_setting.ts new file mode 100644 index 00000000..d8506767 --- /dev/null +++ b/examples/default_format_setting.ts @@ -0,0 +1,17 @@ +import { createClient, ResultSet } from '@clickhouse/client' // or '@clickhouse/client-web' + +void (async () => { + const client = createClient() + const format = 'JSONCompactEachRowWithNamesAndTypes' + const { stream, query_id } = await client.exec({ + // this query fails without `default_format` setting + // as it does not have the FORMAT clause + query: `SELECT database, name, engine FROM system.tables LIMIT 5`, + clickhouse_settings: { + default_format: format, + }, + }) + const rs = new ResultSet(stream, format, query_id) + console.log(await rs.json()) + await client.close() +})() diff --git a/karma.config.cjs b/karma.config.cjs index a3255758..18021641 100644 --- a/karma.config.cjs +++ b/karma.config.cjs @@ -1,5 +1,7 @@ const webpackConfig = require('./webpack.dev.js') +const TEST_TIMEOUT_MS = 120_000 + module.exports = function (config) { config.set({ // base path that will be used to resolve all patterns (eg. files, exclude) @@ -43,6 +45,8 @@ module.exports = function (config) { autoWatch: false, // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher browsers: ['ChromeHeadless', 'FirefoxHeadless'], + browserNoActivityTimeout: TEST_TIMEOUT_MS, + browserDisconnectTimeout: TEST_TIMEOUT_MS, // if true, Karma captures browsers, runs the tests and exits singleRun: true, client: { @@ -50,7 +54,7 @@ module.exports = function (config) { random: false, stopOnSpecFailure: false, stopSpecOnExpectationFailure: true, - timeoutInterval: 60000, + timeoutInterval: TEST_TIMEOUT_MS, }, }, }) diff --git a/packages/client-common/__tests__/integration/error_parsing.test.ts b/packages/client-common/__tests__/integration/error_parsing.test.ts index 785d1c2c..612bfa5a 100644 --- a/packages/client-common/__tests__/integration/error_parsing.test.ts +++ b/packages/client-common/__tests__/integration/error_parsing.test.ts @@ -25,13 +25,17 @@ describe('ClickHouse server errors parsing', () => { }) it('returns "unknown table" error', async () => { + // possible error messages here: + // (since 23.8+) Table foo.unknown_table does not exist. + // (pre-23.8) Table foo.unknown_table doesn't exist. + const errorMessagePattern = `^Table ${getTestDatabaseName()}.unknown_table does(n't| not) exist.*$` await expectAsync( client.query({ query: 'SELECT * FROM unknown_table', }) ).toBeRejectedWith( jasmine.objectContaining({ - message: `Table ${getTestDatabaseName()}.unknown_table doesn't exist. `, + message: jasmine.stringMatching(errorMessagePattern), code: '60', type: 'UNKNOWN_TABLE', }) diff --git a/packages/client-common/__tests__/integration/insert.test.ts b/packages/client-common/__tests__/integration/insert.test.ts index 1d5f1571..05ad1226 100644 --- a/packages/client-common/__tests__/integration/insert.test.ts +++ b/packages/client-common/__tests__/integration/insert.test.ts @@ -128,7 +128,10 @@ describe('insert', () => { }) ).toBeRejectedWith( jasmine.objectContaining({ - message: jasmine.stringContaining('Unknown setting foobar'), + // Possible error messages: + // Unknown setting foobar + // Setting foobar is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. + message: jasmine.stringContaining('foobar'), code: '115', type: 'UNKNOWN_SETTING', }) diff --git a/packages/client-common/__tests__/integration/select.test.ts b/packages/client-common/__tests__/integration/select.test.ts index 41b03fd8..b09d44e8 100644 --- a/packages/client-common/__tests__/integration/select.test.ts +++ b/packages/client-common/__tests__/integration/select.test.ts @@ -153,7 +153,10 @@ describe('select', () => { }) ).toBeRejectedWith( jasmine.objectContaining({ - message: jasmine.stringContaining('Unknown setting foobar'), + // Possible error messages: + // Unknown setting foobar + // Setting foobar is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. + message: jasmine.stringContaining('foobar'), code: '115', type: 'UNKNOWN_SETTING', }) diff --git a/packages/client-common/__tests__/integration/select_query_binding.test.ts b/packages/client-common/__tests__/integration/select_query_binding.test.ts index c5ad55a0..f11ed5f7 100644 --- a/packages/client-common/__tests__/integration/select_query_binding.test.ts +++ b/packages/client-common/__tests__/integration/select_query_binding.test.ts @@ -260,8 +260,11 @@ describe('select with query binding', () => { }) ).toBeRejectedWith( jasmine.objectContaining({ - message: jasmine.stringContaining( - 'Query parameter `min_limit` was not set' + message: jasmine.stringMatching( + // possible error messages here: + // (since 23.8+) Substitution `min_limit` is not set. + // (pre-23.8) Query parameter `min_limit` was not set + /^.+?`min_limit`.+?not set.*$/ ), code: '456', type: 'UNKNOWN_QUERY_PARAMETER', diff --git a/packages/client-common/__tests__/utils/client.ts b/packages/client-common/__tests__/utils/client.ts index a1187418..22b66fa7 100644 --- a/packages/client-common/__tests__/utils/client.ts +++ b/packages/client-common/__tests__/utils/client.ts @@ -9,9 +9,10 @@ import { guid } from './guid' import { getClickHouseTestEnvironment, TestEnv } from './test_env' import { TestLogger } from './test_logger' +jasmine.DEFAULT_TIMEOUT_INTERVAL = 120_000 + let databaseName: string beforeAll(async () => { - jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000 if ( getClickHouseTestEnvironment() === TestEnv.Cloud && databaseName === undefined diff --git a/packages/client-common/src/settings.ts b/packages/client-common/src/settings.ts index 8f10e972..f2f7f559 100644 --- a/packages/client-common/src/settings.ts +++ b/packages/client-common/src/settings.ts @@ -1,3 +1,4 @@ +import type { DataFormat } from './data_formatter' /** * @see {@link https://github.com/ClickHouse/ClickHouse/blob/46ed4f6cdf68fbbdc59fbe0f0bfa9a361cc0dec1/src/Core/Settings.h} * @see {@link https://github.com/ClickHouse/ClickHouse/blob/eae2667a1c29565c801be0ffd465f8bfcffe77ef/src/Storages/MergeTree/MergeTreeSettings.h} @@ -1579,6 +1580,7 @@ interface ClickHouseServerSettings { interface ClickHouseHTTPSettings { wait_end_of_query: Bool + default_format: DataFormat } export type ClickHouseSettings = Partial & diff --git a/packages/client-common/src/version.ts b/packages/client-common/src/version.ts index c6bd90a2..95199fa6 100644 --- a/packages/client-common/src/version.ts +++ b/packages/client-common/src/version.ts @@ -1 +1 @@ -export default '0.2.1' +export default '0.2.2' diff --git a/packages/client-node/__tests__/integration/node_exec.test.ts b/packages/client-node/__tests__/integration/node_exec.test.ts index d91a371f..76ea9030 100644 --- a/packages/client-node/__tests__/integration/node_exec.test.ts +++ b/packages/client-node/__tests__/integration/node_exec.test.ts @@ -2,6 +2,7 @@ import type { ClickHouseClient } from '@clickhouse/client-common' import { createTestClient } from '@test/utils' import type Stream from 'stream' import { getAsText } from '../../src/utils' +import { ResultSet } from '../../src' describe('[Node.js] exec result streaming', () => { let client: ClickHouseClient @@ -44,5 +45,17 @@ describe('[Node.js] exec result streaming', () => { }) expect(await getAsText(result.stream)).toEqual('0\n') }) + + it('should work with default_format', async () => { + const format = 'JSONEachRow' + const { stream, query_id } = await client.exec({ + query: 'SELECT number FROM system.numbers LIMIT 1', + clickhouse_settings: { + default_format: format, + }, + }) + const rs = new ResultSet(stream, format, query_id) + expect(await rs.json()).toEqual([{ number: '0' }]) + }) }) }) diff --git a/packages/client-node/src/version.ts b/packages/client-node/src/version.ts index c6bd90a2..95199fa6 100644 --- a/packages/client-node/src/version.ts +++ b/packages/client-node/src/version.ts @@ -1 +1 @@ -export default '0.2.1' +export default '0.2.2' diff --git a/packages/client-web/__tests__/integration/web_exec.test.ts b/packages/client-web/__tests__/integration/web_exec.test.ts index 318c3872..7a3bbfa6 100644 --- a/packages/client-web/__tests__/integration/web_exec.test.ts +++ b/packages/client-web/__tests__/integration/web_exec.test.ts @@ -1,6 +1,7 @@ import type { ClickHouseClient } from '@clickhouse/client-common' import { createTestClient } from '@test/utils' import { getAsText } from '../../src/utils' +import { ResultSet } from '../../src' describe('[Web] exec result streaming', () => { let client: ClickHouseClient @@ -43,5 +44,17 @@ describe('[Web] exec result streaming', () => { }) expect(await getAsText(result.stream)).toEqual('0\n') }) + + it('should work with default_format', async () => { + const format = 'JSONEachRow' + const { stream, query_id } = await client.exec({ + query: 'SELECT number FROM system.numbers LIMIT 1', + clickhouse_settings: { + default_format: format, + }, + }) + const rs = new ResultSet(stream, format, query_id) + expect(await rs.json()).toEqual([{ number: '0' }]) + }) }) }) diff --git a/packages/client-web/src/version.ts b/packages/client-web/src/version.ts index c6bd90a2..95199fa6 100644 --- a/packages/client-web/src/version.ts +++ b/packages/client-web/src/version.ts @@ -1 +1 @@ -export default '0.2.1' +export default '0.2.2'