diff --git a/examples/README.md b/examples/README.md index 47784144..129408b9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -41,14 +41,15 @@ We aim to cover various scenarios of client usage with these examples. - [insert_values_and_functions.ts](insert_values_and_functions.ts) - generating an `INSERT INTO ... VALUES` statement that uses a combination of values and function calls. - [insert_ephemeral_columns.ts](insert_ephemeral_columns.ts) - inserting data into a table that has [ephemeral columns](https://clickhouse.com/docs/en/sql-reference/statements/create/table#ephemeral). -#### Selecting the data +#### Selecting data - [select_json_each_row.ts](select_json_each_row.ts) - simple select of the data in the `JSONEachRow` format. - [select_json_with_metadata.ts](select_json_with_metadata.ts) - select result as a JSON object with query metadata. - [query_with_parameter_binding.ts](query_with_parameter_binding.ts) - query parameter binding example. - [select_parquet_as_file.ts](node/select_parquet_as_file.ts) - (Node.js only) select data from ClickHouse and save it as a Parquet file. This example can be adjusted to save the data in other formats, such as CSV/TSV/TabSeparated, by changing the format in the query. -- [select_streaming_for_await.ts](node/select_streaming_for_await.ts) - (Node.js only) streaming the data from ClickHouse and processing it with `for await` loop. -- [select_streaming_on_data.ts](node/select_streaming_on_data.ts) - (Node.js only) streaming the data from ClickHouse and processing it with `on('data')` event. +- [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts) - (Node.js only) streaming JSON\* formats from ClickHouse and processing it with `on('data')` event. +- [select_streaming_json_each_row_for_await.ts](node/select_streaming_json_each_row_for_await.ts) - (Node.js only) similar to [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts), but using the `for await` loop syntax. +- [select_streaming_text_line_by_line.ts](node/select_streaming_text_line_by_line.ts) - (Node.js only) streaming text formats from ClickHouse and processing it line by line. In this example, CSV format is used. #### Special cases diff --git a/examples/node/select_streaming_json_each_row.ts b/examples/node/select_streaming_json_each_row.ts new file mode 100644 index 00000000..3eaf6542 --- /dev/null +++ b/examples/node/select_streaming_json_each_row.ts @@ -0,0 +1,49 @@ +import { createClient, type Row } from '@clickhouse/client' + +/** + * Can be used for consuming large datasets for reducing memory overhead, + * or if your response exceeds built in Node.js limitations, such as 512Mb for strings. + * + * Each of the response chunks will be transformed into a relatively small arrays of rows instead + * (the size of this array depends on the size of a particular chunk the client receives from the server, + * as it may vary, and the size of an individual row), one chunk at a time. + * + * The following JSON formats can be streamed (note "EachRow" in the format name, with JSONObjectEachRow as an exception to the rule): + * - JSONEachRow + * - JSONStringsEachRow + * - JSONCompactEachRow + * - JSONCompactStringsEachRow + * - JSONCompactEachRowWithNames + * - JSONCompactEachRowWithNamesAndTypes + * - JSONCompactStringsEachRowWithNames + * - JSONCompactStringsEachRowWithNamesAndTypes + * + * See other supported formats for streaming: + * https://clickhouse.com/docs/en/integrations/language-clients/javascript#supported-data-formats + * + * NB: There might be confusion between JSON as a general format and ClickHouse JSON format (https://clickhouse.com/docs/en/sql-reference/formats#json). + * The client supports streaming JSON objects with JSONEachRow and other JSON*EachRow formats (see the list above); + * it's just that ClickHouse JSON format and a few others are represented as a single object in the response and cannot be streamed by the client. + */ +void (async () => { + const client = createClient() + const rows = await client.query({ + query: 'SELECT number FROM system.numbers_mt LIMIT 5', + format: 'JSONEachRow', // or JSONCompactEachRow, JSONStringsEachRow, etc. + }) + const stream = rows.stream() + stream.on('data', (rows: Row[]) => { + rows.forEach((row: Row) => { + console.log(row.json()) // or `row.text` to avoid parsing JSON + }) + }) + await new Promise((resolve, reject) => { + stream.on('end', () => { + console.log('Completed!') + resolve(0) + }) + stream.on('error', reject) + }) + await client.close() + process.exit(0) +})() diff --git a/examples/node/select_streaming_for_await.ts b/examples/node/select_streaming_json_each_row_for_await.ts similarity index 62% rename from examples/node/select_streaming_for_await.ts rename to examples/node/select_streaming_json_each_row_for_await.ts index 9a473400..4f4bff70 100644 --- a/examples/node/select_streaming_for_await.ts +++ b/examples/node/select_streaming_json_each_row_for_await.ts @@ -1,11 +1,10 @@ import { createClient, type Row } from '@clickhouse/client' // or '@clickhouse/client-web' /** - * NB (Node.js platform): `for await const` has quite significant overhead - * (up to 2 times worse) vs old school `on(data)` approach - * for that example, see `select_streaming_on_data.ts` + * Similar to `select_streaming_text_line_by_line.ts`, but using `for await const` syntax instead of `on(data)`. * - * See also: https://github.com/nodejs/node/issues/31979 + * NB (Node.js platform): `for await const` has some overhead (up to 2 times worse) vs the old-school `on(data)` approach. + * See the related Node.js issue: https://github.com/nodejs/node/issues/31979 */ void (async () => { const client = createClient() @@ -20,5 +19,6 @@ void (async () => { console.log(row.json()) }) } + console.log('Completed!') await client.close() })() diff --git a/examples/node/select_streaming_on_data.ts b/examples/node/select_streaming_on_data.ts deleted file mode 100644 index 0ad8274e..00000000 --- a/examples/node/select_streaming_on_data.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { createClient, type Row } from '@clickhouse/client' - -/** - * Can be used for consuming large datasets for reducing memory overhead, - * or if your response exceeds built in Node.js limitations, - * such as 512Mb for strings. - * - * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#description - * for more information. - * - * As `for await const` has quite significant overhead (up to 2 times worse) - * vs old school `on(data)` approach, this example covers `on(data)` usage - */ -void (async () => { - const client = createClient() - const rows = await client.query({ - query: 'SELECT number FROM system.numbers_mt LIMIT 5', - // See all supported formats for streaming: - // https://clickhouse.com/docs/en/integrations/language-clients/javascript#supported-data-formats - format: 'CSV', - }) - const stream = rows.stream() - stream.on('data', (rows: Row[]) => { - rows.forEach((row: Row) => { - console.log(row.text) - }) - }) - await new Promise((resolve) => { - stream.on('end', () => { - console.log('Completed!') - resolve(0) - }) - }) - await client.close() - process.exit(0) -})() diff --git a/examples/node/select_streaming_text_line_by_line.ts b/examples/node/select_streaming_text_line_by_line.ts new file mode 100644 index 00000000..81fc1cef --- /dev/null +++ b/examples/node/select_streaming_text_line_by_line.ts @@ -0,0 +1,48 @@ +import { createClient, type Row } from '@clickhouse/client' + +/** + * Can be used for consuming large datasets for reducing memory overhead, + * or if your response exceeds built in Node.js limitations, such as 512Mb for strings. + * + * Each of the response chunks will be transformed into a relatively small arrays of rows instead + * (the size of this array depends on the size of a particular chunk the client receives from the server, + * as it may vary, and the size of an individual row), one chunk at a time. + * + * The following "raw" formats can be streamed: + * - CSV + * - CSVWithNames + * - CSVWithNamesAndTypes + * - TabSeparated + * - TabSeparatedRaw + * - TabSeparatedWithNames + * - TabSeparatedWithNamesAndTypes + * - CustomSeparated + * - CustomSeparatedWithNames + * - CustomSeparatedWithNamesAndTypes + * - Parquet (see also: select_parquet_as_file.ts) + * + * See other supported formats for streaming: + * https://clickhouse.com/docs/en/integrations/language-clients/javascript#supported-data-formats + */ +void (async () => { + const client = createClient() + const rows = await client.query({ + query: 'SELECT number, number + 1 FROM system.numbers_mt LIMIT 5', + format: 'CSV', // or TabSeparated, CustomSeparated, etc. + }) + const stream = rows.stream() + stream.on('data', (rows: Row[]) => { + rows.forEach((row: Row) => { + console.log(row.text) + }) + }) + await new Promise((resolve, reject) => { + stream.on('end', () => { + console.log('Completed!') + resolve(0) + }) + stream.on('error', reject) + }) + await client.close() + process.exit(0) +})()