From 714f572e4472bf557950fa49ac8e664d176141b5 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 19 Aug 2021 20:51:20 +0200 Subject: [PATCH 1/5] Fix throwing tests --- README.MD | 16 ++++++++---- .../{basic.tests.js => combine.tests.js} | 20 +++++++++++++-- misc/some_empty/empty.json | 0 misc/some_empty/far_away.json | 25 +++++++++++++++++++ src/json-validity.js | 12 ++++----- 5 files changed, 60 insertions(+), 13 deletions(-) rename __tests__/{basic.tests.js => combine.tests.js} (85%) create mode 100644 misc/some_empty/empty.json create mode 100644 misc/some_empty/far_away.json diff --git a/README.MD b/README.MD index cbdb8e5..69a3884 100644 --- a/README.MD +++ b/README.MD @@ -8,10 +8,16 @@ -turbo-json.js is a tool that combines all json files found in a directory into one big json file using streaming to avoid out-of-memory. +turbo-json.js is a tool that combines all json files found in a directory into one big json file using **streaming** to avoid out-of-memory. -Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size. +Example: +```bash +turbo-json data +``` +All json files found in `data` directory will be combined into one file that is named `combined.json` par default. + +Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size. ## Input/Output @@ -37,7 +43,7 @@ Output file: ``` **Array exception**: -There is one exception to this rule. If your JSON file contains an array, it will be deconstructed in the final file (_could become an option please make an issue if you'd like that_). +There is one exception to this rule. If your JSON file contains an array, it will be deconstructed/flattened in the final file (_could become an option please make an issue if you'd like that_). Input files: @@ -94,12 +100,12 @@ It accepts relative path but also fully qualified paths. ### CLI usage: ```bash -turbo-json --input-dir --output-file (default: "combined.json") +turbo-json --output-file (default: "combined.json") ``` **Example** ```bash -turbo-json /data combined_data.json +turbo-json /data -o combined_data.json ``` ### Library usage diff --git a/__tests__/basic.tests.js b/__tests__/combine.tests.js similarity index 85% rename from __tests__/basic.tests.js rename to __tests__/combine.tests.js index 5f90fb5..7b7e821 100644 --- a/__tests__/basic.tests.js +++ b/__tests__/combine.tests.js @@ -9,7 +9,7 @@ beforeAll(() => { fs.mkdirSync(OUTPUT_DIR); } }); -// doesnt work with one file + test('Tests on 1 empty file', async () => { const res = await combineJson({ inputDir: 'misc/one_empty', @@ -40,6 +40,22 @@ test('Tests on multiple empty files', async () => { expect(data).toEqual(expected); }); +test.only('Tests on some invalid files and some valid', async () => { + const res = await combineJson({ + inputDir: 'misc/multiple_empty', + outputFile: 'test-output/combine_multiple_empty.json', + }); + const data = JSON.parse( + fs.readFileSync( + `${process.cwd()}/test-output/combine_multiple_empty.json`, + 'utf-8' + ) + ); + const expected = []; + expect(res).toBe(1); + expect(data).toEqual(expected); +}); + test('Tests if on 1 file containing one primitive', async () => { const res = await combineJson({ inputDir: 'misc/one_primitive', @@ -97,6 +113,6 @@ test('Tests if on all files', async () => { afterAll(() => { if (fs.existsSync(OUTPUT_DIR)) { - rimraf.sync(OUTPUT_DIR); + // rimraf.sync(OUTPUT_DIR); } }); diff --git a/misc/some_empty/empty.json b/misc/some_empty/empty.json new file mode 100644 index 0000000..e69de29 diff --git a/misc/some_empty/far_away.json b/misc/some_empty/far_away.json new file mode 100644 index 0000000..b5711b0 --- /dev/null +++ b/misc/some_empty/far_away.json @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + +[ + { + "name": "far away" + } +] diff --git a/src/json-validity.js b/src/json-validity.js index d42ddc6..5a8349c 100644 --- a/src/json-validity.js +++ b/src/json-validity.js @@ -1,3 +1,4 @@ +const { rejects } = require('assert'); const fs = require('fs'); const Verifier = require('stream-json/utils/Verifier'); @@ -6,16 +7,15 @@ async function verifyJson({ jsonFile }) { const verifier = new Verifier(); - verifier.on('error', error => { - console.log(error) - throw `Json file is not valid: ${jsonFile}` - }); - const verifierStream = fs.createReadStream(jsonFile).pipe(verifier); - await new Promise(function (resolve) { + await new Promise(function (resolve, rejects) { verifierStream.on('close', function () { resolve(); }); + verifier.on('error', error => { + console.log(error) + rejects(`Json file is not valid: ${jsonFile}`) + }); }); } From 3feddeaba5d9e0630ecdea4570d7f3a3bb52e901 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 19 Aug 2021 22:02:09 +0200 Subject: [PATCH 2/5] Catch invalid files and ignore them --- .gitignore | 4 + README.MD | 2 - __tests__/assets/combine_all.json | 1 + __tests__/combine.tests.js | 2 +- src/cli.js | 4 + src/combine-json.js | 121 ++++++++++++++++-------------- src/json-validity.js | 3 +- 7 files changed, 75 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index fa4ed88..fffb2a1 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,7 @@ combined.json # Ignore artifacts: build coverage + +# Misc +combine.json +test-output diff --git a/README.MD b/README.MD index 69a3884..6fb8504 100644 --- a/README.MD +++ b/README.MD @@ -1,5 +1,3 @@ - -
# turbo-json.js diff --git a/__tests__/assets/combine_all.json b/__tests__/assets/combine_all.json index 343c86b..6524983 100644 --- a/__tests__/assets/combine_all.json +++ b/__tests__/assets/combine_all.json @@ -1,4 +1,5 @@ [ + 1, { "name": "far away" } diff --git a/__tests__/combine.tests.js b/__tests__/combine.tests.js index 7b7e821..2ca934d 100644 --- a/__tests__/combine.tests.js +++ b/__tests__/combine.tests.js @@ -40,7 +40,7 @@ test('Tests on multiple empty files', async () => { expect(data).toEqual(expected); }); -test.only('Tests on some invalid files and some valid', async () => { +test('Tests on some invalid files and some valid', async () => { const res = await combineJson({ inputDir: 'misc/multiple_empty', outputFile: 'test-output/combine_multiple_empty.json', diff --git a/src/cli.js b/src/cli.js index de7b42a..09c33bb 100755 --- a/src/cli.js +++ b/src/cli.js @@ -5,6 +5,7 @@ const combineJson = require('./combine-json'); const program = new commander.Command() + program .argument( '', @@ -28,6 +29,9 @@ program (async () => { try { + if (process.argv.length < 3) { + console.log( program.helpInformation() ); + } await program.parse() } catch (e) { console.error(e); diff --git a/src/combine-json.js b/src/combine-json.js index 88d056b..0d62065 100755 --- a/src/combine-json.js +++ b/src/combine-json.js @@ -16,81 +16,88 @@ const BUFFER_SIZE = 1000; async function combine({ inputFiles, inputDirPath, outputFilePath }) { createOutputArrayFile(outputFilePath); const numberOfFiles = inputFiles.length; - + let first = true; for (let index = 0; index < numberOfFiles; index++) { - let fileName = inputFiles[index]; - let inputFile = `${inputDirPath}${fileName}`; - - await verifyJson({jsonFile: inputFile}) - const inputFileFd = openFile(inputFile); - - const { isArray, startPosition, empty } = jsonRootType({ - fd: inputFileFd, - bufferSize: BUFFER_SIZE, - }); + try { + let fileName = inputFiles[index]; + let inputFile = `${inputDirPath}${fileName}`; - let stopPosition = undefined; - - if (isArray) { - stopPosition = - closingArrayIndex({ - fd: inputFileFd, - position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0, - bufferSize: BUFFER_SIZE - }); - } + await verifyJson({jsonFile: inputFile}) + const inputFileFd = openFile(inputFile); - // open destination file for appending - var writeStream = fs.createWriteStream(outputFilePath, { - flags: 'a', - }); + const { isArray, startPosition, empty } = jsonRootType({ + fd: inputFileFd, + bufferSize: BUFFER_SIZE, + }); - // open source file for reading - var readStream = fs.createReadStream(inputFile, { - start: startPosition, - end: stopPosition, - }); + let stopPosition = undefined; - readStream.pipe(writeStream); + if (isArray) { + stopPosition = + closingArrayIndex({ + fd: inputFileFd, + position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0, + bufferSize: BUFFER_SIZE + }); + } - await new Promise(function (resolve) { - writeStream.on('close', function () { - resolve(); - }); - }); + if (!empty && !first) { + let comaWrite = fs.createWriteStream(outputFilePath, { + flags: 'a', + }); - let last = index === numberOfFiles - 1; + await new Promise(function (resolve) { + comaWrite.write(',', () => { + resolve(''); + }); + }); + } else if (!empty) { + first = false + } - if (!last && !empty) { - let comaWrite = fs.createWriteStream(outputFilePath, { + // open destination file for appending + var writeStream = fs.createWriteStream(outputFilePath, { flags: 'a', }); - await new Promise(function (resolve) { - comaWrite.write(',', () => { - resolve(''); - }); - }); - } else if (last) { - let closingBracketWrite = fs.createWriteStream(outputFilePath, { - flags: 'a', + // open source file for reading + var readStream = fs.createReadStream(inputFile, { + start: startPosition, + end: stopPosition, }); + readStream.pipe(writeStream); + await new Promise(function (resolve) { - closingBracketWrite.write(']', () => { - resolve(''); + writeStream.on('close', function () { + resolve(); }); }); - } - console.log( - chalk.green( - 'file: ' + - chalk.blue.underline.bold(fileName) + - ` has been added! last : ${last}, index: ${index}, numberOfFiles: ${numberOfFiles}` - ) - ); + console.log( + chalk.green( + 'file: ' + + chalk.blue.underline.bold(fileName) + + ` has been added! index: ${index}, number of files: ${numberOfFiles}` + ) + ); + } + catch (e) { + console.log(chalk.yellow( + `Invalid file is ignored: ${chalk.blue.underline.bold(e.file)}: ${e.error}` + )) + } } + + let closingBracketWrite = fs.createWriteStream(outputFilePath, { + flags: 'a', + }); + + await new Promise(function (resolve) { + closingBracketWrite.write(']', () => { + resolve(''); + }); + }); await verifyJson({jsonFile: outputFilePath}) return 1; } diff --git a/src/json-validity.js b/src/json-validity.js index 5a8349c..de6e263 100644 --- a/src/json-validity.js +++ b/src/json-validity.js @@ -13,8 +13,7 @@ async function verifyJson({ jsonFile }) { resolve(); }); verifier.on('error', error => { - console.log(error) - rejects(`Json file is not valid: ${jsonFile}`) + rejects({message: `Json file is not valid: ${jsonFile}: ${error.message}`, file: jsonFile, error: error.message}) }); }); } From ed4ff8ca3dc5c8a825ddb88c0ca377d045a6229d Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 19 Aug 2021 22:06:44 +0200 Subject: [PATCH 3/5] Remove testing in node 10 --- .github/workflows/test.yml | 2 +- package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f95b693..2740aca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: ["10", "12", "14"] + node: ["12", "14"] name: integration-tests (Node.js ${{ matrix.node }}) steps: - uses: actions/checkout@v2 diff --git a/package.json b/package.json index abc99d5..093bf0f 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,12 @@ }, "preferGlobal": true, "bugs": { - "url": "https://github.com/bidoubiwa/turbo-json/issues", + "url": "https://github.com/bidoubiwa/turbo-json.js/issues", "email": "charlottevermandel@gmail.com" }, "repository": { "type": "git", - "url": "git://github.com/bidoubiwa/turbo-json.git" + "url": "git://github.com/bidoubiwa/turbo-json.js.git" }, "author": { "name": "Charlotte Vermandel", From 85f90da92c1b94ca717a0fa289446ac8b9a58f4b Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 19 Aug 2021 22:49:00 +0200 Subject: [PATCH 4/5] Remove support for node 12 until event listener bug is fixed --- .github/workflows/test.yml | 2 +- package.json | 3 +-- src/json-validity.js | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2740aca..41ae759 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: ["12", "14"] + node: ["14", "16"] name: integration-tests (Node.js ${{ matrix.node }}) steps: - uses: actions/checkout@v2 diff --git a/package.json b/package.json index 093bf0f..94ad23e 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "main": "src/index.js", "scripts": { "start": "src/cli.js", - "lint": "prettier --write .", "test": "jest", "test:watch": "npx jest --watch" }, @@ -29,7 +28,7 @@ "src/" ], "engines": { - "node": ">=0.10.3 <17" + "node": ">=12 <15" }, "license": "MIT", "dependencies": { diff --git a/src/json-validity.js b/src/json-validity.js index de6e263..abdc51d 100644 --- a/src/json-validity.js +++ b/src/json-validity.js @@ -1,4 +1,3 @@ -const { rejects } = require('assert'); const fs = require('fs'); const Verifier = require('stream-json/utils/Verifier'); From 585f6acc724f599f4effee394794173d082f902d Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 19 Aug 2021 22:52:17 +0200 Subject: [PATCH 5/5] Add support only for node 14 --- .github/workflows/test.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41ae759..8e70bc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: ["14", "16"] + node: ["14"] name: integration-tests (Node.js ${{ matrix.node }}) steps: - uses: actions/checkout@v2 diff --git a/package.json b/package.json index 94ad23e..ba7d0ba 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "src/" ], "engines": { - "node": ">=12 <15" + "node": ">=14 <15" }, "license": "MIT", "dependencies": {