diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f95b693..8e70bc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: ["10", "12", "14"] + node: ["14"] name: integration-tests (Node.js ${{ matrix.node }}) steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index fa4ed88..fffb2a1 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,7 @@ combined.json # Ignore artifacts: build coverage + +# Misc +combine.json +test-output diff --git a/README.MD b/README.MD index cbdb8e5..6fb8504 100644 --- a/README.MD +++ b/README.MD @@ -1,5 +1,3 @@ - -
# turbo-json.js @@ -8,10 +6,16 @@
-turbo-json.js is a tool that combines all json files found in a directory into one big json file using streaming to avoid out-of-memory. +turbo-json.js is a tool that combines all json files found in a directory into one big json file using **streaming** to avoid out-of-memory. -Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size. +Example: +```bash +turbo-json data +``` + +All json files found in `data` directory will be combined into one file that is named `combined.json` par default. +Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size. ## Input/Output @@ -37,7 +41,7 @@ Output file: ``` **Array exception**: -There is one exception to this rule. If your JSON file contains an array, it will be deconstructed in the final file (_could become an option please make an issue if you'd like that_). +There is one exception to this rule. If your JSON file contains an array, it will be deconstructed/flattened in the final file (_could become an option please make an issue if you'd like that_). Input files: @@ -94,12 +98,12 @@ It accepts relative path but also fully qualified paths. ### CLI usage: ```bash -turbo-json --input-dir --output-file (default: "combined.json") +turbo-json --output-file (default: "combined.json") ``` **Example** ```bash -turbo-json /data combined_data.json +turbo-json /data -o combined_data.json ``` ### Library usage diff --git a/__tests__/assets/combine_all.json b/__tests__/assets/combine_all.json index 343c86b..6524983 100644 --- a/__tests__/assets/combine_all.json +++ b/__tests__/assets/combine_all.json @@ -1,4 +1,5 @@ [ + 1, { "name": "far away" } diff --git a/__tests__/basic.tests.js b/__tests__/combine.tests.js similarity index 85% rename from __tests__/basic.tests.js rename to __tests__/combine.tests.js index 5f90fb5..2ca934d 100644 --- a/__tests__/basic.tests.js +++ b/__tests__/combine.tests.js @@ -9,7 +9,7 @@ beforeAll(() => { fs.mkdirSync(OUTPUT_DIR); } }); -// doesnt work with one file + test('Tests on 1 empty file', async () => { const res = await combineJson({ inputDir: 'misc/one_empty', @@ -40,6 +40,22 @@ test('Tests on multiple empty files', async () => { expect(data).toEqual(expected); }); +test('Tests on some invalid files and some valid', async () => { + const res = await combineJson({ + inputDir: 'misc/multiple_empty', + outputFile: 'test-output/combine_multiple_empty.json', + }); + const data = JSON.parse( + fs.readFileSync( + `${process.cwd()}/test-output/combine_multiple_empty.json`, + 'utf-8' + ) + ); + const expected = []; + expect(res).toBe(1); + expect(data).toEqual(expected); +}); + test('Tests if on 1 file containing one primitive', async () => { const res = await combineJson({ inputDir: 'misc/one_primitive', @@ -97,6 +113,6 @@ test('Tests if on all files', async () => { afterAll(() => { if (fs.existsSync(OUTPUT_DIR)) { - rimraf.sync(OUTPUT_DIR); + // rimraf.sync(OUTPUT_DIR); } }); diff --git a/misc/some_empty/empty.json b/misc/some_empty/empty.json new file mode 100644 index 0000000..e69de29 diff --git a/misc/some_empty/far_away.json b/misc/some_empty/far_away.json new file mode 100644 index 0000000..b5711b0 --- /dev/null +++ b/misc/some_empty/far_away.json @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + +[ + { + "name": "far away" + } +] diff --git a/package.json b/package.json index abc99d5..ba7d0ba 100644 --- a/package.json +++ b/package.json @@ -5,18 +5,17 @@ "main": "src/index.js", "scripts": { "start": "src/cli.js", - "lint": "prettier --write .", "test": "jest", "test:watch": "npx jest --watch" }, "preferGlobal": true, "bugs": { - "url": "https://github.com/bidoubiwa/turbo-json/issues", + "url": "https://github.com/bidoubiwa/turbo-json.js/issues", "email": "charlottevermandel@gmail.com" }, "repository": { "type": "git", - "url": "git://github.com/bidoubiwa/turbo-json.git" + "url": "git://github.com/bidoubiwa/turbo-json.js.git" }, "author": { "name": "Charlotte Vermandel", @@ -29,7 +28,7 @@ "src/" ], "engines": { - "node": ">=0.10.3 <17" + "node": ">=14 <15" }, "license": "MIT", "dependencies": { diff --git a/src/cli.js b/src/cli.js index de7b42a..09c33bb 100755 --- a/src/cli.js +++ b/src/cli.js @@ -5,6 +5,7 @@ const combineJson = require('./combine-json'); const program = new commander.Command() + program .argument( '', @@ -28,6 +29,9 @@ program (async () => { try { + if (process.argv.length < 3) { + console.log( program.helpInformation() ); + } await program.parse() } catch (e) { console.error(e); diff --git a/src/combine-json.js b/src/combine-json.js index 88d056b..0d62065 100755 --- a/src/combine-json.js +++ b/src/combine-json.js @@ -16,81 +16,88 @@ const BUFFER_SIZE = 1000; async function combine({ inputFiles, inputDirPath, outputFilePath }) { createOutputArrayFile(outputFilePath); const numberOfFiles = inputFiles.length; - + let first = true; for (let index = 0; index < numberOfFiles; index++) { - let fileName = inputFiles[index]; - let inputFile = `${inputDirPath}${fileName}`; - - await verifyJson({jsonFile: inputFile}) - const inputFileFd = openFile(inputFile); - - const { isArray, startPosition, empty } = jsonRootType({ - fd: inputFileFd, - bufferSize: BUFFER_SIZE, - }); + try { + let fileName = inputFiles[index]; + let inputFile = `${inputDirPath}${fileName}`; - let stopPosition = undefined; - - if (isArray) { - stopPosition = - closingArrayIndex({ - fd: inputFileFd, - position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0, - bufferSize: BUFFER_SIZE - }); - } + await verifyJson({jsonFile: inputFile}) + const inputFileFd = openFile(inputFile); - // open destination file for appending - var writeStream = fs.createWriteStream(outputFilePath, { - flags: 'a', - }); + const { isArray, startPosition, empty } = jsonRootType({ + fd: inputFileFd, + bufferSize: BUFFER_SIZE, + }); - // open source file for reading - var readStream = fs.createReadStream(inputFile, { - start: startPosition, - end: stopPosition, - }); + let stopPosition = undefined; - readStream.pipe(writeStream); + if (isArray) { + stopPosition = + closingArrayIndex({ + fd: inputFileFd, + position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0, + bufferSize: BUFFER_SIZE + }); + } - await new Promise(function (resolve) { - writeStream.on('close', function () { - resolve(); - }); - }); + if (!empty && !first) { + let comaWrite = fs.createWriteStream(outputFilePath, { + flags: 'a', + }); - let last = index === numberOfFiles - 1; + await new Promise(function (resolve) { + comaWrite.write(',', () => { + resolve(''); + }); + }); + } else if (!empty) { + first = false + } - if (!last && !empty) { - let comaWrite = fs.createWriteStream(outputFilePath, { + // open destination file for appending + var writeStream = fs.createWriteStream(outputFilePath, { flags: 'a', }); - await new Promise(function (resolve) { - comaWrite.write(',', () => { - resolve(''); - }); - }); - } else if (last) { - let closingBracketWrite = fs.createWriteStream(outputFilePath, { - flags: 'a', + // open source file for reading + var readStream = fs.createReadStream(inputFile, { + start: startPosition, + end: stopPosition, }); + readStream.pipe(writeStream); + await new Promise(function (resolve) { - closingBracketWrite.write(']', () => { - resolve(''); + writeStream.on('close', function () { + resolve(); }); }); - } - console.log( - chalk.green( - 'file: ' + - chalk.blue.underline.bold(fileName) + - ` has been added! last : ${last}, index: ${index}, numberOfFiles: ${numberOfFiles}` - ) - ); + console.log( + chalk.green( + 'file: ' + + chalk.blue.underline.bold(fileName) + + ` has been added! index: ${index}, number of files: ${numberOfFiles}` + ) + ); + } + catch (e) { + console.log(chalk.yellow( + `Invalid file is ignored: ${chalk.blue.underline.bold(e.file)}: ${e.error}` + )) + } } + + let closingBracketWrite = fs.createWriteStream(outputFilePath, { + flags: 'a', + }); + + await new Promise(function (resolve) { + closingBracketWrite.write(']', () => { + resolve(''); + }); + }); await verifyJson({jsonFile: outputFilePath}) return 1; } diff --git a/src/json-validity.js b/src/json-validity.js index d42ddc6..abdc51d 100644 --- a/src/json-validity.js +++ b/src/json-validity.js @@ -6,16 +6,14 @@ async function verifyJson({ jsonFile }) { const verifier = new Verifier(); - verifier.on('error', error => { - console.log(error) - throw `Json file is not valid: ${jsonFile}` - }); - const verifierStream = fs.createReadStream(jsonFile).pipe(verifier); - await new Promise(function (resolve) { + await new Promise(function (resolve, rejects) { verifierStream.on('close', function () { resolve(); }); + verifier.on('error', error => { + rejects({message: `Json file is not valid: ${jsonFile}: ${error.message}`, file: jsonFile, error: error.message}) + }); }); }