Merge pull request #25 from bidoubiwa/fix_tests

Fix tests failing because of not catched wrongly formated files

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: ["10", "12", "14"]
+        node: ["14"]
     name: integration-tests (Node.js ${{ matrix.node }})
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -97,3 +97,7 @@ combined.json
 # Ignore artifacts:
 build
 coverage
+
+# Misc
+combine.json
+test-output

README.MD

@@ -1,5 +1,3 @@
-
-
 <center>
 
 # turbo-json.js
@@ -8,10 +6,16 @@
 
 </center>
 
-turbo-json.js is a tool that combines all json files found in a directory into one big json file using streaming to avoid out-of-memory.
+turbo-json.js is a tool that combines all json files found in a directory into one big json file using **streaming** to avoid out-of-memory.
 
-Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size.
+Example:
+```bash
+turbo-json data
+```
+
+All json files found in `data` directory will be combined into one file that is named `combined.json` par default.
 
+Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size.
 
 ## Input/Output
 
@@ -37,7 +41,7 @@ Output file:
 ```
 
 **Array exception**:
-There is one exception to this rule. If your JSON file contains an array, it will be deconstructed in the final file (_could become an option please make an issue if you'd like that_).
+There is one exception to this rule. If your JSON file contains an array, it will be deconstructed/flattened in the final file (_could become an option please make an issue if you'd like that_).
 
 Input files:
 
@@ -94,12 +98,12 @@ It accepts relative path but also fully qualified paths.
 ### CLI usage:
 
 ```bash
-turbo-json --input-dir <dir-path> --output-file <file-path> (default: "combined.json")
+turbo-json <input-directory> --output-file <file-path> (default: "combined.json")
 ```
 
 **Example**
 ```bash
-turbo-json /data combined_data.json
+turbo-json /data -o combined_data.json
 ```
 
 ### Library usage

__tests__/assets/combine_all.json

@@ -1,4 +1,5 @@
 [
+  1,
   {
   "name": "far away"
   }

__tests__/basic.tests.js → __tests__/combine.tests.js

@@ -9,7 +9,7 @@ beforeAll(() => {
     fs.mkdirSync(OUTPUT_DIR);
   }
 });
-// doesnt work with one file
+
 test('Tests on 1 empty file', async () => {
   const res = await combineJson({
     inputDir: 'misc/one_empty',
@@ -40,6 +40,22 @@ test('Tests on multiple empty files', async () => {
   expect(data).toEqual(expected);
 });
 
+test('Tests on some invalid files and some valid', async () => {
+  const res = await combineJson({
+    inputDir: 'misc/multiple_empty',
+    outputFile: 'test-output/combine_multiple_empty.json',
+  });
+  const data = JSON.parse(
+    fs.readFileSync(
+      `${process.cwd()}/test-output/combine_multiple_empty.json`,
+      'utf-8'
+    )
+  );
+  const expected = [];
+  expect(res).toBe(1);
+  expect(data).toEqual(expected);
+});
+
 test('Tests if on 1 file containing one primitive', async () => {
   const res = await combineJson({
     inputDir: 'misc/one_primitive',
@@ -97,6 +113,6 @@ test('Tests if on all files', async () => {
 
 afterAll(() => {
   if (fs.existsSync(OUTPUT_DIR)) {
-    rimraf.sync(OUTPUT_DIR);
+    // rimraf.sync(OUTPUT_DIR);
   }
 });

misc/some_empty/far_away.json

@@ -0,0 +1,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+[
+    {
+    "name": "far away"
+    }
+]

package.json

@@ -5,18 +5,17 @@
   "main": "src/index.js",
   "scripts": {
     "start": "src/cli.js",
-    "lint": "prettier --write .",
     "test": "jest",
     "test:watch": "npx jest --watch"
   },
   "preferGlobal": true,
   "bugs": {
-    "url": "https://github.com/bidoubiwa/turbo-json/issues",
+    "url": "https://github.com/bidoubiwa/turbo-json.js/issues",
     "email": "[email protected]"
   },
   "repository": {
     "type": "git",
-    "url": "git://github.com/bidoubiwa/turbo-json.git"
+    "url": "git://github.com/bidoubiwa/turbo-json.js.git"
   },
   "author": {
     "name": "Charlotte Vermandel",
@@ -29,7 +28,7 @@
     "src/"
   ],
   "engines": {
-    "node": ">=0.10.3 <17"
+    "node": ">=14 <15"
   },
   "license": "MIT",
   "dependencies": {

src/cli.js

@@ -5,6 +5,7 @@ const combineJson = require('./combine-json');
 
 const program = new commander.Command()
 
+
 program
   .argument(
     '<input-directory>',
@@ -28,6 +29,9 @@ program
 
 (async () => {
   try {
+    if (process.argv.length < 3) {
+      console.log( program.helpInformation() );
+    }
     await program.parse()
   } catch (e) {
     console.error(e);

src/combine-json.js

@@ -16,81 +16,88 @@ const BUFFER_SIZE = 1000;
 async function combine({ inputFiles, inputDirPath, outputFilePath }) {
   createOutputArrayFile(outputFilePath);
   const numberOfFiles = inputFiles.length;
-
+  let first = true;
   for (let index = 0; index < numberOfFiles; index++) {
-    let fileName = inputFiles[index];
-    let inputFile = `${inputDirPath}${fileName}`;
-
-    await verifyJson({jsonFile: inputFile})
-    const inputFileFd = openFile(inputFile);
-
-    const { isArray, startPosition, empty } = jsonRootType({
-      fd: inputFileFd,
-      bufferSize: BUFFER_SIZE,
-    });
+    try {
+      let fileName = inputFiles[index];
+      let inputFile = `${inputDirPath}${fileName}`;
 
-    let stopPosition = undefined;
-
-    if (isArray) {
-      stopPosition =
-        closingArrayIndex({
-          fd: inputFileFd,
-          position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0,
-          bufferSize: BUFFER_SIZE
-        });
-    }
+      await verifyJson({jsonFile: inputFile})
+      const inputFileFd = openFile(inputFile);
 
-    // open destination file for appending
-    var writeStream = fs.createWriteStream(outputFilePath, {
-      flags: 'a',
-    });
+      const { isArray, startPosition, empty } = jsonRootType({
+        fd: inputFileFd,
+        bufferSize: BUFFER_SIZE,
+      });
 
-    // open source file for reading
-    var readStream = fs.createReadStream(inputFile, {
-      start: startPosition,
-      end: stopPosition,
-    });
+      let stopPosition = undefined;
 
-    readStream.pipe(writeStream);
+      if (isArray) {
+        stopPosition =
+          closingArrayIndex({
+            fd: inputFileFd,
+            position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0,
+            bufferSize: BUFFER_SIZE
+          });
+      }
 
-    await new Promise(function (resolve) {
-      writeStream.on('close', function () {
-        resolve();
-      });
-    });
+      if (!empty && !first) {
+        let comaWrite = fs.createWriteStream(outputFilePath, {
+          flags: 'a',
+        });
 
-    let last = index === numberOfFiles - 1;
+        await new Promise(function (resolve) {
+          comaWrite.write(',', () => {
+            resolve('');
+          });
+        });
+      } else if (!empty) {
+        first = false
+      }
 
-    if (!last && !empty) {
-      let comaWrite = fs.createWriteStream(outputFilePath, {
+      // open destination file for appending
+      var writeStream = fs.createWriteStream(outputFilePath, {
         flags: 'a',
       });
 
-      await new Promise(function (resolve) {
-        comaWrite.write(',', () => {
-          resolve('');
-        });
-      });
-    } else if (last) {
-      let closingBracketWrite = fs.createWriteStream(outputFilePath, {
-        flags: 'a',
+      // open source file for reading
+      var readStream = fs.createReadStream(inputFile, {
+        start: startPosition,
+        end: stopPosition,
       });
 
+      readStream.pipe(writeStream);
+
       await new Promise(function (resolve) {
-        closingBracketWrite.write(']', () => {
-          resolve('');
+        writeStream.on('close', function () {
+          resolve();
         });
       });
-    }
 
-    console.log(
-      chalk.green(
-        'file: ' +
-          chalk.blue.underline.bold(fileName) +
-          ` has been added! last : ${last}, index: ${index}, numberOfFiles: ${numberOfFiles}`
-      )
-    );
+      console.log(
+        chalk.green(
+          'file: ' +
+            chalk.blue.underline.bold(fileName) +
+            ` has been added! index: ${index}, number of files: ${numberOfFiles}`
+        )
+      );
+    }
+    catch (e) {
+      console.log(chalk.yellow(
+        `Invalid file is ignored: ${chalk.blue.underline.bold(e.file)}: ${e.error}`
+      ))
+    }
   }
+
+  let closingBracketWrite = fs.createWriteStream(outputFilePath, {
+    flags: 'a',
+  });
+
+  await new Promise(function (resolve) {
+    closingBracketWrite.write(']', () => {
+      resolve('');
+    });
+  });
   await verifyJson({jsonFile: outputFilePath})
   return 1;
 }

src/json-validity.js

@@ -6,16 +6,14 @@ async function verifyJson({ jsonFile }) {
 
   const verifier = new Verifier();
 
-  verifier.on('error', error => {
-    console.log(error)
-    throw `Json file is not valid: ${jsonFile}`
-  });
-
   const verifierStream = fs.createReadStream(jsonFile).pipe(verifier);
-  await new Promise(function (resolve) {
+  await new Promise(function (resolve, rejects) {
     verifierStream.on('close', function () {
       resolve();
     });
+    verifier.on('error', error => {
+      rejects({message: `Json file is not valid: ${jsonFile}: ${error.message}`, file: jsonFile, error: error.message})
+    });
   });
 }