diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f95b693..8e70bc7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- node: ["10", "12", "14"]
+ node: ["14"]
name: integration-tests (Node.js ${{ matrix.node }})
steps:
- uses: actions/checkout@v2
diff --git a/.gitignore b/.gitignore
index fa4ed88..fffb2a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -97,3 +97,7 @@ combined.json
# Ignore artifacts:
build
coverage
+
+# Misc
+combine.json
+test-output
diff --git a/README.MD b/README.MD
index cbdb8e5..6fb8504 100644
--- a/README.MD
+++ b/README.MD
@@ -1,5 +1,3 @@
-
-
# turbo-json.js
@@ -8,10 +6,16 @@
-turbo-json.js is a tool that combines all json files found in a directory into one big json file using streaming to avoid out-of-memory.
+turbo-json.js is a tool that combines all json files found in a directory into one big json file using **streaming** to avoid out-of-memory.
-Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size.
+Example:
+```bash
+turbo-json data
+```
+
+All json files found in `data` directory will be combined into one file that is named `combined.json` par default.
+Both `read` and `write` actions are done using `streaming`. The maximum data stored in memory is the buffer size.
## Input/Output
@@ -37,7 +41,7 @@ Output file:
```
**Array exception**:
-There is one exception to this rule. If your JSON file contains an array, it will be deconstructed in the final file (_could become an option please make an issue if you'd like that_).
+There is one exception to this rule. If your JSON file contains an array, it will be deconstructed/flattened in the final file (_could become an option please make an issue if you'd like that_).
Input files:
@@ -94,12 +98,12 @@ It accepts relative path but also fully qualified paths.
### CLI usage:
```bash
-turbo-json --input-dir --output-file (default: "combined.json")
+turbo-json --output-file (default: "combined.json")
```
**Example**
```bash
-turbo-json /data combined_data.json
+turbo-json /data -o combined_data.json
```
### Library usage
diff --git a/__tests__/assets/combine_all.json b/__tests__/assets/combine_all.json
index 343c86b..6524983 100644
--- a/__tests__/assets/combine_all.json
+++ b/__tests__/assets/combine_all.json
@@ -1,4 +1,5 @@
[
+ 1,
{
"name": "far away"
}
diff --git a/__tests__/basic.tests.js b/__tests__/combine.tests.js
similarity index 85%
rename from __tests__/basic.tests.js
rename to __tests__/combine.tests.js
index 5f90fb5..2ca934d 100644
--- a/__tests__/basic.tests.js
+++ b/__tests__/combine.tests.js
@@ -9,7 +9,7 @@ beforeAll(() => {
fs.mkdirSync(OUTPUT_DIR);
}
});
-// doesnt work with one file
+
test('Tests on 1 empty file', async () => {
const res = await combineJson({
inputDir: 'misc/one_empty',
@@ -40,6 +40,22 @@ test('Tests on multiple empty files', async () => {
expect(data).toEqual(expected);
});
+test('Tests on some invalid files and some valid', async () => {
+ const res = await combineJson({
+ inputDir: 'misc/multiple_empty',
+ outputFile: 'test-output/combine_multiple_empty.json',
+ });
+ const data = JSON.parse(
+ fs.readFileSync(
+ `${process.cwd()}/test-output/combine_multiple_empty.json`,
+ 'utf-8'
+ )
+ );
+ const expected = [];
+ expect(res).toBe(1);
+ expect(data).toEqual(expected);
+});
+
test('Tests if on 1 file containing one primitive', async () => {
const res = await combineJson({
inputDir: 'misc/one_primitive',
@@ -97,6 +113,6 @@ test('Tests if on all files', async () => {
afterAll(() => {
if (fs.existsSync(OUTPUT_DIR)) {
- rimraf.sync(OUTPUT_DIR);
+ // rimraf.sync(OUTPUT_DIR);
}
});
diff --git a/misc/some_empty/empty.json b/misc/some_empty/empty.json
new file mode 100644
index 0000000..e69de29
diff --git a/misc/some_empty/far_away.json b/misc/some_empty/far_away.json
new file mode 100644
index 0000000..b5711b0
--- /dev/null
+++ b/misc/some_empty/far_away.json
@@ -0,0 +1,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+[
+ {
+ "name": "far away"
+ }
+]
diff --git a/package.json b/package.json
index abc99d5..ba7d0ba 100644
--- a/package.json
+++ b/package.json
@@ -5,18 +5,17 @@
"main": "src/index.js",
"scripts": {
"start": "src/cli.js",
- "lint": "prettier --write .",
"test": "jest",
"test:watch": "npx jest --watch"
},
"preferGlobal": true,
"bugs": {
- "url": "https://github.com/bidoubiwa/turbo-json/issues",
+ "url": "https://github.com/bidoubiwa/turbo-json.js/issues",
"email": "charlottevermandel@gmail.com"
},
"repository": {
"type": "git",
- "url": "git://github.com/bidoubiwa/turbo-json.git"
+ "url": "git://github.com/bidoubiwa/turbo-json.js.git"
},
"author": {
"name": "Charlotte Vermandel",
@@ -29,7 +28,7 @@
"src/"
],
"engines": {
- "node": ">=0.10.3 <17"
+ "node": ">=14 <15"
},
"license": "MIT",
"dependencies": {
diff --git a/src/cli.js b/src/cli.js
index de7b42a..09c33bb 100755
--- a/src/cli.js
+++ b/src/cli.js
@@ -5,6 +5,7 @@ const combineJson = require('./combine-json');
const program = new commander.Command()
+
program
.argument(
'',
@@ -28,6 +29,9 @@ program
(async () => {
try {
+ if (process.argv.length < 3) {
+ console.log( program.helpInformation() );
+ }
await program.parse()
} catch (e) {
console.error(e);
diff --git a/src/combine-json.js b/src/combine-json.js
index 88d056b..0d62065 100755
--- a/src/combine-json.js
+++ b/src/combine-json.js
@@ -16,81 +16,88 @@ const BUFFER_SIZE = 1000;
async function combine({ inputFiles, inputDirPath, outputFilePath }) {
createOutputArrayFile(outputFilePath);
const numberOfFiles = inputFiles.length;
-
+ let first = true;
for (let index = 0; index < numberOfFiles; index++) {
- let fileName = inputFiles[index];
- let inputFile = `${inputDirPath}${fileName}`;
-
- await verifyJson({jsonFile: inputFile})
- const inputFileFd = openFile(inputFile);
-
- const { isArray, startPosition, empty } = jsonRootType({
- fd: inputFileFd,
- bufferSize: BUFFER_SIZE,
- });
+ try {
+ let fileName = inputFiles[index];
+ let inputFile = `${inputDirPath}${fileName}`;
- let stopPosition = undefined;
-
- if (isArray) {
- stopPosition =
- closingArrayIndex({
- fd: inputFileFd,
- position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0,
- bufferSize: BUFFER_SIZE
- });
- }
+ await verifyJson({jsonFile: inputFile})
+ const inputFileFd = openFile(inputFile);
- // open destination file for appending
- var writeStream = fs.createWriteStream(outputFilePath, {
- flags: 'a',
- });
+ const { isArray, startPosition, empty } = jsonRootType({
+ fd: inputFileFd,
+ bufferSize: BUFFER_SIZE,
+ });
- // open source file for reading
- var readStream = fs.createReadStream(inputFile, {
- start: startPosition,
- end: stopPosition,
- });
+ let stopPosition = undefined;
- readStream.pipe(writeStream);
+ if (isArray) {
+ stopPosition =
+ closingArrayIndex({
+ fd: inputFileFd,
+ position: (fileSize(inputFile) - BUFFER_SIZE > 0) ? fileSize(inputFile) - BUFFER_SIZE : 0,
+ bufferSize: BUFFER_SIZE
+ });
+ }
- await new Promise(function (resolve) {
- writeStream.on('close', function () {
- resolve();
- });
- });
+ if (!empty && !first) {
+ let comaWrite = fs.createWriteStream(outputFilePath, {
+ flags: 'a',
+ });
- let last = index === numberOfFiles - 1;
+ await new Promise(function (resolve) {
+ comaWrite.write(',', () => {
+ resolve('');
+ });
+ });
+ } else if (!empty) {
+ first = false
+ }
- if (!last && !empty) {
- let comaWrite = fs.createWriteStream(outputFilePath, {
+ // open destination file for appending
+ var writeStream = fs.createWriteStream(outputFilePath, {
flags: 'a',
});
- await new Promise(function (resolve) {
- comaWrite.write(',', () => {
- resolve('');
- });
- });
- } else if (last) {
- let closingBracketWrite = fs.createWriteStream(outputFilePath, {
- flags: 'a',
+ // open source file for reading
+ var readStream = fs.createReadStream(inputFile, {
+ start: startPosition,
+ end: stopPosition,
});
+ readStream.pipe(writeStream);
+
await new Promise(function (resolve) {
- closingBracketWrite.write(']', () => {
- resolve('');
+ writeStream.on('close', function () {
+ resolve();
});
});
- }
- console.log(
- chalk.green(
- 'file: ' +
- chalk.blue.underline.bold(fileName) +
- ` has been added! last : ${last}, index: ${index}, numberOfFiles: ${numberOfFiles}`
- )
- );
+ console.log(
+ chalk.green(
+ 'file: ' +
+ chalk.blue.underline.bold(fileName) +
+ ` has been added! index: ${index}, number of files: ${numberOfFiles}`
+ )
+ );
+ }
+ catch (e) {
+ console.log(chalk.yellow(
+ `Invalid file is ignored: ${chalk.blue.underline.bold(e.file)}: ${e.error}`
+ ))
+ }
}
+
+ let closingBracketWrite = fs.createWriteStream(outputFilePath, {
+ flags: 'a',
+ });
+
+ await new Promise(function (resolve) {
+ closingBracketWrite.write(']', () => {
+ resolve('');
+ });
+ });
await verifyJson({jsonFile: outputFilePath})
return 1;
}
diff --git a/src/json-validity.js b/src/json-validity.js
index d42ddc6..abdc51d 100644
--- a/src/json-validity.js
+++ b/src/json-validity.js
@@ -6,16 +6,14 @@ async function verifyJson({ jsonFile }) {
const verifier = new Verifier();
- verifier.on('error', error => {
- console.log(error)
- throw `Json file is not valid: ${jsonFile}`
- });
-
const verifierStream = fs.createReadStream(jsonFile).pipe(verifier);
- await new Promise(function (resolve) {
+ await new Promise(function (resolve, rejects) {
verifierStream.on('close', function () {
resolve();
});
+ verifier.on('error', error => {
+ rejects({message: `Json file is not valid: ${jsonFile}: ${error.message}`, file: jsonFile, error: error.message})
+ });
});
}