diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..a0be7a4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,22 @@ +# top-most EditorConfig file +root = true + +# all files +[*] +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +charset = utf-8 +trim_trailing_whitespace = true +max_line_length = 80 + +[*.{js,ts}] +quote_type = single +curly_bracket_next_line = false +spaces_around_brackets = inside +indent_brace_style = BSD KNF + +# HTML +[*.html] +quote_type = double diff --git a/.gitignore b/.gitignore index 00f8a3e..14b0db6 100644 --- a/.gitignore +++ b/.gitignore @@ -90,4 +90,4 @@ lerna-debug.log .DS_Store Thumbs.db -combined.json \ No newline at end of file +combined.json diff --git a/README.MD b/README.MD index 0aa5dc3..bdf7e1d 100644 --- a/README.MD +++ b/README.MD @@ -1,6 +1,6 @@ # Combine-json -Combine-json is a CLI tool that combines all json files found in a directory into one big json file. +Combine-json is a CLI tool that combines all json files found in a directory into one big json file using streaming. It takes as argument a directory in which to find the json files and a output file name in which the json files will combined. The json file will be created in the directory where the CLI is used. @@ -14,7 +14,7 @@ npm i npm run link ``` -## Usage +## Usage `combine-json [inputDir] [outputFile(optionnal)]` ```bash combine-json directory @@ -22,7 +22,7 @@ combine-json directory default output file is combined.json at the root of where you execute the cli. ## Try it out -Go at the root of the CLI +Go at the root of the CLI ```bash combine-json misc ``` diff --git a/combined.json b/combined.json index e4700c0..6a020a4 100644 --- a/combined.json +++ b/combined.json @@ -10,7 +10,61 @@ { "name": "Hello" }, { "name": "Hello" }, { "name": "Hello" } -,{ +, + { + "id": "106678", + "title": "Jungle Erotic", + "poster": "https://image.tmdb.org/t/p/w1280/rkMHlp42CDJUEB4GKsn3LC5znGC.jpg", + "overview": "Young women have an adventure in the African jungle - where they are harassed by gentle men, and wild monkeys.", + "release_date": 26265600, + "genre": [ + "Adventure", + "Drama" + ] + }, + { + "id": "490410", + "title": "48 Christmas Wishes", + "poster": "https://image.tmdb.org/t/p/w1280/mrcA62O9j8y4gsSpftMKVcvrzCI.jpg", + "overview": "Holiday magic has never gleamed brighter as Santa’s Elves head out on an adventure to bring every boy and girl their Christmas wish while also bringing a family back together in this heartfelt Christmas movie from Gaumont. After accidentally destroying a bag of Christmas wishes from a small town on the eve of Christmas Eve, two junior elves learn that if even one wish goes unfulfilled, Christmas could be extinguished forever. For the first time in their lives, the elves Mindy and Cam venture out of the North Pole and sneak into Minnedoza to collect the lost wishes. Along the way they enlist the help of young Blake, whose family has a difficult time celebrating Christmas since his father died. With their deadline fast approaching, Mindy and Cam have only one more wish to find…but whose could it be? It’s up to Mindy and Cam to find out and save Christmas!", + "release_date": 1512086400, + "genre": [ + "Family", + "Adventure", + "Comedy", + "TV Movie" + ] + }, + { + "id": "285841", + "title": "Elephant Song", + "poster": "https://image.tmdb.org/t/p/w1280/9pHl6RmMIDyPmQW1XUJ2Mahsr3M.jpg", + "overview": "A psychiatrist is drawn into a complex mind game when he questions a disturbed patient about the disappearance of a colleague.", + "release_date": 1409792400, + "genre": [] + } +, + { + "id": "481370", + "title": "Bigger", + "poster": "https://image.tmdb.org/t/p/w1280/f6rmiFTfT3Rf9XDrXfLO6XcAgWn.jpg", + "overview": "The inspirational tale of the grandfathers of the fitness movement as we now know it, Joe & Ben Weider. Battling anti-Semitism, racism and extreme poverty, the brothers beat all odds to build an empire & inspire future generations.", + "release_date": 1539392400, + "genre": [ + "Drama" + ] + }, + { + "id": "31357", + "title": "Waiting to Exhale", + "poster": "https://image.tmdb.org/t/p/w1280/4wjGMwPsdlvi025ZqR4rXnFDvBz.jpg", + "overview": "Cheated on, mistreated and stepped on, the women are holding their breath, waiting for the elusive 'good man' to break a string of less-than-stellar lovers. Friends and confidants Vannah, Bernie, Glo and Robin talk it all out, determined to find a better way to breathe.", + "release_date": 819590400, + "genre": [ + "Comedy", + "Drama", + "Romance" + ,{ "name": "not an array" }, 1, @@ -20,4 +74,4 @@ { "name": "world" }, { "name": "world" }, { "name": "world" } -] \ No newline at end of file +] diff --git a/misc/far_away.json b/misc/far_away.json index 0efa659..b5711b0 100644 --- a/misc/far_away.json +++ b/misc/far_away.json @@ -22,4 +22,4 @@ { "name": "far away" } -] \ No newline at end of file +] diff --git a/misc/hello.json b/misc/hello.json index 71fccf9..878b5fd 100644 --- a/misc/hello.json +++ b/misc/hello.json @@ -2,4 +2,4 @@ { "name": "Hello" }, { "name": "Hello" }, { "name": "Hello" } -] \ No newline at end of file +] diff --git a/misc/nested_array.json b/misc/nested_array.json new file mode 100644 index 0000000..c2549e5 --- /dev/null +++ b/misc/nested_array.json @@ -0,0 +1,34 @@ +[ + { + "id": "106678", + "title": "Jungle Erotic", + "poster": "https://image.tmdb.org/t/p/w1280/rkMHlp42CDJUEB4GKsn3LC5znGC.jpg", + "overview": "Young women have an adventure in the African jungle - where they are harassed by gentle men, and wild monkeys.", + "release_date": 26265600, + "genre": [ + "Adventure", + "Drama" + ] + }, + { + "id": "490410", + "title": "48 Christmas Wishes", + "poster": "https://image.tmdb.org/t/p/w1280/mrcA62O9j8y4gsSpftMKVcvrzCI.jpg", + "overview": "Holiday magic has never gleamed brighter as Santa’s Elves head out on an adventure to bring every boy and girl their Christmas wish while also bringing a family back together in this heartfelt Christmas movie from Gaumont. After accidentally destroying a bag of Christmas wishes from a small town on the eve of Christmas Eve, two junior elves learn that if even one wish goes unfulfilled, Christmas could be extinguished forever. For the first time in their lives, the elves Mindy and Cam venture out of the North Pole and sneak into Minnedoza to collect the lost wishes. Along the way they enlist the help of young Blake, whose family has a difficult time celebrating Christmas since his father died. With their deadline fast approaching, Mindy and Cam have only one more wish to find…but whose could it be? It’s up to Mindy and Cam to find out and save Christmas!", + "release_date": 1512086400, + "genre": [ + "Family", + "Adventure", + "Comedy", + "TV Movie" + ] + }, + { + "id": "285841", + "title": "Elephant Song", + "poster": "https://image.tmdb.org/t/p/w1280/9pHl6RmMIDyPmQW1XUJ2Mahsr3M.jpg", + "overview": "A psychiatrist is drawn into a complex mind game when he questions a disturbed patient about the disappearance of a colleague.", + "release_date": 1409792400, + "genre": [] + } +] diff --git a/misc/nested_array_2.json b/misc/nested_array_2.json new file mode 100644 index 0000000..37da1dd --- /dev/null +++ b/misc/nested_array_2.json @@ -0,0 +1,24 @@ +[ + { + "id": "481370", + "title": "Bigger", + "poster": "https://image.tmdb.org/t/p/w1280/f6rmiFTfT3Rf9XDrXfLO6XcAgWn.jpg", + "overview": "The inspirational tale of the grandfathers of the fitness movement as we now know it, Joe & Ben Weider. Battling anti-Semitism, racism and extreme poverty, the brothers beat all odds to build an empire & inspire future generations.", + "release_date": 1539392400, + "genre": [ + "Drama" + ] + }, + { + "id": "31357", + "title": "Waiting to Exhale", + "poster": "https://image.tmdb.org/t/p/w1280/4wjGMwPsdlvi025ZqR4rXnFDvBz.jpg", + "overview": "Cheated on, mistreated and stepped on, the women are holding their breath, waiting for the elusive 'good man' to break a string of less-than-stellar lovers. Friends and confidants Vannah, Bernie, Glo and Robin talk it all out, determined to find a better way to breathe.", + "release_date": 819590400, + "genre": [ + "Comedy", + "Drama", + "Romance" + ] + } +] diff --git a/misc/not_array.json b/misc/not_array.json index b980166..4a773be 100644 --- a/misc/not_array.json +++ b/misc/not_array.json @@ -1,3 +1,3 @@ { "name": "not an array" -} \ No newline at end of file +} diff --git a/misc/number.json b/misc/number.json index 58052e6..d62de05 100644 --- a/misc/number.json +++ b/misc/number.json @@ -2,4 +2,4 @@ 1, 2, 3 -] \ No newline at end of file +] diff --git a/misc/world.json b/misc/world.json index 480f2c2..4f08739 100644 --- a/misc/world.json +++ b/misc/world.json @@ -2,4 +2,4 @@ { "name": "world" }, { "name": "world" }, { "name": "world" } -] \ No newline at end of file +] diff --git a/package.json b/package.json index 4c961c7..b868577 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,12 @@ "name": "combine-json", "version": "1.0.0", "description": "", - "main": "src/combine-json.js", + "main": "src/index.js", "preferGlobal": true, - "bin": "./src/combine-json.js", + "bin": "./src/cli.js", "scripts": { "link": "npm link", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "./src/cli.js misc" }, "author": "", "license": "ISC", diff --git a/src/cli.js b/src/cli.js new file mode 100755 index 0000000..5c0f697 --- /dev/null +++ b/src/cli.js @@ -0,0 +1,18 @@ +#!/usr/bin/env node +const combineJson = require('./combine-json'); + +(async () => { + console.log(combineJson); + try { + if (process.argv.length === 2) { + console.log(chalk.red('Error: Missing path argument')); + } + else { + await combineJson(process.argv[2], process.argv[3]) + } + } + catch(e) { + console.error(e); + throw(e) + } +})() diff --git a/src/combine-json.js b/src/combine-json.js index ab0db3e..b662e94 100755 --- a/src/combine-json.js +++ b/src/combine-json.js @@ -1,30 +1,16 @@ -#!/usr/bin/env node const fs = require('fs'); const chalk = require('chalk'); const path = require('path') - -function outputPath(outputDir) { - let fileName = (process.argv[3]) ? process.argv[3] : 'combined.json'; - return `${outputDir}/${fileName}`; -} - -function createIfNotExist(file) { - if (fs.existsSync(file)) { - fs.writeFileSync(file, ""); - } -} - -function listOfJsonFiles(files){ - return files.reduce((acc, file)=>{ - if (path.extname(file)=== '.json') return [...acc, file]; - return acc; - }, []); -} +const { + inputFilesAndDir, + resolveOutputFilePath, + filterNonJson + } = require('./file_utils') function findLastBracket(filePath, fd, buffer, position) { let charRed = fs.readSync(fd, buffer, 0, 8, position); let array = [...buffer].map(char => String.fromCharCode(char)); - let bracket = array.indexOf("]"); + let bracket = array.indexOf("]"); if (charRed === 0) return null; if (bracket > -1) return position + bracket + 1; fs.closeSync(fd) @@ -40,13 +26,13 @@ function getLastBracket(filePath, fd, position) { function findFirstBracketType(fileFd, buffer, position) { let charRed = fs.readSync(fileFd, buffer, 0, 8) let array = [...buffer].map(char => String.fromCharCode(char)); - let curly = array.indexOf("{"); - let bracket = array.indexOf("["); + let curly = array.indexOf("{"); + let bracket = array.indexOf("["); if (charRed === 0) return null; if ((curly < bracket || bracket === -1) && curly > -1) return { type: "{", pos: position + curly - } + } if ((bracket < curly || curly === -1) && bracket > -1) return { type: "[", pos: position + bracket @@ -59,26 +45,29 @@ function getFirstBracketType(fd) { return findFirstBracketType(fd, buffer, 0); } -async function combineJson(files, dir, outputDir) { - - let outputFile = outputPath(outputDir); - let filePath = dir + ((dir[dir.length - 1] === '/') ? '' : '/') // add slash at the end of the dir if it is not there yet - createIfNotExist(outputFile) - - fs.writeFileSync(outputFile, "["); // start of new file - const jsonFiles = listOfJsonFiles(files); - const numberOfFiles = jsonFiles.length +async function combine({inputFiles, inputDirPath, outputFilePath}) { + fs.writeFileSync(outputFilePath, "["); // start of new file + const numberOfFiles = inputFiles.length + numberOfFiles.map(( fileName, index) => { + let inputFile = `${inputDirPath}${fileName}`; + + // open destination file for appending + const writeStreamPath = fs.createWriteStream(outputFilePath, { + flags: 'a' + }); + + let start = (isArray) ? firstBracketType.pos + 1 : firstBracketType.pos; + + }) for (let index = 0; index < numberOfFiles; index++) { - let file = jsonFiles[index]; - let inputFile = `${filePath}${file}`; - // console.log({ inputFile }); - - // let content = require(inputFile); - const fd = fs.openSync(`${filePath}${file}`); + let file = inputFiles[index]; + let inputFile = `${inputDirPath}${file}`; + + const fd = fs.openSync(`${inputDirPath}${file}`); let firstBracketType = getFirstBracketType(fd); let lastBracket = undefined; - - + + if (firstBracketType) { let isArray = firstBracketType.type === '['; if (isArray) { @@ -86,16 +75,16 @@ async function combineJson(files, dir, outputDir) { lastBracket = getLastBracket(inputFile, fd, stats.size - 8) - 2; } // open destination file for appending - var w = fs.createWriteStream(outputFile, { + var w = fs.createWriteStream(outputFilePath, { flags: 'a' }); // open source file for reading - let start = (isArray) ? firstBracketType.pos + 1 : firstBracketType.pos; + let start = (isArray) ? firstBracketType.pos + 1 : firstBracketType.pos; var r = fs.createReadStream(inputFile, { start, end: lastBracket }); - + r.pipe(w); const combineFiles = new Promise(function(resolve, reject) { w.on('close', function() { @@ -105,12 +94,12 @@ async function combineJson(files, dir, outputDir) { }); await combineFiles; - - let last = (index === numberOfFiles - 1); + + let last = (index === numberOfFiles - 1); if (!last) { let coma = path.resolve(__dirname, '../assets/coma') - let comaWrite = fs.createWriteStream(outputFile, { + let comaWrite = fs.createWriteStream(outputFilePath, { flags: 'a' }); let comaRead = fs.createReadStream(coma); @@ -124,7 +113,7 @@ async function combineJson(files, dir, outputDir) { await addComa } else { let closingBracket = path.resolve(__dirname, '../assets/closing_bracket'); - let closingBracketWrite = fs.createWriteStream(outputFile, { + let closingBracketWrite = fs.createWriteStream(outputFilePath, { flags: 'a' }); let closingBracketRead = fs.createReadStream(closingBracket); @@ -147,34 +136,17 @@ async function combineJson(files, dir, outputDir) { } -function determineDir(dir) { - dir = path.resolve(dir); - if (!fs.existsSync(dir)) { // test for Fully Qualified path - console.log(`Error: ${process.argv[2]} no such named directory`); - process.exit() +async function combineJson(inputDir, outputFile = undefined) { + try { + const { inputDirPath, filesName } = inputFilesAndDir({ inputDir }) + const outputFilePath = resolveOutputFilePath({ fileName: outputFile }) + const inputFiles = filterNonJson({ filesName }); + await combine({inputFiles, inputDirPath, outputFilePath}) + }catch(e) { + throw(e) } - return dir; } -(async () => { - console.log(); - - - if (process.argv.length === 2) { - console.log(chalk.red('Error: Missing path argument')); - } - else { - try { - let dir = determineDir(process.argv[2]) - console.log({ dir }); - let outputDir = process.cwd(); - let files = fs.readdirSync(dir) - - await combineJson(files, dir, outputDir) - }catch(e) { - throw(e) - } - } -})() +module.exports = combineJson diff --git a/src/file_utils.js b/src/file_utils.js new file mode 100644 index 0000000..3d697b1 --- /dev/null +++ b/src/file_utils.js @@ -0,0 +1,56 @@ +const path = require('path') +const fs = require('fs') + +function resolveDir({ dir }) { + dir = path.resolve(dir); + if (!fs.existsSync(dir)) { // test for Fully Qualified path + console.log(`Error: ${dir} no such named directory`); + process.exit() + } + return dir; +} + +function outputFile({ dirName, fileName = undefined }) { + fileName = fileName || 'combined.json'; + return `${dirName}/${fileName}`; +} + +function inputFilesAndDir({ inputDir }) { + const resolvedDir = resolveDir({ dir: inputDir}) + const inputDirPath = resolvedDir + ((resolvedDir[resolvedDir.length - 1] === '/') ? '' : '/') // add slash at the end of the dir if it is not there yet + const filesName = fs.readdirSync(inputDirPath) // read all files names in dir + return { + inputDirPath, + filesName + } +} + +function resolveOutputFilePath({ fileName }) { + const workingDir = process.cwd(); + const outputFilePath = outputFile({ dirName: workingDir, fileName}); + createFileIfNotExist(outputFilePath) + return outputFilePath +} + + +function createFileIfNotExist({ file }) { + if (fs.existsSync(file)) { + fs.writeFileSync(file, ""); + } +} + +function filterNonJson({ filesName }){ + return filesName.reduce((acc, file)=>{ + if (path.extname(file)=== '.json') return [...acc, file]; + return acc; + }, []); +} + +module.exports = { + resolveDir, + outputFile, + inputFilesAndDir, + resolveOutputFilePath, + createFileIfNotExist, + filterNonJson +} diff --git a/src/index.js b/src/index.js index e69de29..28666db 100644 --- a/src/index.js +++ b/src/index.js @@ -0,0 +1,4 @@ +const combineJson = require('./combine-json') + + +module.exports = combineJson diff --git a/src/slow-combine.js b/src/slow-combine.js index aa9d0ef..26e0474 100644 --- a/src/slow-combine.js +++ b/src/slow-combine.js @@ -23,31 +23,31 @@ function listOfJsonFiles(files){ async function combineJson(files, dir, outputDir) { - - let outputFile = outputPath(outputDir); + + let outputFile = outputPath(outputDir); let filePath = dir + ((dir[dir.length - 1] === '/') ? '' : '/') // add slash at the end of the dir if it is not there yet createIfNotExist(outputFile) - - + + fs.writeFileSync(outputFile, "["); // start of new file const jsonFiles = listOfJsonFiles(files); const numberOfFiles = jsonFiles.length - + for (let index = 0; index < numberOfFiles; index++) { let file = jsonFiles[index]; - - let inputFile = `${filePath}${file}`; + + let inputFile = `${filePath}${file}`; let content = require(inputFile); content = JSON.stringify(content); if (Array.isArray(content)) content = content.substr(1, content.length - 2); - let last = (index === numberOfFiles - 1); + let last = (index === numberOfFiles - 1); fs.appendFileSync(outputFile, `${content}${(last)? "" : ","}\n`); console.log(chalk.green( 'file: ' + chalk.blue.underline.bold(file) + ` has been added! last : ${last}, index: ${index}, numberOfFiles: ${numberOfFiles}` )) - + } fs.appendFileSync(outputFile, `]`); } @@ -65,14 +65,14 @@ function determineDir(dir) { (async () => { if (process.argv.length === 2) { console.log(chalk.red('Error: Missing path argument')); - } + } else { try { - let dir = determineDir(process.argv[2]) + let dir = determineDir(process.argv[2]) console.log({ dir }); let outputDir = process.cwd(); let files = fs.readdirSync(dir) - + await combineJson(files, dir, outputDir) }catch(e) { throw(e)