diff --git a/lib/file-writers.js b/lib/file-writers.js index 6c5bc59..2bd88d7 100644 --- a/lib/file-writers.js +++ b/lib/file-writers.js @@ -1,24 +1,20 @@ -const fs = require('fs'); -const util = require('util'); const yaml = require('yaml'); -const writeFile = util.promisify(fs.writeFile); - -module.exports.writeFrontmatterMarkdown = (filePath, { body = '', frontmatter = {} }) => { +module.exports.writeFrontmatterMarkdown = ({ body = '', frontmatter = {} }) => { const lines = ['---', yaml.stringify(frontmatter).trim(), '---', body ? body.toString().trim() : '', '']; const content = lines.join('\n'); - return writeFile(filePath, content); + return content; }; -module.exports.writeJSON = (filePath, data) => { +module.exports.writeJSON = data => { const content = JSON.stringify(data, null, 2); - return writeFile(filePath, content); + return content; }; -module.exports.writeYAML = function(filePath, data) { +module.exports.writeYAML = function(data) { const content = yaml.stringify(data); - return writeFile(filePath, content); + return content; }; diff --git a/lib/file-writers.test.js b/lib/file-writers.test.js index ef5c5df..4ed12dc 100644 --- a/lib/file-writers.test.js +++ b/lib/file-writers.test.js @@ -20,15 +20,8 @@ describe('`writeFrontmatterMarkdown()`', () => { name: 'John Doe' } }; - const mockPath = '/Users/johndoe/file.md'; + const content = fileWriters.writeFrontmatterMarkdown(mockContent); - fileWriters.writeFrontmatterMarkdown(mockPath, mockContent); - - expect(mockPromisifiedFunction).toHaveBeenCalledTimes(1); - - const [filePath, content] = mockPromisifiedFunction.mock.calls[0]; - - expect(filePath).toBe(mockPath); expect(content).toBe(`---\nname: ${mockContent.frontmatter.name}\n---\n${mockContent.body}\n`); }); @@ -39,15 +32,8 @@ describe('`writeFrontmatterMarkdown()`', () => { name: 'John Doe' } }; - const mockPath = '/Users/johndoe/file.md'; - - fileWriters.writeFrontmatterMarkdown(mockPath, mockContent); - - expect(mockPromisifiedFunction).toHaveBeenCalledTimes(1); - - const [filePath, content] = mockPromisifiedFunction.mock.calls[0]; + const content = fileWriters.writeFrontmatterMarkdown(mockContent); - expect(filePath).toBe(mockPath); expect(content).toBe('---\nname: John Doe\n---\n1,2,3\n'); }); }); diff --git a/lib/sourcebit.js b/lib/sourcebit.js index 86a9949..9632506 100644 --- a/lib/sourcebit.js +++ b/lib/sourcebit.js @@ -1,22 +1,27 @@ +const { cloneDeep } = require('lodash'); const debug = require('debug'); +const { diff: generateDiff } = require('deep-diff'); const fs = require('fs'); const mkdirp = require('mkdirp'); const ora = require('ora'); const path = require('path'); -const { cloneDeep } = require('lodash'); +const util = require('util'); + const { writeFrontmatterMarkdown, writeJSON, writeYAML } = require('./file-writers'); -const FILE_WRITERS = { +const fileWriters = { 'frontmatter-md': writeFrontmatterMarkdown, json: writeJSON, yml: writeYAML }; +const writeFile = util.promisify(fs.writeFile); class Sourcebit { constructor({ cacheFile = path.join(process.cwd(), '.sourcebit-cache.json'), runtimeParameters = {}, transformCallback } = {}) { this.cacheFilePath = cacheFile; this.context = {}; - this.fileWriterCache = []; + this.dataForPlugin = []; + this.fileWriterCache = new Map(); this.onTransform = transformCallback; this.pluginBlocks = []; this.pluginModules = {}; @@ -206,7 +211,7 @@ class Sourcebit { return queue; } - return queue.then(data => { + return queue.then(async data => { const plugin = this.pluginModules[index]; const pluginName = this.getNameOfPluginAtIndex(index); @@ -226,13 +231,29 @@ class Sourcebit { return data; } - return plugin.transform({ - data, + const { __diff, ...currentDataForPlugin } = data; + const previousData = this.dataForPlugin[index] || initialData; + const diffs = Object.keys(currentDataForPlugin).reduce((diffs, dataBucketKey) => { + return { + ...diffs, + [dataBucketKey]: generateDiff(previousData[dataBucketKey], currentDataForPlugin[dataBucketKey]) || [] + }; + }, {}); + + this.dataForPlugin[index] = currentDataForPlugin; + + const transformedData = await plugin.transform({ + data: { + ...data, + __diff: diffs + }, debug: this.getDebugMethodForPlugin(pluginName), getPluginContext: () => contextSnapshot[pluginName] || {}, log: this.logFromPlugin.bind(this), options: this.parsePluginOptions(plugin, pluginBlock.options) }); + + return transformedData; }); }, Promise.resolve(initialData)); @@ -305,11 +326,13 @@ class Sourcebit { // We start by deleting any files that were previously created by this plugin // but that are not part of the site after the update. - this.fileWriterCache.forEach(filePath => { + this.fileWriterCache.forEach((_, filePath) => { if (!filesByPath[filePath]) { try { fs.unlinkSync(filePath); + this.fileWriterCache.delete(filePath); + this.log(`Deleted ${filePath}`, 'info'); } catch (error) { this.debug(error); @@ -318,12 +341,10 @@ class Sourcebit { } }); - this.fileWriterCache = Object.keys(filesByPath); - // Now we write all the files that need to be created. const queue = Object.keys(filesByPath).map(async filePath => { const file = filesByPath[filePath]; - const writerFunction = FILE_WRITERS[file.format]; + const writerFunction = fileWriters[file.format]; if (typeof writerFunction !== 'function') { this.log(`Could not create ${filePath}. "${file.format}" is not a supported format.`, 'fail'); @@ -335,9 +356,21 @@ class Sourcebit { mkdirp.sync(path.dirname(filePath)); try { - await writerFunction(filePath, file.content); + const fileContent = await writerFunction(file.content); + const hasDiff = this.fileWriterCache.get(filePath) !== fileContent; + + // If the contents of the file hasn't changed since we last wrote it, we skip it. + if (!hasDiff) { + return true; + } + + writeFile(filePath, fileContent); + + const isNewFile = Boolean(this.fileWriterCache.get(filePath)); + + this.fileWriterCache.set(filePath, fileContent); - this.log(`Created ${filePath}`, 'succeed'); + this.log(`${isNewFile ? 'Updated' : 'Created'} ${filePath}`, 'succeed'); return true; } catch (error) { diff --git a/lib/sourcebit.test.js b/lib/sourcebit.test.js index a740693..30fab38 100644 --- a/lib/sourcebit.test.js +++ b/lib/sourcebit.test.js @@ -1432,6 +1432,93 @@ describe('writing files', () => { jest.useFakeTimers(); }); + test('does not re-write files whose contents have not changed', async () => { + jest.useRealTimers(); + + const elements = ['Hydrogen', 'Lithium', 'Sodium', 'Potassium', 'Rubidium', 'Caesium', 'Francium']; + const mockContent = [ + { + groups: [{ elements: elements.slice(0, 2) }, { elements: elements.slice(2, 4) }] + }, + { + groups: [{ elements: elements.slice(0, 2) }, { elements: elements.slice(3, 5) }] + } + ]; + const mockPaths = ['/Users/foo/bar/table1.json', '/Users/foo/bar/table2.json']; + const plugins = [ + { + module: { + name: 'sourcebit-test1', + bootstrap: ({ refresh, setPluginContext }) => { + setPluginContext(mockContent[0]); + + setTimeout(() => { + setPluginContext(mockContent[1]); + + refresh(); + }, 100); + }, + transform: ({ data, getPluginContext }) => { + const { groups } = getPluginContext(); + const files = groups.map((group, index) => ({ + path: mockPaths[index], + format: 'json', + content: group + })); + + return { + ...data, + files: data.files.concat(files) + }; + } + } + } + ]; + + const sourcebit = new Sourcebit({ + watch: true + }); + const callback = jest.fn(); + const logFn = jest.spyOn(sourcebit, 'log'); + const fsWriteFileFn = jest.spyOn(fs, 'writeFile'); + + sourcebit.loadPlugins(plugins); + sourcebit.onTransform = callback; + + await sourcebit.bootstrapAll(); + await sourcebit.transform(); + + await new Promise(resolve => { + setTimeout(() => { + expect(callback).toHaveBeenCalledTimes(2); + expect(mockPromisifiedFunction).toHaveBeenCalledTimes(3); + + const [firstCall, secondCall, thirdCall] = mockPromisifiedFunction.mock.calls; + + expect(firstCall[0]).toBe(mockPaths[0]); + expect(firstCall[1]).toBe(JSON.stringify(mockContent[0].groups[0], null, 2)); + + expect(secondCall[0]).toBe(mockPaths[1]); + expect(secondCall[1]).toBe(JSON.stringify(mockContent[0].groups[1], null, 2)); + + expect(thirdCall[0]).toBe(mockPaths[1]); + expect(thirdCall[1]).toBe(JSON.stringify(mockContent[1].groups[1], null, 2)); + + expect(logFn).toHaveBeenCalledTimes(3); + expect(logFn.mock.calls[0][0]).toBe(`Created ${mockPaths[0]}`); + expect(logFn.mock.calls[0][1]).toBe('succeed'); + expect(logFn.mock.calls[1][0]).toBe(`Created ${mockPaths[1]}`); + expect(logFn.mock.calls[1][1]).toBe('succeed'); + expect(logFn.mock.calls[2][0]).toBe(`Updated ${mockPaths[1]}`); + expect(logFn.mock.calls[2][1]).toBe('succeed'); + + resolve(); + }, 150); + }); + + jest.useFakeTimers(); + }); + test('gracefully handles an error when deleting any files previously created that are no longer part of the `files` data bucket', async () => { jest.useRealTimers(); diff --git a/package-lock.json b/package-lock.json index f81d666..0b30fc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1641,6 +1641,11 @@ "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=", "dev": true }, + "deep-diff": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/deep-diff/-/deep-diff-1.0.2.tgz", + "integrity": "sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==" + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", diff --git a/package.json b/package.json index d05f986..a98d347 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dependencies": { "commander": "^4.1.1", "debug": "^4.1.1", + "deep-diff": "^1.0.2", "dotenv": "^8.2.0", "lodash": "^4.17.15", "mkdirp": "^1.0.3",