diff --git a/file_system_cache/file_system_cache.js b/file_system_cache/file_system_cache.js new file mode 100644 index 0000000..9210578 --- /dev/null +++ b/file_system_cache/file_system_cache.js @@ -0,0 +1,183 @@ +"use strict"; + +const fs = require('fs'); +const Promise = require('bluebird'); +const {stringToMurmur} = require('../utils/hash'); +const {LazyPromise} = require('../utils/lazy_promise'); + +const defaultReadFile = Promise.promisify(fs.readFile); +const defaultStat = Promise.promisify(fs.stat); + +class StaleFileIntercept extends Error { + constructor(path, job) { + super(); + this.message = `File "${path}" was intercepted for job "${job}" as the reference was invalidated during processing`; + } +} + +module.exports = { + createFileSystemCache, + createFileObject, + createFileSystemObject +}; + +function createFileSystemCache(fileSystemOverrides={}) { + const fileSystem = createFileSystemObject(fileSystemOverrides); + + let files = Object.create(null); + + function ensureFileIsValid(file, job) { + if (files[file.path] !== file) { + throw new StaleFileIntercept(file.path, job); + } + } + + function getOrCreateFile(path) { + let file = files[path]; + if (!file) { + file = createFileObject(path, fileSystem); + files[path] = file; + } + return file; + } + + function evaluateFileDataProperty(property, path) { + const file = getOrCreateFile(path); + return file[property] + .catch(err => { + ensureFileIsValid(file, property); + return Promise.reject(err); + }) + .then(data => { + ensureFileIsValid(file, property); + return data; + }); + } + + function stat(path) { + return evaluateFileDataProperty('stat', path) + } + function readFileModifiedTime(path) { + return evaluateFileDataProperty('modifiedTime', path) + } + function isFile(path) { + return evaluateFileDataProperty('isFile', path) + } + function readTextFile(path) { + return evaluateFileDataProperty('text', path) + } + function readTextHash(path) { + return evaluateFileDataProperty('textHash', path) + } + + function invalidateFile(path) { + files[path] = null; + } + + function createContext() { + const dependencies = {}; + + function getDependency(path) { + let dependency = dependencies[path]; + if (!dependency) { + dependency = {}; + dependencies[path] = dependency; + } + return dependency; + } + + return { + describeDependencies() { + return dependencies; + }, + stat(path) { + return Promise.all([ + readFileModifiedTime(path), + stat(path) + ]) + .then(data => { + getDependency(path).modifiedTime = data[0]; + return data[1]; + }); + }, + readFileModifiedTime(path) { + return readFileModifiedTime(path) + .then(modifiedTime => { + getDependency(path).modifiedTime = modifiedTime; + return modifiedTime; + }) + }, + isFile(path) { + return isFile(path) + .then(isFile => { + getDependency(path).isFile = isFile; + return isFile; + }); + }, + readTextFile(path) { + return Promise.all([ + readTextHash(path), + readTextFile(path) + ]) + .then(data => { + getDependency(path).textHash = data[0]; + return data[1]; + }); + }, + readTextHash(path) { + return readTextHash(path) + .then(textHash => { + getDependency(path).textHash = textHash; + return textHash; + }); + } + }; + } + + return { + StaleFileIntercept, + invalidateFile, + createContext, + stat, + readFileModifiedTime, + isFile, + readTextFile, + readTextHash + }; +} + +function createFileObject(path, fileSystem) { + // Lazily-evaluated interactions with the file system + const stat = new LazyPromise(() => fileSystem.stat(path)); + const text = new LazyPromise(() => fileSystem.readFile(path, 'utf8')); + + // Data derived from the file system interactions + const modifiedTime = new LazyPromise(() => stat.then(stat => stat.mtime.getTime())); + const isFile = new LazyPromise(() => { + return stat + .then(stat => stat.isFile()) + .catch(err => { + if (err.code === 'ENOENT') { + return false; + } + return Promise.reject(err); + }); + }); + const textHash = new LazyPromise(() => text.then(stringToMurmur)); + + return { + path, + stat, + modifiedTime, + isFile, + text, + textHash + }; +} + +function createFileSystemObject(overrides) { + return Object.assign({ + readFile: defaultReadFile, + stat: defaultStat + }, overrides); +} \ No newline at end of file diff --git a/file_system_cache/tests/test_file_system_cache.js b/file_system_cache/tests/test_file_system_cache.js new file mode 100644 index 0000000..ff51cd9 --- /dev/null +++ b/file_system_cache/tests/test_file_system_cache.js @@ -0,0 +1,235 @@ +"use strict"; + +const fs = require('fs'); +const {assert} = require('../../utils/assert'); +const {stringToMurmur} = require('../../utils/hash'); +const {createFileSystemCache, createFileObject, createFileSystemObject} = require('../file_system_cache'); + +describe('file_system_cache/file_system_cache', () => { + describe('#createFileSystemCache', () => { + it('should produce the expected dataset of a file', () => { + const fsCache = createFileSystemCache(); + return Promise.all([ + fsCache.stat(__filename), + fsCache.readFileModifiedTime(__filename), + fsCache.isFile(__filename), + fsCache.readTextFile(__filename), + fsCache.readTextHash(__filename) + ]).then(([stat, modifiedTime, isFile, text, textHash]) => { + const actualText = fs.readFileSync(__filename, 'utf8'); + const actualStat = fs.statSync(__filename); + assert.equal(stat.mtime.getTime(), actualStat.mtime.getTime()); + assert.equal(modifiedTime, actualStat.mtime.getTime()); + assert.equal(isFile, true); + assert.equal(text, actualText); + assert.equal(textHash, stringToMurmur(actualText)); + }); + }); + it('should only hit the filesystem once for a particular job on a file', () => { + let called = false; + function readFile() { + if (called) { + throw new Error('should not be called twice'); + } + called = true; + return Promise.resolve('text'); + } + + const fsCache = createFileSystemCache({readFile}); + + return Promise.all([ + fsCache.readTextFile('/some/file.js'), + fsCache.readTextFile('/some/file.js') + ]) + .then(([text1, text2]) => { + assert.equal(text1, 'text'); + assert.equal(text2, 'text'); + return fsCache.readTextFile('/some/file.js') + .then(text => assert.equal(text, 'text')); + }); + }); + it('should handle multiple concurrent file requests', () => { + function readFile(path) { + if (path === 'test 1') { + return Promise.resolve('text 1'); + } + if (path === 'test 2') { + return Promise.resolve('text 2'); + } + throw new Error('should not reach this'); + } + function stat(path) { + if (path === 'test 1') { + return Promise.resolve('stat 1'); + } + if (path === 'test 2') { + return Promise.resolve('stat 2'); + } + throw new Error('should not reach this'); + } + + const fsCache = createFileSystemCache({readFile, stat}); + + return Promise.all([ + fsCache.readTextFile('test 1'), + fsCache.stat('test 1'), + fsCache.readTextFile('test 2'), + fsCache.stat('test 2') + ]) + .then(([text1, stat1, text2, stat2]) => { + assert.equal(text1, 'text 1'); + assert.equal(stat1, 'stat 1'); + assert.equal(text2, 'text 2'); + assert.equal(stat2, 'stat 2'); + }); + }); + it('should intercept jobs for files that are invalidated during processing', () => { + const fsCache = createFileSystemCache(); + const job = fsCache.stat(__filename) + .then(() => { + throw new Error('should not be reached'); + }) + .catch(err => { + assert.instanceOf(err, fsCache.StaleFileIntercept); + return 'done'; + }); + fsCache.invalidateFile(__filename); + return assert.becomes(job, 'done'); + }); + it('should enable contexts to be applied that indicate the nature of a file dependency', () => { + const fsCache = createFileSystemCache(); + const context = fsCache.createContext(); + return Promise.all([ + context.stat(__filename), + context.readFileModifiedTime(__filename), + context.isFile(__filename), + context.readTextFile(__filename), + context.readTextHash(__filename) + ]).then(([stat, modifiedTime, isFile, text, textHash]) => { + const actualText = fs.readFileSync(__filename, 'utf8'); + const actualStat = fs.statSync(__filename); + assert.equal(stat.mtime.getTime(), actualStat.mtime.getTime()); + assert.equal(modifiedTime, actualStat.mtime.getTime()); + assert.equal(isFile, true); + assert.equal(text, actualText); + assert.equal(textHash, stringToMurmur(actualText)); + + assert.deepEqual( + context.describeDependencies(), + { + [__filename]: { + isFile: true, + modifiedTime: modifiedTime, + textHash: textHash + } + } + ); + }); + }); + it('should enable contexts to be applied that indicate multiple files dependencies', () => { + const fsCache = createFileSystemCache(); + const context = fsCache.createContext(); + return Promise.all([ + context.isFile(__filename), + context.isFile('__NON_EXISTENT_FILE_1__'), + context.isFile('__NON_EXISTENT_FILE_2__') + ]).then(([isFile1, isFile2, isFile3]) => { + assert.equal(isFile1, true); + assert.equal(isFile2, false); + assert.equal(isFile3, false); + + assert.deepEqual( + context.describeDependencies(), + { + [__filename]: { + isFile: true + }, + __NON_EXISTENT_FILE_1__: { + isFile: false + }, + __NON_EXISTENT_FILE_2__: { + isFile: false + } + } + ); + }); + }); + }); + describe('#createFileObject', () => { + it('should produce the expected dataset of a file', () => { + const fileSystem = createFileSystemObject(); + const file = createFileObject(__filename, fileSystem); + assert.equal(file.path, __filename); + return Promise.all([ + file.stat, + file.modifiedTime, + file.isFile, + file.text, + file.textHash + ]).then(([stat, modifiedTime, isFile, text, textHash]) => { + const actualText = fs.readFileSync(__filename, 'utf8'); + const actualStat = fs.statSync(__filename); + assert.equal(stat.mtime.getTime(), actualStat.mtime.getTime()); + assert.equal(modifiedTime, actualStat.mtime.getTime()); + assert.equal(isFile, true); + assert.equal(text, actualText); + assert.equal(textHash, stringToMurmur(actualText)); + }) + }); + it('should indicate if a file does not exist', () => { + const fileSystem = createFileSystemObject(); + const file = createFileObject('___NON_EXISTENT_FILE__', fileSystem); + return file.isFile + .then(isFile => { + assert.equal(isFile, false); + }); + }); + describe('file system interactions', () => { + it('should create an object that lazily evaluates text files and preserves the value', () => { + let called = false; + function readFile(path, encoding) { + if (called) { + throw new Error('should not be called twice'); + } + called = true; + assert.equal(path, '/some/file'); + assert.equal(encoding, 'utf8'); + return Promise.resolve('text'); + } + const file = createFileObject('/some/file', {readFile}); + return assert.isFulfilled( + file.text + .then(text => { + assert.equal(text, 'text'); + return file.text + .then(text => { + assert.equal(text, 'text'); + }); + }) + ); + }); + it('should create an object that lazily evaluates file stats and preserves the value', () => { + let called = false; + function stat(path) { + if (called) { + throw new Error('should not be called twice'); + } + called = true; + assert.equal(path, '/some/file'); + return Promise.resolve('stat'); + } + const file = createFileObject('/some/file', {stat}); + return assert.isFulfilled( + file.stat + .then(text => { + assert.equal(text, 'stat'); + return file.stat + .then(text => { + assert.equal(text, 'stat'); + }); + }) + ); + }); + }); + }); +}); \ No newline at end of file diff --git a/utils/lazy_promise.js b/utils/lazy_promise.js new file mode 100644 index 0000000..fb49e24 --- /dev/null +++ b/utils/lazy_promise.js @@ -0,0 +1,50 @@ +'use strict'; + +const Promise = require('bluebird'); + +/** + * Hacky implementation of a lazily executed Promise. + * + * Waits until the first `then` or `catch` call before + * evaluating the function provided and wrapping the + * returned (or thrown) value in a promise. + * + * Note that execution is synchronous, but the promise + * will resolve asynchronously. This is an implementation + * detail that may change at a future point + */ +class LazyPromise { + constructor(evaluationFunction) { + this._evaluated = null; + this._evaluationFunction = evaluationFunction; + } + then(val) { + if (this._evaluated === null) { + this._evaluateFunction(); + } + return this._evaluated.then(val); + } + catch(val) { + if (this._evaluated === null) { + this._evaluateFunction(); + } + return this._evaluated.catch(val); + } + _evaluateFunction() { + const func = this._evaluationFunction; + // Ensure memory reclamation by clearing any references + this._evaluationFunction = null; + let val; + try { + val = func(); + } catch(err) { + this._evaluated = Promise.reject(err); + return; + } + this._evaluated = Promise.resolve(val); + } +} + +module.exports = { + LazyPromise +}; \ No newline at end of file diff --git a/utils/tests/test_lazy_promise.js b/utils/tests/test_lazy_promise.js new file mode 100644 index 0000000..c4f4077 --- /dev/null +++ b/utils/tests/test_lazy_promise.js @@ -0,0 +1,43 @@ +'use strict'; + +const {assert} = require('../assert'); +const {LazyPromise} = require('../lazy_promise'); + +describe('utils/lazy_promise', () => { + describe('#LazyPromise', () => { + it('should not evaluate until `.then` is called', () => { + let evaluated = false; + const promise = new LazyPromise(() => { + evaluated = true; + return 'test'; + }); + assert.isFalse(evaluated); + return assert.isFulfilled( + Promise.resolve().then(() => { + assert.isFalse(evaluated); + return promise.then(val => { + assert.isTrue(evaluated); + assert.equal(val, 'test'); + }) + }) + ); + }); + it('should not evaluate until `.catch` is called', () => { + let evaluated = false; + const promise = new LazyPromise(() => { + evaluated = true; + throw 'test'; + }); + assert.isFalse(evaluated); + return assert.isFulfilled( + Promise.resolve().then(() => { + assert.isFalse(evaluated); + return promise.catch(err => { + assert.isTrue(evaluated); + assert.equal(err, 'test'); + }); + }) + ); + }); + }); +}); \ No newline at end of file