From 6b535f5d1c2a064dda0958aabe55da947dd769c3 Mon Sep 17 00:00:00 2001 From: Raymond Feng Date: Wed, 29 Jan 2014 12:04:09 -0800 Subject: [PATCH 1/2] Add a file option for the memeory connector to persist data --- .gitignore | 1 + lib/connector.js | 1 + lib/connectors/memory.js | 75 ++++++++++++++++++++++++++++------ test/memory.test.js | 88 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 test/memory.test.js diff --git a/.gitignore b/.gitignore index 8dd1e071..d56edcad 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs/html docs/man npm-debug.log .project +test/memory.json diff --git a/lib/connector.js b/lib/connector.js index 30acc763..efb3843d 100644 --- a/lib/connector.js +++ b/lib/connector.js @@ -98,6 +98,7 @@ Connector.prototype.defineProperty = function (model, propertyName, propertyDefi */ Connector.prototype.disconnect = function disconnect(cb) { // NO-OP + cb && process.nextTick(cb); }; /** diff --git a/lib/connectors/memory.js b/lib/connectors/memory.js index 72e0e125..549bf6f2 100644 --- a/lib/connectors/memory.js +++ b/lib/connectors/memory.js @@ -2,6 +2,8 @@ var util = require('util'); var Connector = require('../connector'); var geo = require('../geo'); var utils = require('../utils'); +var fs = require('fs'); +var async = require('async'); /** * Initialize the Oracle connector against the given data source @@ -10,24 +12,24 @@ var utils = require('../utils'); * @param {Function} [callback] The callback function */ exports.initialize = function initializeDataSource(dataSource, callback) { - dataSource.connector = new Memory(); + dataSource.connector = new Memory(null, dataSource.settings); dataSource.connector.connect(callback); }; exports.Memory = Memory; -function Memory(m) { - if (m) { +function Memory(m, settings) { + if (m instanceof Memory) { this.isTransaction = true; this.cache = m.cache; this.ids = m.ids; - this.constructor.super_.call(this, 'memory'); + this.constructor.super_.call(this, 'memory', settings); this._models = m._models; } else { this.isTransaction = false; this.cache = {}; this.ids = {}; - this.constructor.super_.call(this, 'memory'); + this.constructor.super_.call(this, 'memory', settings); } } @@ -36,11 +38,62 @@ util.inherits(Memory, Connector); Memory.prototype.connect = function (callback) { if (this.isTransaction) { this.onTransactionExec = callback; + } else { + this.loadFromFile(callback); + } +}; + +Memory.prototype.loadFromFile = function(callback) { + var self = this; + if (self.settings.file) { + fs.readFile(self.settings.file, {encoding: 'utf8', flag: 'r'}, function (err, data) { + if (err && err.code !== 'ENOENT') { + callback && callback(err); + } else { + if (data) { + data = JSON.parse(data.toString()); + self.ids = data.ids || {}; + self.cache = data.models || {}; + } + callback && callback(); + } + }); } else { process.nextTick(callback); } }; +/*! + * Flush the cache into the json file if necessary + * @param {Function} callback + */ +Memory.prototype.saveToFile = function (result, callback) { + var self = this; + if (this.settings.file) { + if(!self.writeQueue) { + // Create a queue for writes + self.writeQueue = async.queue(function (task, cb) { + // Flush out the models/ids + var data = JSON.stringify({ + ids: self.ids, + models: self.cache + }, null, ' '); + + fs.writeFile(self.settings.file, data, function (err) { + cb(err, result); + task && task(err, result); + }); + }, 1); + } + // Enqueue the write + self.writeQueue.push(callback); + } else { + process.nextTick(function () { + callback && callback(null, result); + }); + } +}; + Memory.prototype.define = function defineModel(definition) { this.constructor.super_.prototype.define.apply(this, [].slice.call(arguments)); var m = definition.model.modelName; @@ -69,9 +122,7 @@ Memory.prototype.create = function create(model, data, callback) { id = (props[idName] && props[idName].type && props[idName].type(id)) || id; this.setIdValue(model, data, id); this.cache[model][id] = JSON.stringify(data); - process.nextTick(function () { - callback(null, id); - }); + this.saveToFile(id, callback); }; Memory.prototype.updateOrCreate = function (model, data, callback) { @@ -90,9 +141,7 @@ Memory.prototype.updateOrCreate = function (model, data, callback) { Memory.prototype.save = function save(model, data, callback) { this.cache[model][this.getIdValue(model, data)] = JSON.stringify(data); - process.nextTick(function () { - callback(null, data); - }); + this.saveToFile(data, callback); }; Memory.prototype.exists = function exists(model, id, callback) { @@ -110,7 +159,7 @@ Memory.prototype.find = function find(model, id, callback) { Memory.prototype.destroy = function destroy(model, id, callback) { delete this.cache[model][id]; - process.nextTick(callback); + this.saveToFile(null, callback); }; Memory.prototype.fromDb = function (model, data) { @@ -273,7 +322,7 @@ Memory.prototype.destroyAll = function destroyAll(model, where, callback) { if (!where) { this.cache[model] = {}; } - process.nextTick(callback); + this.saveToFile(null, callback); }; Memory.prototype.count = function count(model, callback, where) { diff --git a/test/memory.test.js b/test/memory.test.js new file mode 100644 index 00000000..dcd8a163 --- /dev/null +++ b/test/memory.test.js @@ -0,0 +1,88 @@ +var jdb = require('../'); +var DataSource = jdb.DataSource; +var path = require('path'); +var fs = require('fs'); +var assert = require('assert'); +var async = require('async'); + +describe('Memory connector', function () { + var file = path.join(__dirname, 'memory.json'); + + function readModels(done) { + fs.readFile(file, function (err, data) { + var json = JSON.parse(data.toString()); + assert(json.models); + assert(json.ids.User); + done(err, json); + }); + } + + before(function (done) { + fs.unlink(file, function (err) { + if (!err || err.code === 'ENOENT') { + done(); + } + }); + }); + + it('should save to a json file', function (done) { + var ds = new DataSource({ + connector: 'memory', + file: file + }); + + var User = ds.createModel('User', { + name: String, + bio: String, + approved: Boolean, + joinedAt: Date, + age: Number + }); + + var count = 0; + var id = 1; + async.eachSeries(['John1', 'John2', 'John3'], function (item, cb) { + User.create({name: item}, function (err, result) { + id = result.id; + count++; + readModels(function (err, json) { + assert.equal(Object.keys(json.models.User).length, count); + cb(err); + }); + }); + }, function (err, results) { + // Now try to delete one + User.deleteById(id, function (err) { + readModels(function (err, json) { + assert.equal(Object.keys(json.models.User).length, 2); + done(); + }); + }); + }); + + }); + + // The saved memory.json from previous test should be loaded + it('should load from the json file', function (done) { + var ds = new DataSource({ + connector: 'memory', + file: file + }); + + var User = ds.createModel('User', { + name: String, + bio: String, + approved: Boolean, + joinedAt: Date, + age: Number + }); + + User.find(function (err, users) { + // There should be 2 records + assert.equal(users.length, 2); + done(err); + }); + + }); +}); + From 130dcdb5824cfd6bb8306417bdf581392a1134e3 Mon Sep 17 00:00:00 2001 From: Raymond Feng Date: Wed, 29 Jan 2014 13:41:42 -0800 Subject: [PATCH 2/2] Fix the write closure to use the correct task info --- lib/connectors/memory.js | 23 ++++++++++++++++++----- test/memory.test.js | 15 +++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/lib/connectors/memory.js b/lib/connectors/memory.js index 549bf6f2..7c337d43 100644 --- a/lib/connectors/memory.js +++ b/lib/connectors/memory.js @@ -54,6 +54,11 @@ Memory.prototype.loadFromFile = function(callback) { data = JSON.parse(data.toString()); self.ids = data.ids || {}; self.cache = data.models || {}; + } else { + if(!self.cache) { + self.ids = {}; + self.cache = {}; + } } callback && callback(); } @@ -80,13 +85,16 @@ Memory.prototype.saveToFile = function (result, callback) { }, null, ' '); fs.writeFile(self.settings.file, data, function (err) { - cb(err, result); - task && task(err, result); + cb(err); + task.callback && task.callback(err, task.data); }); }, 1); } // Enqueue the write - self.writeQueue.push(callback); + self.writeQueue.push({ + data: result, + callback: callback + }); } else { process.nextTick(function () { callback && callback(null, result); @@ -97,8 +105,10 @@ Memory.prototype.saveToFile = function (result, callback) { Memory.prototype.define = function defineModel(definition) { this.constructor.super_.prototype.define.apply(this, [].slice.call(arguments)); var m = definition.model.modelName; - this.cache[m] = {}; - this.ids[m] = 1; + if(!this.cache[m]) { + this.cache[m] = {}; + this.ids[m] = 1; + } }; Memory.prototype.create = function create(model, data, callback) { @@ -121,6 +131,9 @@ Memory.prototype.create = function create(model, data, callback) { var idName = this.idName(model); id = (props[idName] && props[idName].type && props[idName].type(id)) || id; this.setIdValue(model, data, id); + if(!this.cache[model]) { + this.cache[model] = {}; + } this.cache[model][id] = JSON.stringify(data); this.saveToFile(id, callback); }; diff --git a/test/memory.test.js b/test/memory.test.js index dcd8a163..4c613270 100644 --- a/test/memory.test.js +++ b/test/memory.test.js @@ -40,10 +40,10 @@ describe('Memory connector', function () { }); var count = 0; - var id = 1; + var ids = []; async.eachSeries(['John1', 'John2', 'John3'], function (item, cb) { User.create({name: item}, function (err, result) { - id = result.id; + ids.push(result.id); count++; readModels(function (err, json) { assert.equal(Object.keys(json.models.User).length, count); @@ -52,10 +52,17 @@ describe('Memory connector', function () { }); }, function (err, results) { // Now try to delete one - User.deleteById(id, function (err) { + User.deleteById(ids[0], function (err) { readModels(function (err, json) { assert.equal(Object.keys(json.models.User).length, 2); - done(); + User.upsert({id: ids[1], name: 'John'}, function(err, result) { + readModels(function (err, json) { + assert.equal(Object.keys(json.models.User).length, 2); + var user = JSON.parse(json.models.User[ids[1]]); + assert.equal(user.name, 'John'); + done(); + }); + }); }); }); });