Load datasources and app cfg from multiple files

Modify loading of `appConfig` and `dataSourceConfig` to look for
the following files:

  - app.json
  - app.local.{js|json}
  - app.{$env}.{js|json}

  - datasources.json
  - datasources.local.{js|json}
  - datasources.{$env}.{js|json}

where $env is the value of `app.get('env')`, which usually defaults
to `process.env.NODE_ENV`.

The values in the additional files are applied to the config object,
overwritting any existing values. The new values must be value types
like String or Number; Object and Array are not supported.

Additional datasource config files cannot define new datasources,
only modify existing ones.

The commit includes refactoring of the config-loading code into
a standalone file.

.gitignore

@@ -13,3 +13,4 @@
 node_modules
 checkstyle.xml
 loopback-boot-*.tgz
+/test/sandbox/

index.js

@@ -2,6 +2,7 @@ var assert = require('assert');
 var fs = require('fs');
 var path = require('path');
 var _ = require('underscore');
+var ConfigLoader = require('./lib/config-loader');
 
 /**
  * Initialize an application from an options object or
@@ -103,7 +104,7 @@ var _ = require('underscore');
  * @header boot(app, [options])
  */
 
-module.exports = function bootLoopBackApp(app, options) {
+exports = module.exports = function bootLoopBackApp(app, options) {
   /*jshint camelcase:false */
   options = options || {};
 
@@ -111,19 +112,12 @@ module.exports = function bootLoopBackApp(app, options) {
     options = { appRootDir: options };
   }
   var appRootDir = options.appRootDir = options.appRootDir || process.cwd();
-  var appConfig = options.app;
+  var env = app.get('env');
-  var modelConfig = options.models;
-  var dataSourceConfig = options.dataSources;
 
-  if(!appConfig) {
+  var appConfig = options.app || ConfigLoader.loadAppConfig(appRootDir, env);
-    appConfig = tryReadConfig(appRootDir, 'app') || {};
+  var modelConfig = options.models || ConfigLoader.loadModels(appRootDir, env);
-  }
+  var dataSourceConfig = options.dataSources ||
-  if(!modelConfig) {
+    ConfigLoader.loadDataSources(appRootDir, env);
-    modelConfig = tryReadConfig(appRootDir, 'models') || {};
-  }
-  if(!dataSourceConfig) {
-    dataSourceConfig = tryReadConfig(appRootDir, 'datasources') || {};
-  }
 
   assertIsValidConfig('app', appConfig);
   assertIsValidConfig('model', modelConfig);
@@ -298,12 +292,4 @@ function tryReadDir() {
   }
 }
 
-function tryReadConfig(cwd, fileName) {
+exports.ConfigLoader = ConfigLoader;
-  try {
-    return require(path.join(cwd, fileName + '.json'));
-  } catch(e) {
-    if(e.code !== 'MODULE_NOT_FOUND') {
-      throw e;
-    }
-  }
-}

lib/config-loader.js

@@ -0,0 +1,154 @@
+var fs = require('fs');
+var path = require('path');
+
+var ConfigLoader = exports;
+
+/**
+ * Load application config from `app.json` and friends.
+ * @param {String} rootDir Directory where to look for files.
+ * @param {String} env Environment, usually `process.env.NODE_ENV`
+ * @returns {Object}
+ */
+ConfigLoader.loadAppConfig = function(rootDir, env) {
+  return loadNamed(rootDir, env, 'app', mergeAppConfig);
+};
+
+/**
+ * Load data-sources config from `datasources.json` and friends.
+ * @param {String} rootDir Directory where to look for files.
+ * @param {String} env Environment, usually `process.env.NODE_ENV`
+ * @returns {Object}
+ */
+ConfigLoader.loadDataSources = function(rootDir, env) {
+  return loadNamed(rootDir, env, 'datasources', mergeDataSourceConfig);
+};
+
+/**
+ * Load models config from `models.json` and friends.
+ * @param {String} rootDir Directory where to look for files.
+ * @param {String} env Environment, usually `process.env.NODE_ENV`
+ * @returns {Object}
+ */
+ConfigLoader.loadModels = function(rootDir, env) {
+  /*jshint unused:false */
+  return tryReadJsonConfig(rootDir, 'models') || {};
+};
+
+/*-- Implementation --*/
+
+/**
+ * Load named configuration.
+ * @param {String} rootDir Directory where to look for files.
+ * @param {String} env Environment, usually `process.env.NODE_ENV`
+ * @param {String} name
+ * @param {function(target:Object, config:Object, filename:String)} mergeFn
+ * @returns {Object}
+ */
+function loadNamed(rootDir, env, name, mergeFn) {
+  var files = findConfigFiles(rootDir, env, name);
+  var configs = loadConfigFiles(files);
+  return mergeConfigurations(configs, mergeFn);
+}
+
+/**
+ * Search `appRootDir` for all files containing configuration for `name`.
+ * @param {String} appRootDir
+ * @param {String} env Environment, usually `process.env.NODE_ENV`
+ * @param {String} name
+ * @returns {Array.<String>} Array of absolute file paths.
+ */
+function findConfigFiles(appRootDir, env, name) {
+  var master = ifExists(name + '.json');
+  if (!master) return [];
+
+  var candidates = [
+    master,
+    ifExistsWithAnyExt(name + '.local'),
+    ifExistsWithAnyExt(name + '.' + env)
+  ];
+
+  return candidates.filter(function(c) { return c !== undefined; });
+
+  function ifExists(fileName) {
+    var filepath = path.resolve(appRootDir, fileName);
+    return fs.existsSync(filepath) ? filepath : undefined;
+  }
+
+  function ifExistsWithAnyExt(fileName) {
+    return ifExists(fileName + '.js') || ifExists(fileName + '.json');
+  }
+}
+
+/**
+ * Load configuration files into an array of objects.
+ * Attach non-enumerable `_filename` property to each object.
+ * @param {Array.<String>} files
+ * @returns {Array.<Object>}
+ */
+function loadConfigFiles(files) {
+  return files.map(function(f) {
+    var config = require(f);
+    Object.defineProperty(config, '_filename', {
+      enumerable: false,
+      value: f
+    });
+    return config;
+  });
+}
+
+/**
+ * Merge multiple configuration objects into a single one.
+ * @param {Array.<Object>} configObjects
+ * @param {function(target:Object, config:Object, filename:String)} mergeFn
+ */
+function mergeConfigurations(configObjects, mergeFn) {
+  var result = configObjects.shift() || {};
+  while(configObjects.length) {
+    var next = configObjects.shift();
+    mergeFn(result, next, next._filename);
+  }
+  return result;
+}
+
+function mergeDataSourceConfig(target, config, fileName) {
+  for (var ds in target) {
+    var err = applyCustomConfig(target[ds], config[ds]);
+    if (err) {
+      throw new Error('Cannot apply ' + fileName + ' to `'  + ds + '`: ' + err);
+    }
+  }
+}
+
+function mergeAppConfig(target, config, fileName) {
+  var err = applyCustomConfig(target, config);
+  if (err) {
+    throw new Error('Cannot apply ' + fileName + ': ' + err);
+  }
+}
+
+function applyCustomConfig(target, config) {
+  for (var key in config) {
+    var value = config[key];
+    if (typeof value === 'object') {
+      return 'override for the option `' + key + '` is not a value type.';
+    }
+    target[key] = value;
+  }
+  return null; // no error
+}
+
+/**
+ * Try to read a config file with .json extension
+ * @param cwd Dirname of the file
+ * @param fileName Name of the file without extension
+ * @returns {Object|undefined} Content of the file, undefined if not found.
+ */
+function tryReadJsonConfig(cwd, fileName) {
+  try {
+    return require(path.join(cwd, fileName + '.json'));
+  } catch(e) {
+    if(e.code !== 'MODULE_NOT_FOUND') {
+      throw e;
+    }
+  }
+}

package.json

@@ -28,7 +28,8 @@
     "loopback": "^1.5.0",
     "mocha": "^1.19.0",
     "must": "^0.11.0",
-    "supertest": "^0.13.0"
+    "supertest": "^0.13.0",
+    "fs-extra": "^0.9.1"
   },
   "peerDependencies": {
     "loopback": "1.x"

test/boot.test.js

@@ -1,12 +1,30 @@
 var boot = require('../');
+var fs = require('fs-extra');
+var extend = require('util')._extend;
 var path = require('path');
 var loopback = require('loopback');
 var assert = require('assert');
 var expect = require('must');
+var sandbox = require('./helpers/sandbox');
 
 var SIMPLE_APP = path.join(__dirname, 'fixtures', 'simple-app');
 
+var appDir;
+
 describe('bootLoopBackApp', function() {
+  beforeEach(sandbox.reset);
+
+  beforeEach(function makeUniqueAppDir(done) {
+    // Node's module loader has a very aggressive caching, therefore
+    // we can't reuse the same path for multiple tests
+    // The code here is used to generate a random string
+    require('crypto').randomBytes(5, function(ex, buf) {
+      var randomStr = buf.toString('hex');
+      appDir = sandbox.resolve(randomStr);
+      done();
+    });
+  });
+
   describe('from options', function () {
     var app;
     beforeEach(function () {
@@ -186,6 +204,102 @@ describe('bootLoopBackApp', function() {
       assert.isFunc(app.models.Foo, 'find');
       assert.isFunc(app.models.Foo, 'create');
     });
+
+    it('merges datasource configs from multiple files', function() {
+      givenAppInSandbox();
+
+      writeAppConfigFile('datasources.local.json', {
+        db: { local: 'applied' }
+      });
+
+      var env = process.env.NODE_ENV || 'development';
+      writeAppConfigFile('datasources.' + env + '.json', {
+        db: { env: 'applied' }
+      });
+
+      var app = loopback();
+      boot(app, appDir);
+
+      var db = app.datasources.db.settings;
+      expect(db).to.have.property('local', 'applied');
+      expect(db).to.have.property('env', 'applied');
+
+      var expectedLoadOrder = ['local', 'env'];
+      var actualLoadOrder = Object.keys(db).filter(function(k) {
+        return expectedLoadOrder.indexOf(k) !== -1;
+      });
+
+      expect(actualLoadOrder, 'load order').to.eql(expectedLoadOrder);
+    });
+
+    it('supports .js for custom datasource config files', function() {
+      givenAppInSandbox();
+      fs.writeFileSync(
+        path.resolve(appDir, 'datasources.local.js'),
+        'module.exports = { db: { fromJs: true } };');
+
+      var app = loopback();
+      boot(app, appDir);
+
+      var db = app.datasources.db.settings;
+      expect(db).to.have.property('fromJs', true);
+    });
+
+    it('refuses to merge Object properties', function() {
+      givenAppInSandbox();
+      writeAppConfigFile('datasources.local.json', {
+        db: { nested: { key: 'value' } }
+      });
+
+      var app = loopback();
+      expect(function() { boot(app, appDir); })
+        .to.throw(/`nested` is not a value type/);
+    });
+
+    it('refuses to merge Array properties', function() {
+      givenAppInSandbox();
+      writeAppConfigFile('datasources.local.json', {
+        db: { nested: ['value'] }
+      });
+
+      var app = loopback();
+      expect(function() { boot(app, appDir); })
+        .to.throw(/`nested` is not a value type/);
+    });
+
+    it('merges app configs from multiple files', function() {
+      givenAppInSandbox();
+
+      writeAppConfigFile('app.local.json', { cfgLocal: 'applied' });
+
+      var env = process.env.NODE_ENV || 'development';
+      writeAppConfigFile('app.' + env + '.json', { cfgEnv: 'applied' });
+
+      var app = loopback();
+      boot(app, appDir);
+
+      expect(app.settings).to.have.property('cfgLocal', 'applied');
+      expect(app.settings).to.have.property('cfgEnv', 'applied');
+
+      var expectedLoadOrder = ['cfgLocal', 'cfgEnv'];
+      var actualLoadOrder = Object.keys(app.settings).filter(function(k) {
+        return expectedLoadOrder.indexOf(k) !== -1;
+      });
+
+      expect(actualLoadOrder, 'load order').to.eql(expectedLoadOrder);
+    });
+
+    it('supports .js for custom app config files', function() {
+      givenAppInSandbox();
+      fs.writeFileSync(
+        path.resolve(appDir, 'app.local.js'),
+        'module.exports = { fromJs: true };');
+
+      var app = loopback();
+      boot(app, appDir);
+
+      expect(app.settings).to.have.property('fromJs', true);
+    });
   });
 });
 
@@ -206,3 +320,27 @@ assert.isFunc = function (obj, name) {
     ' on object that does not exist');
   assert(typeof obj[name] === 'function', name + ' is not a function');
 };
+
+function givenAppInSandbox(appConfig, dataSources, models) {
+  fs.mkdirsSync(appDir);
+
+  appConfig = extend({
+  }, appConfig);
+  writeAppConfigFile('app.json', appConfig);
+
+  dataSources = extend({
+    db: {
+      connector: 'memory',
+      defaultForType: 'db'
+    }
+  }, dataSources);
+  writeAppConfigFile('datasources.json', dataSources);
+
+  models = extend({
+  }, models);
+  writeAppConfigFile('models.json', models);
+}
+
+function writeAppConfigFile(name, json) {
+  fs.writeJsonFileSync(path.resolve(appDir, name), json);
+}

test/helpers/sandbox.js

@@ -0,0 +1,16 @@
+var fs = require('fs-extra');
+var path = require('path');
+
+var sandbox = exports;
+sandbox.PATH = path.join(__dirname, '..', 'sandbox');
+
+sandbox.reset = function() {
+  fs.removeSync(sandbox.PATH);
+  fs.mkdirsSync(sandbox.PATH);
+};
+
+sandbox.resolve = function() {
+  var args = Array.prototype.slice.apply(arguments);
+  args.unshift(sandbox.PATH);
+  return path.resolve.apply(path.resolve, args);
+};