Rework model configuration

Rework the way how models are configured, the goal is to allow
loopback-boot to automatically determine the correct order
of the model definitions to ensure base models are defined
before they are extended.

 1. The model .js file no longer creates the model, it exports
 a config function instead:

  ```js
  module.exports = function(Car, Base) {
    // Car is the model constructor
    // Base is the parent model (e.g. loopback.PersistedModel)

    Car.prototype.honk = function(duration, cb) {
      // make some noise for `duration` seconds
      cb();
    };
  };
  ```

 2. The model is created by loopback-boot from model .json file.
  The .js file must have the same base file name.

 3. The `boot()` function has a new parameter `modelSources` to
  specify the list of directories where to look for model definitions.
  The parameter defaults to `['./models']`.

As a side effect, only models configured in `models.json` and their
base clases are defined. This should keep the size of the browserified
bundle small, because unused models are not included.

docs/browserify.md

@@ -46,7 +46,6 @@ contained in the browser bundle:
 /*-- app.js --*/
 var loopback = require('loopback');
 var boot = require('loopback-boot');
-require('./models');
 
 var app = module.exports = loopback();
 boot(app);

docs/configuration.md

@@ -2,49 +2,99 @@
 
 ### Model Definitions
 
-The following is example JSON for two `Model` definitions:
-"dealership" and "location".
+The following two examples demonstrate how to define models.
+
+*models/dealership.json*
+
+```json
+{
+  "name": "dealership",
+  "relations": {
+    "cars": {
+      "type": "hasMany",
+      "model": "Car",
+      "foreignKey": "dealerId"
+    }
+  },
+  "properties": {
+    "id": {"id": true},
+    "name": "String",
+    "zip": "Number",
+    "address": "String"
+  }
+}
+```
+
+*models/car.json*
+
+```json
+{
+  "name": "car",
+  "properties": {
+    "id": {
+      "type": "String",
+      "required": true,
+      "id": true
+    },
+    "make": {
+      "type": "String",
+      "required": true
+    },
+    "model": {
+      "type": "String",
+      "required": true
+    }
+  }
+}
+```
+
+To add custom methods to your models, create a `.js` file with the same name
+as the `.json` file:
+
+*models/car.js*
 
 ```js
+module.exports = function(Car, Base) {
+  // Car is the model constructor
+  // Base is the parent model (e.g. loopback.PersistedModel)
+
+  // Define a static method
+  Car.customMethod = function(cb) {
+    // do some work
+    cb();
+  };
+
+  Car.prototype.honk = function(duration, cb) {
+    // make some noise for `duration` seconds
+    cb();
+  };
+
+  Car.setup = function() {
+    Base.setup.call(this);
+
+    // configure validations,
+    // configure remoting for methods, etc.
+  };
+}
+```
+
+### Model Configuration
+
+The following is an example JSON configuring the models defined above
+for use in an loopback application.
+
+`dataSource` options is a reference, by name, to a data-source defined
+in `datasources.json`.
+
+*models.json*
+
+```json
 {
   "dealership": {
-    // a reference, by name, to a dataSource definition
     "dataSource": "my-db",
-    // the options passed to Model.extend(name, properties, options)
-    "options": {
-      "relations": {
-        "cars": {
-          "type": "hasMany",
-          "model": "Car",
-          "foreignKey": "dealerId"
-        }
-      }
-    },
-    // the properties passed to Model.extend(name, properties, options)
-    "properties": {
-      "id": {"id": true},
-      "name": "String",
-      "zip": "Number",
-      "address": "String"
-    }
   },
   "car": {
     "dataSource": "my-db"
-    "properties": {
-      "id": {
-        "type": "String",
-        "required": true,
-        "id": true
-      },
-      "make": {
-        "type": "String",
-        "required": true
-      },
-      "model": {
-        "type": "String",
-        "required": true
-      }
-    }
   }
 }
 ```
@@ -72,8 +122,9 @@ The following is example JSON for two `Model` definitions:
 var app = require('../app');
 var Car = app.models.Car;
 
-Car.prototype.honk = function() {
-  // make some noise
+Car.prototype.honk = function(duration, cb) {
+  // make some noise for `duration` seconds
+  cb();
 };
 ```
 
@@ -92,7 +143,7 @@ developer to create them before booting the app.**
 
 The folder `models/` has a different semantincs in 2.x than in 1.x. Instead
 of extending Models already defined by `app.boot` and `models.json`,
-it is an encapsulated component that defines all Models independently of
+it provides a set of Model definitions that do not depend on
 any application that may use them.
 
 Perform the following steps to update a 1.x project for loopback-boot 2.x.
@@ -122,37 +173,33 @@ All code samples are referring to the sample project described above.
     }
   ```
 
- 2. Change per-model javascript files to build and export the Model class:
+ 2. Change per-model javascript files to export a function that adds
+ custom methods to the model class.
+
 
   *models/car.js*
 
   ```js
-    var loopback = require('loopback');
-    var Car = module.exports = loopback.createModel(require('./car.json'));
-
-    Car.prototype.honk = function() {
-      // make some noise
+    module.exports = function(Car, Base) {
+      Car.prototype.honk = function(duration, cb) {
+        // make some noise for `duration` seconds
+        cb();
+      };
     };
   ```
 
- 3. Add a new file `models/index.js` to build all models:
-
- *models/index.js*
-
- ```js
-   exports.Car = require('./car');
- ```
-
- 4. Modify the main application file to load model definitions before booting
- the application.
+ 4. Modify the boot configuration to list the directory containing
+  model definitions.
 
  ```js
     var loopback = require('loopback');
     var boot = require('loopback-boot');
-    require('./models');
 
     var app = loopback();
-    boot(app, __dirname);
+    boot(app, {
+     appRootDir: __dirname,
+     modelSources: ['./models']
+   });
   ```
 
 #### Attaching built-in models

index.js

@@ -33,9 +33,9 @@ var addInstructionsToBrowserify = require('./lib/bundler');
  * `/boot` subdirectory of the application root directory with `require()`.
  *
  *  **NOTE:** The version 2.0 of loopback-boot changed the way how models
- *  are created. loopback-boot no longer creates the models for you,
- *  the `models.json` file contains only configuration options like
- *  dataSource and extra relations.
+ *  are created. The `models.json` file contains only configuration options like
+ *  dataSource and extra relations. To define a model, create a per-model
+ *  JSON file in `models/` directory.
  *
  *  **NOTE:** mixing `app.boot()` and `app.model(name, config)` in multiple
  *  files may result in models being **undefined** due to race conditions.
@@ -60,6 +60,8 @@ var addInstructionsToBrowserify = require('./lib/bundler');
  * @property {String} [env] Environment type, defaults to `process.env.NODE_ENV`
  * or `development`. Common values are `development`, `staging` and
  * `production`; however the applications are free to use any names.
+ * @property {Array.<String>} [modelSources] List of directories where to look
+ * for files containing model definitions.
  * @end
  *
  * @header bootLoopBackApp(app, [options])

lib/bundler.js

@@ -9,32 +9,53 @@ var commondir = require('commondir');
  */
 
 module.exports = function addInstructionsToBrowserify(instructions, bundler) {
-  bundleScripts(instructions.files, bundler);
+  bundleModelScripts(instructions, bundler);
+  bundleOtherScripts(instructions, bundler);
   bundleInstructions(instructions, bundler);
 };
 
-function bundleScripts(files, bundler) {
-  for (var key in files) {
-    var list = files[key];
-    if (!list.length) continue;
+function bundleOtherScripts(instructions, bundler) {
+  for (var key in instructions.files) {
+    addScriptsToBundle(key, instructions.files[key], bundler);
+  }
+}
 
-    var root = commondir(files[key].map(path.dirname));
+function bundleModelScripts(instructions, bundler) {
+  var files = instructions.models
+    .map(function(m) { return m.sourceFile; })
+    .filter(function(f) { return !!f; });
 
-    for (var ix in list) {
-      var filepath = list[ix];
+  var modelToFileMapping = instructions.models
+    .map(function(m) { return files.indexOf(m.sourceFile); });
 
-      // Build a short unique id that does not expose too much
-      // information about the file system, but still preserves
-      // useful information about where is the file coming from.
-      var fileid = 'loopback-boot#' + key + '#' + path.relative(root, filepath);
+  addScriptsToBundle('models', files, bundler);
 
-      // Add the file to the bundle.
-      bundler.require(filepath, { expose: fileid });
+  // Update `sourceFile` properties with the new paths
+  modelToFileMapping.forEach(function(fileIx, modelIx) {
+    if (fileIx === -1) return;
+    instructions.models[modelIx].sourceFile = files[fileIx];
+  });
+}
 
-      // Rewrite the instructions entry with the new id that will be
-      // used to load the file via `require(fileid)`.
-      list[ix] = fileid;
-    }
+function addScriptsToBundle(name, list, bundler) {
+  if (!list.length) return;
+
+  var root = commondir(list.map(path.dirname));
+
+  for (var ix in list) {
+    var filepath = list[ix];
+
+    // Build a short unique id that does not expose too much
+    // information about the file system, but still preserves
+    // useful information about where is the file coming from.
+    var fileid = 'loopback-boot#' + name + '#' + path.relative(root, filepath);
+
+    // Add the file to the bundle.
+    bundler.require(filepath, { expose: fileid });
+
+    // Rewrite the instructions entry with the new id that will be
+    // used to load the file via `require(fileid)`.
+    list[ix] = fileid;
   }
 }
 

lib/compiler.js

@@ -42,10 +42,14 @@ module.exports = function compile(options) {
   // require directories
   var bootScripts = findScripts(path.join(appRootDir, 'boot'));
 
+  var modelSources = options.modelSources || ['./models'];
+  var modelInstructions = buildAllModelInstructions(
+    appRootDir, modelsConfig, modelSources);
+
   return {
     app: appConfig,
     dataSources: dataSourcesConfig,
-    models: modelsConfig,
+    models: modelInstructions,
     files: {
       boot: bootScripts
     }
@@ -138,3 +142,111 @@ function tryReadDir() {
     return [];
   }
 }
+
+function buildAllModelInstructions(rootDir, modelsConfig, sources) {
+  var registry = findModelDefinitions(rootDir, sources);
+
+  var modelNamesToBuild = addAllBaseModels(registry, Object.keys(modelsConfig));
+
+  var instructions = modelNamesToBuild
+    .map(function createModelInstructions(name) {
+      var config = modelsConfig[name];
+      var definition = registry[name] || {};
+
+      debug('Using model "%s"\nConfiguration: %j\nDefinition %j',
+        name, config, definition.definition);
+
+      return {
+        name: name,
+        config: config,
+        definition: definition.definition,
+        sourceFile: definition.sourceFile
+      };
+    });
+
+  return sortByInheritance(instructions);
+}
+
+function addAllBaseModels(registry, modelNames) {
+  var result = [];
+  var visited = {};
+
+  while (modelNames.length) {
+    var name = modelNames.shift();
+    result.push(name);
+
+    var definition = registry[name] && registry[name].definition;
+    if (!definition) continue;
+
+    var base = definition.base || definition.options && definition.options.base;
+    if (!base || base in visited) continue;
+
+    visited[base] = true;
+
+    // ignore built-in models like User
+    if (!registry[base]) continue;
+
+    modelNames.push(base);
+  }
+
+  return result;
+}
+
+function sortByInheritance(instructions) {
+  // TODO implement topological sort
+  return instructions.reverse();
+}
+
+function findModelDefinitions(rootDir, sources) {
+  var registry = {};
+
+  sources.forEach(function(src) {
+    var srcDir = path.resolve(rootDir, src);
+    var files = tryReadDir(srcDir);
+    files
+      .filter(function(f) {
+        return f[0] !== '_' && path.extname(f) === '.json';
+      })
+      .forEach(function(f) {
+        var fullPath = path.resolve(srcDir, f);
+        var entry = loadModelDefinition(rootDir, fullPath);
+        var modelName = entry.definition.name;
+        if (!modelName) {
+          debug('Skipping model definition without Model name: %s',
+            path.relative(srcDir, fullPath));
+          return;
+        }
+        registry[modelName] = entry;
+      });
+  });
+
+  return registry;
+}
+
+function loadModelDefinition(rootDir, jsonFile) {
+  var definition = require(jsonFile);
+
+  var sourceFile = path.join(
+    path.dirname(jsonFile),
+    path.basename(jsonFile, path.extname(jsonFile)));
+
+  try {
+    // resolve the file to `.js` or any other supported extension like `.coffee`
+    sourceFile = require.resolve(sourceFile);
+  } catch (err) {
+    debug('Model source code not found: %s - %s', sourceFile, err.code || err);
+    sourceFile = undefined;
+  }
+
+  if (sourceFile === jsonFile)
+    sourceFile = undefined;
+
+  debug('Found model "%s" - %s %s', definition.name,
+    path.relative(rootDir, jsonFile),
+    sourceFile ? path.relative(rootDir, sourceFile) : '(no source file)');
+
+  return {
+    definition: definition,
+    sourceFile: sourceFile
+  };
+}

lib/executor.js

@@ -95,12 +95,37 @@ function setupDataSources(app, instructions) {
 }
 
 function setupModels(app, instructions) {
-  forEachKeyedObject(instructions.models, function(key, obj) {
-    var model = loopback.getModel(key);
-    if (!model) {
-      throw new Error('Cannot configure unknown model ' + key);
+  instructions.models.forEach(function(data) {
+    var name = data.name;
+    var model;
+
+    if (!data.definition) {
+      model = loopback.getModel(name);
+      if (!model) {
+        throw new Error('Cannot configure unknown model ' + name);
+      }
+      debug('Configuring existing model %s', name);
+    } else {
+      debug('Creating new model %s %j', name, data.definition);
+      model = loopback.createModel(data.definition);
+      if (data.sourceFile) {
+        debug('Loading customization script %s', data.sourceFile);
+        var code = require(data.sourceFile);
+        if (typeof code === 'function') {
+          debug('Customizing model %s', name);
+          // NOTE model.super_ is set by Node's util.inherits
+          code(model, model.super_);
+        } else {
+          debug('Skipping model file %s - `module.exports` is not a function',
+            data.sourceFile);
+        }
+      }
     }
-    app.model(model, obj);
+
+    // Skip base models that are not exported to the app
+    if (!data.config) return;
+
+    app.model(model, data.config);
   });
 }
 

test/browser.test.js

@@ -17,6 +17,9 @@ describe('browser support', function() {
 
       // configured in fixtures/browser-app/boot/configure.js
       expect(app.settings).to.have.property('custom-key', 'custom-value');
+      expect(Object.keys(app.models)).to.include('Customer');
+      expect(app.models.Customer.settings)
+        .to.have.property('_customized', 'Customer');
 
       done();
     });
@@ -53,12 +56,17 @@ function executeBundledApp(bundlePath) {
 }
 
 function createBrowserLikeContext() {
-  return vm.createContext({
+  var context = {
     // required by browserify
     XMLHttpRequest: function() { throw new Error('not implemented'); },
 
-    // used by loopback to detect browser runtime
-    window: {},
+    localStorage: {
+      // used by `debug` module
+      debug: process.env.DEBUG
+    },
+
+    // used by `debug` module
+    document: { documentElement: { style: {} } },
 
     // allow the browserified code to log messages
     // call `printContextLogs(context)` to print the accumulated messages
@@ -78,7 +86,12 @@ function createBrowserLikeContext() {
         error: []
       },
     }
-  });
+  };
+
+  // `window` is used by loopback to detect browser runtime
+  context.window = context;
+
+  return vm.createContext(context);
 }
 
 function printContextLogs(context) {

test/compiler.test.js

@@ -1,7 +1,6 @@
 var boot = require('../');
 var fs = require('fs-extra');
 var path = require('path');
-var assert = require('assert');
 var expect = require('must');
 var sandbox = require('./helpers/sandbox');
 var appdir = require('./helpers/appdir');
@@ -59,7 +58,15 @@ describe('compiler', function() {
     });
 
     it('has models definition', function() {
-      expect(instructions.models).to.eql(options.models);
+      expect(instructions.models).to.have.length(1);
+      expect(instructions.models[0]).to.eql({
+        name: 'foo-bar-bat-baz',
+        config: {
+          dataSource: 'the-db'
+        },
+        definition: undefined,
+        sourceFile: undefined
+      });
     });
 
     it('has datasources definition', function() {
@@ -70,8 +77,16 @@ describe('compiler', function() {
   describe('from directory', function() {
     it('loads config files', function() {
       var instructions = boot.compile(SIMPLE_APP);
-      assert(instructions.models.User);
-      assert(instructions.models.User.dataSource);
+
+      expect(instructions.models).to.have.length(1);
+      expect(instructions.models[0]).to.eql({
+        name: 'User',
+        config: {
+          dataSource: 'db'
+        },
+        definition: undefined,
+        sourceFile: undefined
+      });
     });
 
     it('merges datasource configs from multiple files', function() {
@@ -191,7 +206,8 @@ describe('compiler', function() {
         modelsRootDir: path.resolve(appdir.PATH, 'custom')
       });
 
-      expect(instructions.models).to.have.property('foo');
+      expect(instructions.models).to.have.length(1);
+      expect(instructions.models[0]).to.have.property('name', 'foo');
     });
 
     it('includes boot/*.js scripts', function() {
@@ -229,5 +245,123 @@ describe('compiler', function() {
         .to.throw(/unsupported 1\.x format/);
     });
 
+    it('loads models from `./models`', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('models/car.json', { name: 'Car' });
+      appdir.writeFileSync('models/car.js', '');
+
+      var instructions = boot.compile(appdir.PATH);
+
+      expect(instructions.models).to.have.length(1);
+      expect(instructions.models[0]).to.eql({
+        name: 'Car',
+        config: {
+          dataSource: 'db'
+        },
+        definition: {
+          name: 'Car'
+        },
+        sourceFile: path.resolve(appdir.PATH, 'models', 'car.js')
+      });
+    });
+
+    it('supports `modelSources` option', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('custom-models/car.json', { name: 'Car' });
+      appdir.writeFileSync('custom-models/car.js', '');
+
+      var instructions = boot.compile({
+        appRootDir: appdir.PATH,
+        modelSources: ['./custom-models']
+      });
+
+      expect(instructions.models).to.have.length(1);
+      expect(instructions.models[0]).to.eql({
+        name: 'Car',
+        config: {
+          dataSource: 'db'
+        },
+        definition: {
+          name: 'Car'
+        },
+        sourceFile: path.resolve(appdir.PATH, 'custom-models', 'car.js')
+      });
+    });
+
+    it('handles model definitions with no code', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('models/car.json', { name: 'Car' });
+
+      var instructions = boot.compile(appdir.PATH);
+
+      expect(instructions.models).to.eql([{
+        name: 'Car',
+        config: {
+          dataSource: 'db'
+        },
+        definition: {
+          name: 'Car'
+        },
+        sourceFile: undefined
+      }]);
+    });
+
+    it('excludes models not listed in `models.json`', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('models/car.json', { name: 'Car' });
+      appdir.writeConfigFileSync('models/bar.json', { name: 'Bar' });
+
+      var instructions = boot.compile(appdir.PATH);
+
+      var models = instructions.models.map(getNameProperty);
+      expect(models).to.eql(['Car']);
+    });
+
+    it('includes models used as Base models', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('models/car.json', {
+        name: 'Car',
+        base: 'Vehicle'
+      });
+      appdir.writeConfigFileSync('models/vehicle.json', {
+        name: 'Vehicle'
+      });
+
+      var instructions = boot.compile(appdir.PATH);
+      var models = instructions.models;
+      var modelNames = models.map(getNameProperty);
+
+      expect(modelNames).to.eql(['Vehicle', 'Car']);
+      expect(models[0].config).to.equal(undefined);
+    });
+
+    it('excludes pre-built base models', function() {
+      appdir.createConfigFilesSync({}, {}, {
+        Car: { dataSource: 'db' }
+      });
+      appdir.writeConfigFileSync('models/car.json', {
+        name: 'Car',
+        base: 'Model'
+      });
+
+      var instructions = boot.compile(appdir.PATH);
+
+      var modelNames = instructions.models.map(getNameProperty);
+      expect(modelNames).to.eql(['Car']);
+    });
   });
 });
+
+function getNameProperty(obj) {
+  return obj.name;
+}

test/executor.test.js

@@ -8,12 +8,8 @@ var appdir = require('./helpers/appdir');
 
 var SIMPLE_APP = path.join(__dirname, 'fixtures', 'simple-app');
 
-// ensure simple-app's models are known by loopback
-require(path.join(SIMPLE_APP, '/models'));
-
 var app;
 
-
 describe('executor', function() {
   beforeEach(sandbox.reset);
 
@@ -31,11 +27,14 @@ describe('executor', function() {
       foo: { bar: 'bat' },
       baz: true
     },
-    models: {
-      'User': {
-        dataSource: 'the-db'
+    models: [
+      {
+        name: 'User',
+        config: {
+          dataSource: 'the-db'
+        }
       }
-    },
+    ],
     dataSources: {
       'the-db': {
         connector: 'memory',
@@ -44,7 +43,7 @@ describe('executor', function() {
     }
   });
 
-  it('instantiates models', function() {
+  it('configures models', function() {
     boot.execute(app, dummyInstructions);
     assert(app.models);
     assert(app.models.User);
@@ -55,6 +54,60 @@ describe('executor', function() {
     assert.isFunc(app.models.User, 'create');
   });
 
+  it('defines and customizes models', function() {
+    appdir.writeFileSync('models/Customer.js', 'module.exports = ' +
+      function(Customer, Base) {
+        Customer.settings._customized = 'Customer';
+        Base.settings._customized = 'Base';
+      }.toString());
+
+    boot.execute(app, someInstructions({
+      dataSources: { db: { connector: 'memory' } },
+      models: [
+        {
+          name: 'Customer',
+          config: { dataSource: 'db' },
+          definition: {
+            name: 'Customer',
+            base: 'User',
+          },
+          sourceFile: path.resolve(appdir.PATH, 'models', 'Customer.js')
+        }
+      ]
+    }));
+
+    expect(app.models.Customer).to.exist();
+    expect(app.models.Customer.settings._customized).to.be.equal('Customer');
+    expect(loopback.User.settings._customized).to.equal('Base');
+  });
+
+  it('defines model without attaching it', function() {
+    boot.execute(app, someInstructions({
+      dataSources: { db: { connector: 'memory' } },
+      models: [
+        {
+          name: 'Vehicle',
+          config: undefined,
+          definition: {
+            name: 'Vehicle'
+          },
+          sourceFile: undefined
+        },
+        {
+          name: 'Car',
+          config: { dataSource: 'db' },
+          definition: {
+            name: 'Car',
+            base: 'Vehicle',
+          },
+          sourceFile: undefined
+        },
+      ]
+    }));
+
+    expect(Object.keys(app.models)).to.eql(['Car']);
+  });
+
   it('attaches models to data sources', function() {
     boot.execute(app, dummyInstructions);
     assert.equal(app.models.User.dataSource, app.dataSources.theDb);
@@ -203,7 +256,7 @@ assert.isFunc = function (obj, name) {
 function someInstructions(values) {
   var result = {
     app: values.app || {},
-    models: values.models || {},
+    models: values.models || [],
     dataSources: values.dataSources || {},
     files: {
       boot: []

test/fixtures/browser-app/datasources.json

@@ -0,0 +1,5 @@
+{
+  "db": {
+    "connector": "remote"
+  }
+}

test/fixtures/browser-app/models.json

@@ -0,0 +1,5 @@
+{
+  "Customer": {
+    "dataSource": "db"
+  }
+}

test/fixtures/browser-app/models/customer.js

@@ -0,0 +1,4 @@
+module.exports = function(Customer, Base) {
+  Customer.settings._customized = 'Customer';
+  Base.settings._customized = 'Base';
+};

test/fixtures/browser-app/models/customer.json

@@ -0,0 +1,4 @@
+{
+  "name": "Customer",
+  "base": "User"
+}