Merge branch 'release/2.15.0' into production

CHANGES.md

@@ -1,3 +1,41 @@
+2015-02-02, Version 2.15.0
+==========================
+
+ * Fix id type issue for update (Raymond Feng)
+
+ * Rename hook "query" to "access" (Miroslav Bajtoš)
+
+ * Implement intent hook `before delete` (Miroslav Bajtoš)
+
+ * Remove redundant `.toObject()` call from `upsert` (Miroslav Bajtoš)
+
+ * Fix regression in `.save()` from 1fd6eff1 (Miroslav Bajtoš)
+
+ * Fix hasOne remoting (Raymond Feng)
+
+ * Make sure batch create calls back with correct data (Raymond Feng)
+
+ * Intent-based hooks for persistence (Miroslav Bajtoš)
+
+ * ModelBaseClass: implement async observe/notify (Miroslav Bajtoš)
+
+ * Upgrade `should` to the latest 1.x version (Miroslav Bajtoš)
+
+ * Fixed nullCheck in validations to correct behavior when dealing with undefined attributes (James Billingham)
+
+ * Supply target to applyProperties function (Fabien Franzen)
+
+ * fix id property for composite ids (Clark Wang)
+
+ * fix id properties should sort by its index (Clark Wang)
+
+ * Fixed typos and logic for protected properties (Christian Enevoldsen)
+
+ * adds support for protected properties. (Christian Enevoldsen)
+
+ * support embeds data for belongsTo relation Signed-off-by: Clark Wang <clark.wangs@gmail.com> (Clark Wang)
+
+
 2015-01-15, Version 2.14.1
 ==========================
 
@@ -395,6 +433,13 @@
 
  * Properly handle LDL for polymorphic relations (Fabien Franzen)
 
+ * Check null (Raymond Feng)
+
+
+2014-08-15, Version 2.4.0
+=========================
+
+
 
 2014-08-15, Version 2.4.1
 =========================
@@ -403,12 +448,6 @@
 
  * Check null (Raymond Feng)
 
-
-2014-08-15, Version 2.4.0
-=========================
-
- * Bump version (Raymond Feng)
-
  * Fix the test cases to avoid hard-coded ids (Raymond Feng)
 
  * Add strict flag to sortObjectsByIds (Fabien Franzen)
@@ -455,19 +494,16 @@
 
  * Cleanup mixin tests (Fabien Franzen)
 
- * Fix a name conflict in scope metadata (Raymond Feng)
-
-
-2014-08-08, Version 2.3.0
-=========================
-
-
 
 2014-08-08, Version 2.3.1
 =========================
 
  * Fix a name conflict in scope metadata (Raymond Feng)
 
+
+2014-08-08, Version 2.3.0
+=========================
+
  * Fix the test case so that it works with other DBs (Raymond Feng)
 
  * Bump version (Raymond Feng)
@@ -580,8 +616,6 @@
 
  * Implemented embedsMany relation (Fabien Franzen)
 
- * Fix a regression where undefined id should not match any record (Raymond Feng)
-
  * Minor tweaks; pass-through properties/scope for hasAndBelongsToMany (Fabien Franzen)
 
  * Implemented polymorphic hasMany through inverse (Fabien Franzen)
@@ -597,11 +631,6 @@
  * Implemented polymorphic hasMany (Fabien Franzen)
 
 
-2014-07-27, Version 2.1.0
-=========================
-
-
-
 2014-07-27, Version 2.1.1
 =========================
 
@@ -609,6 +638,12 @@
 
  * Fix a regression where undefined id should not match any record (Raymond Feng)
 
+
+2014-07-27, Version 2.1.0
+=========================
+
+ * Bump version (Raymond Feng)
+
  * datasource: support connectors without `getTypes` (Miroslav Bajtoš)
 
  * relation: add `scope._target` for `hasOne` (Miroslav Bajtoš)

lib/connectors/memory.js

@@ -573,6 +573,9 @@ Memory.prototype.update =
     async.each(ids, function (id, done) {
       var inst = self.fromDb(model, cache[id]);
       if (!filter || filter(inst)) {
+        // The id value from the cache is string
+        // Get the real id from the inst
+        id = self.getIdValue(model, inst);
         self.updateAttributes(model, id, data, done);
       } else {
         process.nextTick(done);
@@ -594,6 +597,9 @@ Memory.prototype.updateAttributes = function updateAttributes(model, id, data, c
     }
   }
 
+  // Do not modify the data object passed in arguments
+  data = Object.create(data);
+
   this.setIdValue(model, data, id);
 
   var cachedModels = this.collection(model);

lib/dao.js

@@ -7,6 +7,7 @@ module.exports = DataAccessObject;
 /*!
  * Module dependencies
  */
+var async = require('async');
 var jutil = require('./jutil');
 var ValidationError = require('./validations').ValidationError;
 var Relation = require('./relations.js');
@@ -62,6 +63,16 @@ function byIdQuery(m, id) {
   return query;
 }
 
+function isWhereByGivenId(Model, where, idValue) {
+  var keys = Object.keys(where);
+  if (keys.length != 1) return false;
+
+  var pk = idName(Model);
+  if (keys[0] !== pk) return false;
+
+  return where[pk] === idValue;
+}
+
 DataAccessObject._forDB = function (data) {
   if (!(this.getDataSource().isRelational && this.getDataSource().isRelational())) {
     return data;
@@ -149,39 +160,38 @@ DataAccessObject.create = function (data, callback) {
   }
 
   if (Array.isArray(data)) {
-    var instances = [];
+    // Undefined item will be skipped by async.map() which internally uses
-    var errors = Array(data.length);
+    // Array.prototype.map(). The following loop makes sure all items are
-    var gotError = false;
+    // iterated
-    var wait = data.length;
+    for (var i = 0, n = data.length; i < n; i++) {
-    if (wait === 0) {
+      if (data[i] === undefined) {
-      callback(null, []);
+        data[i] = {};
       }
-
-    for (var i = 0; i < data.length; i += 1) {
-      (function (d, i) {
-        Model = self.lookupModel(d); // data-specific
-        instances.push(Model.create(d, function (err, inst) {
-          if (err) {
-            errors[i] = err;
-            gotError = true;
     }
-          modelCreated();
+    async.map(data, function(item, done) {
-        }));
+      self.create(item, function(err, result) {
-      })(data[i], i);
+        // Collect all errors and results
-    }
+        done(null, {err: err, result: result || item});
-
-    return instances;
-
-    function modelCreated() {
-      if (--wait === 0) {
-        callback(gotError ? errors : null, instances);
-        if(!gotError) {
-          instances.forEach(function(inst) {
-            inst.constructor.emit('changed');
       });
+    }, function(err, results) {
+      if (err) {
+        return callback && callback(err, results);
       }
+      // Convert the results into two arrays
+      var errors = null;
+      var data = [];
+      for (var i = 0, n = results.length; i < n; i++) {
+        if (results[i].err) {
+          if (!errors) {
+            errors = [];
           }
+          errors[i] = results[i].err;
         }
+        data[i] = results[i].result;
+      }
+      callback && callback(errors, data);
+    });
+    return data;
   }
 
   var enforced = {};
@@ -201,6 +211,9 @@ DataAccessObject.create = function (data, callback) {
   Model = this.lookupModel(data); // data-specific
   if (Model !== obj.constructor) obj = new Model(data);
 
+  Model.notifyObserversOf('before save', { Model: Model, instance: obj }, function(err) {
+    if (err) return callback(err);
+
     data = obj.toObject(true);
 
     // validation required
@@ -211,11 +224,11 @@ DataAccessObject.create = function (data, callback) {
         callback(new ValidationError(obj), obj);
       }
     }, data);
+  });
 
   function create() {
     obj.trigger('create', function (createDone) {
       obj.trigger('save', function (saveDone) {
-
         var _idName = idName(Model);
         var modelName = Model.modelName;
         this._adapter().create(modelName, this.constructor._forDB(obj.toObject(true)), function (err, id, rev) {
@@ -232,10 +245,15 @@ DataAccessObject.create = function (data, callback) {
           obj.__persisted = true;
           saveDone.call(obj, function () {
             createDone.call(obj, function () {
+              if (err) {
+                return callback(err, obj);
+              }
+              Model.notifyObserversOf('after save', { Model: Model, instance: obj }, function(err) {
                 callback(err, obj);
                 if(!err) Model.emit('changed', obj);
               });
             });
+          });
         }, obj);
       }, obj, callback);
     }, obj, callback);
@@ -267,20 +285,41 @@ DataAccessObject.updateOrCreate = DataAccessObject.upsert = function upsert(data
   }
   var self = this;
   var Model = this;
-  if (!getIdValue(this, data)) {
+  var id = getIdValue(this, data);
+  if (!id) {
     return this.create(data, callback);
   }
-  if (this.getDataSource().connector.updateOrCreate) {
+
+  Model.notifyObserversOf('access', { Model: Model, query: byIdQuery(Model, id) }, doUpdateOrCreate);
+
+  function doUpdateOrCreate(err, ctx) {
+    if (err) return callback(err);
+
+    var isOriginalQuery = isWhereByGivenId(Model, ctx.query.where, id)
+    if (Model.getDataSource().connector.updateOrCreate && isOriginalQuery) {
+      var context = { Model: Model, where: ctx.query.where, data: data };
+      Model.notifyObserversOf('before save', context, function(err, ctx) {
+        if (err) return callback(err);
+
+        data = ctx.data;
         var update = data;
         var inst = data;
         if(!(data instanceof Model)) {
           inst = new Model(data);
         }
         update = inst.toObject(false);
-    this.applyProperties(update, inst);
+
+        Model.applyProperties(update, inst);
+        Model = Model.lookupModel(update);
+
+        // FIXME(bajtos) validate the model!
+        // https://github.com/strongloop/loopback-datasource-juggler/issues/262
+
         update = removeUndefined(update);
-    Model = this.lookupModel(update);
+        self.getDataSource().connector
-    this.getDataSource().connector.updateOrCreate(Model.modelName, update, function (err, data) {
+          .updateOrCreate(Model.modelName, update, done);
+
+        function done(err, data) {
           var obj;
           if (data && !(data instanceof Model)) {
             inst._initProperties(data);
@@ -288,16 +327,31 @@ DataAccessObject.updateOrCreate = DataAccessObject.upsert = function upsert(data
           } else {
             obj = data;
           }
+          if (err) {
+            callback(err, obj);
+            if(!err) {
+              Model.emit('changed', inst);
+            }
+          } else {
+            Model.notifyObserversOf('after save', { Model: Model, instance: obj }, function(err) {
               callback(err, obj);
               if(!err) {
                 Model.emit('changed', inst);
               }
             });
+          }
+        }
+      });
     } else {
-    this.findById(getIdValue(this, data), function (err, inst) {
+      Model.findOne({ where: ctx.query.where }, { notify: false }, function (err, inst) {
         if (err) {
           return callback(err);
         }
+        if (!isOriginalQuery) {
+          // The custom query returned from a hook may hide the fact that
+          // there is already a model with `id` value `data[idName(Model)]`
+          delete data[idName(Model)];
+        }
         if (inst) {
           inst.updateAttributes(data, callback);
         } else {
@@ -307,6 +361,7 @@ DataAccessObject.updateOrCreate = DataAccessObject.upsert = function upsert(data
         }
       });
     }
+  }
 };
 
 /**
@@ -332,11 +387,11 @@ DataAccessObject.findOrCreate = function findOrCreate(query, data, callback) {
     };
   }
 
-  var t = this;
+  var Model = this;
-  this.findOne(query, function (err, record) {
+  Model.findOne(query, function (err, record) {
     if (err) return callback(err);
     if (record) return callback(null, record, false);
-    t.create(data, function (err, record) {
+    Model.create(data, function (err, record) {
       callback(err, record, record != null);
     });
   });
@@ -731,13 +786,22 @@ DataAccessObject._coerce = function (where) {
  * @param {Function} callback Required callback function.  Call this function with two arguments: `err` (null or Error) and an array of instances.
  */
 
-DataAccessObject.find = function find(query, cb) {
+DataAccessObject.find = function find(query, options, cb) {
   if (stillConnecting(this.getDataSource(), this, arguments)) return;
 
   if (arguments.length === 1) {
     cb = query;
     query = null;
+    options = {};
   }
+
+  if (cb === undefined && typeof options === 'function') {
+    cb = options;
+    options = {};
+  }
+
+  if (!options) options = {};
+
   var self = this;
 
   query = query || {};
@@ -763,7 +827,11 @@ DataAccessObject.find = function find(query, cb) {
       // do in memory query
       // using all documents
       // TODO [fabien] use default scope here?
-      this.getDataSource().connector.all(this.modelName, {}, function (err, data) {
+
+      self.notifyObserversOf('access', { Model: self, query: query }, function(err, ctx) {
+        if (err) return cb(err);
+
+        self.getDataSource().connector.all(self.modelName, {}, function (err, data) {
           var memory = new Memory();
           var modelName = self.modelName;
 
@@ -782,18 +850,20 @@ DataAccessObject.find = function find(query, cb) {
               });
             });
 
-          memory.all(modelName, query, cb);
+            // FIXME: apply "includes" and other transforms - see allCb below
+            memory.all(modelName, ctx.query, cb);
           } else {
             cb(null, []);
           }
-      }.bind(this));
+        });
+      });
 
       // already handled
       return;
     }
   }
 
-  this.getDataSource().connector.all(this.modelName, query, function (err, data) {
+  var allCb = function (err, data) {
     if (data && data.forEach) {
       data.forEach(function (d, i) {
         var Model = self.lookupModel(d);
@@ -840,7 +910,18 @@ DataAccessObject.find = function find(query, cb) {
     }
     else
       cb(err, []);
+  }
+
+  var self = this;
+  if (options.notify === false) {
+    self.getDataSource().connector.all(self.modelName, query, allCb);
+  } else {
+    this.notifyObserversOf('access', { Model: this, query: query }, function(err, ctx) {
+      if (err) return cb(err);
+      var query = ctx.query;
+      self.getDataSource().connector.all(self.modelName, query, allCb);
     });
+  }
 };
 
 /**
@@ -850,16 +931,22 @@ DataAccessObject.find = function find(query, cb) {
  * For example: `{where: {test: 'me'}}`.
  * @param {Function} cb Callback function called with (err, instance)
  */
-DataAccessObject.findOne = function findOne(query, cb) {
+DataAccessObject.findOne = function findOne(query, options, cb) {
   if (stillConnecting(this.getDataSource(), this, arguments)) return;
 
   if (typeof query === 'function') {
     cb = query;
     query = {};
   }
+
+  if (cb === undefined && typeof options === 'function') {
+    cb = options;
+    options = {};
+  }
+
   query = query || {};
   query.limit = 1;
-  this.find(query, function (err, collection) {
+  this.find(query, options, function (err, collection) {
     if (err || !collection || !collection.length > 0) return cb(err, null);
     cb(err, collection[0]);
   });
@@ -878,41 +965,83 @@ DataAccessObject.findOne = function findOne(query, cb) {
  * @param {Object} [where] Optional object that defines the criteria.  This is a "where" object. Do NOT pass a filter object.
  * @param {Function} [cb] Callback called with (err)
  */
-DataAccessObject.remove = DataAccessObject.deleteAll = DataAccessObject.destroyAll = function destroyAll(where, cb) {
+DataAccessObject.remove = DataAccessObject.deleteAll = DataAccessObject.destroyAll = function destroyAll(where, options, cb) {
   if (stillConnecting(this.getDataSource(), this, arguments)) return;
+
   var Model = this;
 
-  if (!cb && 'function' === typeof where) {
+  if (!cb && !options && 'function' === typeof where) {
     cb = where;
     where = undefined;
   }
 
+  if (!cb && typeof options === 'function') {
+    cb = options;
+  }
+
+  if (!cb) cb = function(){};
+  if (!options) options = {};
+
   var query = { where: where };
   this.applyScope(query);
   where = query.where;
 
-  if (!where || (typeof where === 'object' && Object.keys(where).length === 0)) {
+  var context = { Model: Model, where: whereIsEmpty(where) ? {} : where };
-    this.getDataSource().connector.destroyAll(this.modelName, function (err, data) {
+  if (options.notify === false) {
-      cb && cb(err, data);
+    doDelete(where);
-      if(!err) Model.emit('deletedAll');
+  } else {
-    }.bind(this));
+    query = { where: whereIsEmpty(where) ? {} : where };
+    Model.notifyObserversOf('access',
+      { Model: Model, query: query },
+      function(err, ctx) {
+        if (err) return cb(err);
+        var context = { Model: Model, where: ctx.query.where };
+        Model.notifyObserversOf('before delete', context, function(err, ctx) {
+          if (err) return cb(err);
+          doDelete(ctx.where);
+        });
+      });
+  }
+
+  function doDelete(where) {
+    if (whereIsEmpty(where)) {
+      Model.getDataSource().connector.destroyAll(Model.modelName, done);
     } else {
       try {
         // Support an optional where object
         where = removeUndefined(where);
-      where = this._coerce(where);
+        where = Model._coerce(where);
       } catch (err) {
         return process.nextTick(function() {
           cb && cb(err);
         });
       }
-    this.getDataSource().connector.destroyAll(this.modelName, where, function (err, data) {
+
-      cb && cb(err, data);
+      Model.getDataSource().connector.destroyAll(Model.modelName, where, done);
-      if(!err) Model.emit('deletedAll', where);
+
-    }.bind(this));
+    }
+
+    function done(err, data) {
+      if (err) return cb(er);
+
+      if (options.notify === false) {
+        return cb(err, data);
+      }
+
+      Model.notifyObserversOf('after delete', { Model: Model, where: where }, function(err) {
+        cb(err, data);
+        if (!err)
+          Model.emit('deletedAll', whereIsEmpty(where) ? undefined : where);
+      });
+    }
   }
 };
 
+function whereIsEmpty(where) {
+  return !where ||
+     (typeof where === 'object' && Object.keys(where).length === 0)
+}
+
 /**
  * Delete the record with the specified ID.
  * Aliases are `destroyById` and `deleteById`.
@@ -969,7 +1098,12 @@ DataAccessObject.count = function (where, cb) {
     });
   }
 
-  this.getDataSource().connector.count(this.modelName, cb, where);
+  var Model = this;
+  this.notifyObserversOf('access', { Model: Model, query: { where: where } }, function(err, ctx) {
+      if (err) return cb(err);
+      where = ctx.query.where;
+      Model.getDataSource().connector.count(Model.modelName, cb, where);
+    });
 };
 
 /**
@@ -1000,18 +1134,20 @@ DataAccessObject.prototype.save = function (options, callback) {
     options.throws = false;
   }
 
-  var inst = this;
-  var data = inst.toObject(true);
-  var modelName = Model.modelName;
-  
-  Model.applyProperties(data, this);
-  
   if (this.isNewRecord()) {
     return Model.create(this, callback);
-  } else {
-    inst.setAttributes(data);
   }
 
+  var inst = this;
+  var modelName = Model.modelName;
+
+  Model.notifyObserversOf('before save', { Model: Model, instance: inst }, function(err) {
+    if (err) return callback(err);
+
+    var data = inst.toObject(true);
+    Model.applyProperties(data, inst);
+    inst.setAttributes(data);
+
     // validate first
     if (!options.validate) {
       return save();
@@ -1040,6 +1176,8 @@ DataAccessObject.prototype.save = function (options, callback) {
               return callback(err, inst);
             }
             inst._initProperties(data, { persisted: true });
+            Model.notifyObserversOf('after save', { Model: Model, instance: inst }, function(err) {
+              if (err) return callback(err, inst);
               updateDone.call(inst, function () {
                 saveDone.call(inst, function () {
                   callback(err, inst);
@@ -1049,9 +1187,11 @@ DataAccessObject.prototype.save = function (options, callback) {
                 });
               });
             });
+          });
         }, data, callback);
       }, data, callback);
     }
+  });
 };
 
 /**
@@ -1100,17 +1240,49 @@ DataAccessObject.updateAll = function (where, data, cb) {
 
   where = query.where;
 
+  var Model = this;
+
+  Model.notifyObserversOf('access', { Model: Model, query: { where: where } }, function(err, ctx) {
+    if (err) return cb && cb(err);
+    Model.notifyObserversOf(
+      'before save',
+      {
+        Model: Model,
+        where: ctx.query.where,
+        data: data
+      },
+      function(err, ctx) {
+        if (err) return cb && cb(err);
+        doUpdate(ctx.where, ctx.data);
+      });
+  });
+
+
+  function doUpdate(where, data) {
     try {
       where = removeUndefined(where);
-    where = this._coerce(where);
+      where = Model._coerce(where);
     } catch (err) {
       return process.nextTick(function () {
         cb && cb(err);
       });
     }
 
-  var connector = this.getDataSource().connector;
+    var connector = Model.getDataSource().connector;
-  connector.update(this.modelName, where, data, cb);
+    connector.update(Model.modelName, where, data, function(err, count) {
+      if (err) return cb && cb (err);
+      Model.notifyObserversOf(
+        'after save',
+        {
+          Model: Model,
+          where: where,
+          data: data
+        },
+        function(err, ctx) {
+          return cb && cb(err, count);
+        });
+    });
+  }
 };
 
 DataAccessObject.prototype.isNewRecord = function () {
@@ -1134,21 +1306,54 @@ DataAccessObject.prototype.remove =
   DataAccessObject.prototype.delete =
     DataAccessObject.prototype.destroy = function (cb) {
       if (stillConnecting(this.getDataSource(), this, arguments)) return;
+      var self = this;
       var Model = this.constructor;
       var id = getIdValue(this.constructor, this);
 
-      this.trigger('destroy', function (destroyed) {
+      Model.notifyObserversOf(
-        this._adapter().destroy(this.constructor.modelName, id, function (err) {
+        'access',
+        { Model: Model, query: byIdQuery(Model, id) },
+        function(err, ctx) {
+          if (err) return cb(err);
+          Model.notifyObserversOf(
+            'before delete',
+            { Model: Model, where: ctx.query.where },
+            function(err, ctx) {
+              if (err) return cb(err);
+              doDeleteInstance(ctx.where);
+            });
+        });
+
+      function doDeleteInstance(where) {
+        if (!isWhereByGivenId(Model, where, id)) {
+          // A hook modified the query, it is no longer
+          // a simple 'delete model with the given id'.
+          // We must switch to full query-based delete.
+          Model.deleteAll(where, { notify: false }, function(err) {
+            if (err) return cb && cb(err);
+            Model.notifyObserversOf('after delete', { Model: Model, where: where }, function(err) {
+              cb && cb(err);
+              if (!err) Model.emit('deleted', id);
+            });
+          });
+          return;
+        }
+
+        self.trigger('destroy', function (destroyed) {
+          self._adapter().destroy(self.constructor.modelName, id, function (err) {
             if (err) {
               return cb(err);
             }
 
             destroyed(function () {
-            if (cb) cb();
+              Model.notifyObserversOf('after delete', { Model: Model, where: where }, function(err) {
-            Model.emit('deleted', id);
+                cb && cb(err);
+                if (!err) Model.emit('deleted', id);
+              });
+            });
           });
-        }.bind(this));
         }, null, cb);
+      }
     };
 
 /**
@@ -1231,15 +1436,35 @@ DataAccessObject.prototype.updateAttributes = function updateAttributes(data, cb
     data = {};
   }
 
+  if (!cb) {
+    cb = function() {};
+  }
+
+  // Convert the data to be plain object so that update won't be confused
+  if (data instanceof Model) {
+    data = data.toObject(false);
+  }
+  data = removeUndefined(data);
+
+  var context = {
+    Model: Model,
+    where: byIdQuery(Model, getIdValue(Model, inst)).where,
+    data: data
+  };
+
+  Model.notifyObserversOf('before save', context, function(err, ctx) {
+    if (err) return cb(err);
+    data = ctx.data;
+
     // update instance's properties
     inst.setAttributes(data);
 
     inst.isValid(function (valid) {
       if (!valid) {
-      if (cb) {
         cb(new ValidationError(inst), inst);
+        return;
       }
-    } else {
+
       inst.trigger('save', function (saveDone) {
         inst.trigger('update', function (done) {
           var typedData = {};
@@ -1260,15 +1485,18 @@ DataAccessObject.prototype.updateAttributes = function updateAttributes(data, cb
             if (!err) inst.__persisted = true;
             done.call(inst, function () {
               saveDone.call(inst, function () {
-                if(cb) cb(err, inst);
+                if (err) return cb(err, inst);
+                Model.notifyObserversOf('after save', { Model: Model, instance: inst }, function(err) {
                   if(!err) Model.emit('changed', inst);
+                  cb(err, inst);
+                });
               });
             });
           });
         }, data, cb);
       }, data, cb);
-    }
     }, data);
+  });
 };
 
 /**

lib/list.js

@@ -72,11 +72,11 @@ List.prototype.push = function (obj) {
   return item;
 };
 
-List.prototype.toObject = function (onlySchema, removeHidden) {
+List.prototype.toObject = function (onlySchema, removeHidden, removeProtected) {
   var items = [];
   this.forEach(function (item) {
     if (item && typeof item === 'object' && item.toObject) {
-      items.push(item.toObject(onlySchema, removeHidden));
+      items.push(item.toObject(onlySchema, removeHidden, removeProtected));
     } else {
       items.push(item);
     }

lib/model-builder.js

@@ -199,6 +199,7 @@ ModelBuilder.prototype.define = function defineClass(className, properties, sett
   hiddenProperty(ModelClass, 'relations', {});
   hiddenProperty(ModelClass, 'http', { path: '/' + pathName });
   hiddenProperty(ModelClass, 'base', ModelBaseClass);
+  hiddenProperty(ModelClass, '_observers', {});
 
   // inherit ModelBaseClass static methods
   for (var i in ModelBaseClass) {
@@ -266,7 +267,8 @@ ModelBuilder.prototype.define = function defineClass(className, properties, sett
       get: function () {
         var compositeId = {};
         var idNames = ModelClass.definition.idNames();
-        for (var p in idNames) {
+        for (var i = 0, p; i < idNames.length; i++) {
+          p = idNames[i];
           compositeId[p] = this.__data[p];
         }
         return compositeId;

lib/model-definition.js

@@ -141,7 +141,7 @@ ModelDefinition.prototype.ids = function () {
     ids.push({name: key, id: id, property: props[key]});
   }
   ids.sort(function (a, b) {
-    return a.key - b.key;
+    return a.id - b.id;
   });
   this._ids = ids;
   return ids;

lib/model.js

@@ -7,12 +7,15 @@ module.exports = ModelBaseClass;
  * Module dependencies
  */
 
+var async = require('async');
 var util = require('util');
 var jutil = require('./jutil');
 var List = require('./list');
 var Hookable = require('./hooks');
 var validations = require('./validations');
 var _extend = util._extend;
+var utils = require('./utils');
+var fieldsToArray = utils.fieldsToArray;
 
 // Set up an object for quick lookup
 var BASE_TYPES = {
@@ -170,7 +173,16 @@ ModelBaseClass.prototype._initProperties = function (data, options) {
       if (relationType === 'belongsTo' && propVal != null) {
         // If the related model is populated
         self.__data[ctor.relations[p].keyFrom] = propVal[ctor.relations[p].keyTo];
+
+        if (ctor.relations[p].options.embedsProperties) {
+          var fields = fieldsToArray(ctor.relations[p].properties, modelTo.definition.properties);
+          if (!~fields.indexOf(ctor.relations[p].keyTo)) {
+            fields.push(ctor.relations[p].keyTo);
           }
+          self.__data[p] = new modelTo(propVal, { fields: fields, applySetters: false, persisted: options.persisted });
+        }
+      }
+
       self.__cachedRelations[p] = propVal;
     } else {
       // Un-managed property
@@ -291,7 +303,7 @@ ModelBaseClass.toString = function () {
  *
  * @param {Boolean} onlySchema Restrict properties to dataSource only.  Default is false.  If true, the function returns only properties defined in the schema;  Otherwise it returns all enumerable properties.
  */
-ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
+ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden, removeProtected) {
   if (onlySchema === undefined) {
     onlySchema = true;
   }
@@ -310,6 +322,7 @@ ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
   var props = Model.definition.properties;
   var keys = Object.keys(props);
   var propertyName, val;
+
   for (var i = 0; i < keys.length; i++) {
     propertyName = keys[i];
     val = self[propertyName];
@@ -323,11 +336,15 @@ ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
       continue;
     }
 
+    if (removeProtected && Model.isProtectedProperty(propertyName)) {
+      continue;
+    }
+
     if (val instanceof List) {
-      data[propertyName] = val.toObject(!schemaLess, removeHidden);
+      data[propertyName] = val.toObject(!schemaLess, removeHidden, true);
     } else {
       if (val !== undefined && val !== null && val.toObject) {
-        data[propertyName] = val.toObject(!schemaLess, removeHidden);
+        data[propertyName] = val.toObject(!schemaLess, removeHidden, true);
       } else {
         data[propertyName] = val;
       }
@@ -351,13 +368,16 @@ ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
       if (removeHidden && Model.isHiddenProperty(propertyName)) {
         continue;
       }
+      if (removeProtected && Model.isProtectedProperty(propertyName)) {
+        continue;
+      }
       val = self[propertyName];
       if (val !== undefined && data[propertyName] === undefined) {
         if (typeof val === 'function') {
           continue;
         }
         if (val !== null && val.toObject) {
-          data[propertyName] = val.toObject(!schemaLess, removeHidden);
+          data[propertyName] = val.toObject(!schemaLess, removeHidden, true);
         } else {
           data[propertyName] = val;
         }
@@ -375,16 +395,18 @@ ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
         if (removeHidden && Model.isHiddenProperty(propertyName)) {
           continue;
         }
+        if (removeProtected && Model.isProtectedProperty(propertyName)) {
+          continue;
+        }
         var ownVal = self[propertyName];
         // The ownVal can be a relation function
-        val = (ownVal !== undefined && (typeof ownVal !== 'function'))
+        val = (ownVal !== undefined && (typeof ownVal !== 'function')) ? ownVal : self.__data[propertyName];
-          ? ownVal : self.__data[propertyName];
         if (typeof val === 'function') {
           continue;
         }
 
         if (val !== undefined && val !== null && val.toObject) {
-          data[propertyName] = val.toObject(!schemaLess, removeHidden);
+          data[propertyName] = val.toObject(!schemaLess, removeHidden, true);
         } else {
           data[propertyName] = val;
         }
@@ -395,6 +417,25 @@ ModelBaseClass.prototype.toObject = function (onlySchema, removeHidden) {
   return data;
 };
 
+ModelBaseClass.isProtectedProperty = function (propertyName) {
+  var Model = this;
+  var settings = Model.definition && Model.definition.settings;
+  var protectedProperties = settings && (settings.protectedProperties || settings.protected);
+  if (Array.isArray(protectedProperties)) {
+    // Cache the protected properties as an object for quick lookup
+    settings.protectedProperties = {};
+    for (var i = 0; i < protectedProperties.length; i++) {
+      settings.protectedProperties[protectedProperties[i]] = true;
+    }
+    protectedProperties = settings.protectedProperties;
+  }
+  if (protectedProperties) {
+    return protectedProperties[propertyName];
+  } else {
+    return false;
+  }
+};
+
 ModelBaseClass.isHiddenProperty = function (propertyName) {
   var Model = this;
   var settings = Model.definition && Model.definition.settings;
@@ -412,10 +453,10 @@ ModelBaseClass.isHiddenProperty = function (propertyName) {
   } else {
     return false;
   }
-}
+};
 
 ModelBaseClass.prototype.toJSON = function () {
-  return this.toObject(false, true);
+  return this.toObject(false, true, false);
 };
 
 ModelBaseClass.prototype.fromObject = function (obj) {
@@ -492,5 +533,52 @@ ModelBaseClass.prototype.setStrict = function (strict) {
   this.__strict = strict;
 };
 
+/**
+ * Register an asynchronous observer for the given operation (event).
+ * @param {String} operation The operation name.
+ * @callback {function} listener The listener function. It will be invoked with
+ * `this` set to the model constructor, e.g. `User`.
+ * @param {Object} context Operation-specific context.
+ * @param {function(Error=)} next The callback to call when the observer
+ *   has finished.
+ * @end
+ */
+ModelBaseClass.observe = function(operation, listener) {
+  if (!this._observers[operation]) {
+    this._observers[operation] = [];
+  }
+
+  this._observers[operation].push(listener);
+};
+
+/**
+ * Invoke all async observers for the given operation.
+ * @param {String} operation The operation name.
+ * @param {Object} context Operation-specific context.
+ * @param {function(Error=)} callback The callback to call when all observers
+ *   has finished.
+ */
+ModelBaseClass.notifyObserversOf = function(operation, context, callback) {
+  var observers = this._observers && this._observers[operation];
+
+  this._notifyBaseObservers(operation, context, function doNotify(err) {
+    if (err) return callback(err, context);
+    if (!observers || !observers.length) return callback(null, context);
+
+    async.eachSeries(
+      observers,
+      function(fn, next) { fn(context, next); },
+      function(err) { callback(err, context) }
+    );
+  });
+}
+
+ModelBaseClass._notifyBaseObservers = function(operation, context, callback) {
+  if (this.base && this.base.notifyObserversOf)
+    this.base.notifyObserversOf(operation, context, callback);
+  else
+    callback();
+}
+
 jutil.mixin(ModelBaseClass, Hookable);
 jutil.mixin(ModelBaseClass, validations.Validatable);

lib/relation-definition.js

@@ -208,11 +208,20 @@ RelationDefinition.prototype.applyProperties = function(modelInstance, obj) {
   if (this.options.invertProperties) {
     source = obj, target = modelInstance;
   }
+  if (this.options.embedsProperties) {
+    target = target.__data[this.name] = {};
+    target[this.keyTo] = source[this.keyTo];
+  }
   if (typeof this.properties === 'function') {
-    var data = this.properties.call(this, source);
+    var data = this.properties.call(this, source, target);
     for(var k in data) {
       target[k] = data[k];
     }
+  } else if (Array.isArray(this.properties)) {
+    for(var k = 0; k < this.properties.length; k++) {
+      var key = this.properties[k];
+      target[key] = source[key];
+    }
   } else if (typeof this.properties === 'object') {
     for(var k in this.properties) {
       var key = this.properties[k];
@@ -1309,7 +1318,7 @@ BelongsTo.prototype.related = function (refresh, params) {
     }
     
     var cb = params;
-    if (cachedValue === undefined) {
+    if (cachedValue === undefined || !(cachedValue instanceof ModelBaseClass)) {
       var query = {where: {}};
       query.where[pk] = modelInstance[fk];
       
@@ -1481,11 +1490,25 @@ RelationDefinition.hasOne = function (modelFrom, modelTo, params) {
   // FIXME: [rfeng] Wrap the property into a function for remoting
   // so that it can be accessed as /api/<model>/<id>/<hasOneRelationName>
   // For example, /api/orders/1/customer
-  var fn = function() {
+  modelFrom.prototype['__get__' + relationName] = function() {
     var f = this[relationName];
     f.apply(this, arguments);
   };
-  modelFrom.prototype['__get__' + relationName] = fn;
+
+  modelFrom.prototype['__create__' + relationName] = function() {
+    var f = this[relationName].create;
+    f.apply(this, arguments);
+  };
+
+  modelFrom.prototype['__update__' + relationName] = function() {
+    var f = this[relationName].update;
+    f.apply(this, arguments);
+  };
+
+  modelFrom.prototype['__destroy__' + relationName] = function() {
+    var f = this[relationName].destroy;
+    f.apply(this, arguments);
+  };
 
   return definition;
 };
@@ -1536,10 +1559,12 @@ HasOne.prototype.create = function (targetModelData, cb) {
   });
 };
 
-HasOne.prototype.update = function (targetModelData, cb) {
+HasOne.prototype.update = function(targetModelData, cb) {
   var definition = this.definition;
+  var fk = this.definition.keyTo;
   this.fetch(function(err, targetModel) {
     if (targetModel instanceof ModelBaseClass) {
+      delete targetModelData[fk];
       targetModel.updateAttributes(targetModelData, cb);
     } else {
       cb(new Error('HasOne relation ' + definition.name

lib/validations.js

@@ -589,8 +589,7 @@ var defaultMessages = {
 };
 
 function nullCheck(attr, conf, err) {
-  var isNull = this[attr] === null || !(attr in this);
+  if (this[attr] == null) {
-  if (isNull) {
     if (!conf.allowNull) {
       err('null');
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "loopback-datasource-juggler",
-  "version": "2.14.1",
+  "version": "2.15.0",
   "description": "LoopBack DataSoure Juggler",
   "keywords": [
     "StrongLoop",
@@ -23,8 +23,8 @@
     "node >= 0.6"
   ],
   "devDependencies": {
-    "should": "~1.2.2",
+    "mocha": "~1.20.1",
-    "mocha": "~1.20.1"
+    "should": "^1.3.0"
   },
   "dependencies": {
     "async": "~0.9.0",

test/async-observer.test.js

@@ -0,0 +1,103 @@
+var ModelBuilder = require('../').ModelBuilder;
+var should = require('./init');
+
+describe('async observer', function() {
+  var TestModel;
+  beforeEach(function defineTestModel() {
+    var modelBuilder = new ModelBuilder();
+    TestModel = modelBuilder.define('TestModel', { name: String });
+  });
+
+  it('calls registered async observers', function(done) {
+    var notifications = [];
+    TestModel.observe('before', pushAndNext(notifications, 'before'));
+    TestModel.observe('after', pushAndNext(notifications, 'after'));
+
+    TestModel.notifyObserversOf('before', {}, function(err) {
+      if (err) return done(err);
+      notifications.push('call');
+      TestModel.notifyObserversOf('after', {}, function(err) {
+        if (err) return done(err);
+
+        notifications.should.eql(['before', 'call', 'after']);
+        done();
+      });
+    });
+  });
+
+  it('allows multiple observers for the same operation', function(done) {
+    var notifications = [];
+    TestModel.observe('event', pushAndNext(notifications, 'one'));
+    TestModel.observe('event', pushAndNext(notifications, 'two'));
+
+    TestModel.notifyObserversOf('event', {}, function(err) {
+      if (err) return done(err);
+      notifications.should.eql(['one', 'two']);
+      done();
+    });
+  });
+
+  it('inherits observers from base model', function(done) {
+    var notifications = [];
+    TestModel.observe('event', pushAndNext(notifications, 'base'));
+
+    var Child = TestModel.extend('Child');
+    Child.observe('event', pushAndNext(notifications, 'child'));
+
+    Child.notifyObserversOf('event', {}, function(err) {
+      if (err) return done(err);
+      notifications.should.eql(['base', 'child']);
+      done();
+    });
+  });
+
+  it('does not modify observers in the base model', function(done) {
+    var notifications = [];
+    TestModel.observe('event', pushAndNext(notifications, 'base'));
+
+    var Child = TestModel.extend('Child');
+    Child.observe('event', pushAndNext(notifications, 'child'));
+
+    TestModel.notifyObserversOf('event', {}, function(err) {
+      if (err) return done(err);
+      notifications.should.eql(['base']);
+      done();
+    });
+  });
+
+  it('always calls inherited observers', function(done) {
+    var notifications = [];
+    TestModel.observe('event', pushAndNext(notifications, 'base'));
+
+    var Child = TestModel.extend('Child');
+    // Important: there are no observers on the Child model
+
+    Child.notifyObserversOf('event', {}, function(err) {
+      if (err) return done(err);
+      notifications.should.eql(['base']);
+      done();
+    });
+  });
+
+  it('handles no observers', function(done) {
+    TestModel.notifyObserversOf('no-observers', {}, function(err) {
+      // the test passes when no error was raised
+      done(err);
+    });
+  });
+
+  it('passes context to final callback', function(done) {
+    var context = {};
+    TestModel.notifyObserversOf('event', context, function(err, ctx) {
+      (ctx || "null").should.equal(context);
+      done();
+    });
+  });
+});
+
+function pushAndNext(array, value) {
+  return function(ctx, next) {
+    array.push(value);
+    process.nextTick(next);
+  };
+}

test/hooks.test.js

@@ -445,7 +445,7 @@ function addHooks(name, done) {
   };
   User['after' + name] = function (next) {
     (new Boolean(called)).should.equal(true);
-    this.email.should.equal(random);
+    this.should.have.property('email', random);
     done();
   };
 }

test/loopback-dl.test.js

@@ -243,6 +243,19 @@ describe('ModelBuilder define model', function () {
     done(null, User);
   });
 
+  it('should define an id property for composite ids', function () {
+    var modelBuilder = new ModelBuilder();
+    var Follow = modelBuilder.define('Follow', {
+      followerId: { type: String, id: 1 },
+      followeeId: { type: String, id: 2 },
+      followAt: Date
+    });
+    var follow = new Follow({ followerId: 1, followeeId: 2 });
+
+    follow.should.have.property('id');
+    assert.deepEqual(follow.id, { followerId: 1, followeeId: 2 });
+  });
+
 });
 
 describe('DataSource ping', function() {

test/manipulation.test.js

@@ -22,6 +22,22 @@ describe('manipulation', function () {
 
   });
 
+  // A simplified implementation of LoopBack's User model
+  // to reproduce problems related to properties with dynamic setters
+  // For the purpose of the tests, we use a counter instead of a hash fn.
+  var StubUser, stubPasswordCounter;
+  before(function setupStubUserModel(done) {
+    StubUser = db.createModel('StubUser', { password: String }, { forceId: true });
+    StubUser.setter.password = function(plain) {
+      this.$password = plain + '-' + (++stubPasswordCounter);
+    };
+    db.automigrate('StubUser', done);
+  });
+
+  beforeEach(function resetStubPasswordCounter() {
+    stubPasswordCounter = 0;
+  });
+
   describe('create', function () {
 
     before(function (done) {
@@ -153,6 +169,29 @@ describe('manipulation', function () {
         }).should.be.instanceOf(Array);
       }).should.have.lengthOf(3);
     });
+
+    it('should create batch of objects with beforeCreate', function(done) {
+      Person.beforeCreate = function(next, data) {
+        if (data && data.name === 'A') {
+          return next(null, {id: 'a', name: 'A'});
+        } else {
+          return next();
+        }
+      };
+      var batch = [
+        {name: 'A'},
+        {name: 'B'},
+        undefined
+      ];
+      Person.create(batch, function(e, ps) {
+        should.not.exist(e);
+        should.exist(ps);
+        ps.should.be.instanceOf(Array);
+        ps.should.have.lengthOf(batch.length);
+        ps[0].should.be.eql({id: 'a', name: 'A'});
+        done();
+      });
+    });
   });
 
   describe('save', function () {
@@ -214,6 +253,23 @@ describe('manipulation', function () {
       });
     });
 
+    it('should preserve properties with dynamic setters', function(done) {
+      // This test reproduces a problem discovered by LoopBack unit-test
+      // "User.hasPassword() should match a password after it is changed"
+      StubUser.create({ password: 'foo' }, function(err, created) {
+        if (err) return done(err);
+        created.password = 'bar';
+        created.save(function(err, saved) {
+          if (err) return done(err);
+          saved.password.should.equal('bar-2');
+          StubUser.findById(created.id, function(err, found) {
+            if (err) return done(err);
+            found.password.should.equal('bar-2');
+            done();
+          });
+        });
+      });
+    });
   });
 
   describe('updateAttributes', function () {
@@ -221,7 +277,7 @@ describe('manipulation', function () {
 
     before(function (done) {
       Person.destroyAll(function () {
-        person = Person.create(done);
+        person = Person.create({name: 'Mary', age: 15}, done);
       });
     });
 
@@ -236,6 +292,63 @@ describe('manipulation', function () {
         });
       });
     });
+
+    it('should ignore undefined values on updateAttributes', function(done) {
+      person.updateAttributes({'name': 'John', age: undefined},
+        function(err, p) {
+          should.not.exist(err);
+          Person.findById(p.id, function(e, p) {
+            should.not.exist(err);
+            p.name.should.equal('John');
+            p.age.should.equal(15);
+            done();
+          });
+        });
+    });
+
+    it('should allows model instance on updateAttributes', function(done) {
+      person.updateAttributes(new Person({'name': 'John', age: undefined}),
+        function(err, p) {
+          should.not.exist(err);
+          Person.findById(p.id, function(e, p) {
+            should.not.exist(err);
+            p.name.should.equal('John');
+            p.age.should.equal(15);
+            done();
+          });
+        });
+    });
+
+  });
+
+  describe('updateOrCreate', function() {
+    it('should preserve properties with dynamic setters on create', function(done) {
+      StubUser.updateOrCreate({ id: 'newid', password: 'foo' }, function(err, created) {
+        if (err) return done(err);
+        created.password.should.equal('foo-1');
+        StubUser.findById(created.id, function(err, found) {
+          if (err) return done(err);
+          found.password.should.equal('foo-1');
+          done();
+        });
+      });
+    });
+
+    it('should preserve properties with dynamic setters on update', function(done) {
+      StubUser.create({ password: 'foo' }, function(err, created) {
+        if (err) return done(err);
+        var data = { id: created.id, password: 'bar' };
+        StubUser.updateOrCreate(data, function(err, updated) {
+          if (err) return done(err);
+          updated.password.should.equal('bar-2');
+          StubUser.findById(created.id, function(err, found) {
+            if (err) return done(err);
+            found.password.should.equal('bar-2');
+            done();
+          });
+        });
+      });
+    });
   });
 
   describe('destroy', function () {

test/memory.test.js

@@ -5,12 +5,13 @@ var fs = require('fs');
 var assert = require('assert');
 var async = require('async');
 var should = require('./init.js');
+var Memory = require('../lib/connectors/memory').Memory;
 
-describe('Memory connector', function () {
+describe('Memory connector', function() {
   var file = path.join(__dirname, 'memory.json');
 
   function readModels(done) {
-    fs.readFile(file, function (err, data) {
+    fs.readFile(file, function(err, data) {
       var json = JSON.parse(data.toString());
       assert(json.models);
       assert(json.ids.User);
@@ -18,82 +19,122 @@ describe('Memory connector', function () {
     });
   }
 
-  before(function (done) {
+  before(function(done) {
-    fs.unlink(file, function (err) {
+    fs.unlink(file, function(err) {
       if (!err || err.code === 'ENOENT') {
         done();
       }
     });
   });
 
-  it('should save to a json file', function (done) {
+  describe('with file', function() {
+    function createUserModel() {
       var ds = new DataSource({
         connector: 'memory',
         file: file
       });
 
       var User = ds.createModel('User', {
+        id: {
+          type: Number,
+          id: true,
+          generated: true
+        },
         name: String,
         bio: String,
         approved: Boolean,
         joinedAt: Date,
         age: Number
       });
+      return User;
+    }
 
-    var count = 0;
+    var User;
     var ids = [];
-    async.eachSeries(['John1', 'John2', 'John3'], function (item, cb) {
+
-      User.create({name: item}, function (err, result) {
+    before(function() {
+      User = createUserModel();
+    });
+
+    it('should persist create', function(done) {
+      var count = 0;
+      async.eachSeries(['John1', 'John2', 'John3'], function(item, cb) {
+        User.create({name: item}, function(err, result) {
           ids.push(result.id);
           count++;
-        readModels(function (err, json) {
+          readModels(function(err, json) {
             assert.equal(Object.keys(json.models.User).length, count);
             cb(err);
           });
         });
-    }, function (err, results) {
+      }, done);
+    });
+
+    it('should persist delete', function(done) {
       // Now try to delete one
-      User.deleteById(ids[0], function (err) {
+      User.deleteById(ids[0], function(err) {
-        readModels(function (err, json) {
+        if (err) {
+          return done(err);
+        }
+        readModels(function(err, json) {
+          if (err) {
+            return done(err);
+          }
           assert.equal(Object.keys(json.models.User).length, 2);
-          User.upsert({id: ids[1], name: 'John'}, function(err, result) {
-            readModels(function (err, json) {
-              assert.equal(Object.keys(json.models.User).length, 2);
-              var user = JSON.parse(json.models.User[ids[1]]);
-              assert.equal(user.name, 'John');
           done();
         });
       });
     });
+
+    it('should persist upsert', function(done) {
+      User.upsert({id: ids[1], name: 'John'}, function(err, result) {
+        if (err) {
+          return done(err);
+        }
+        readModels(function(err, json) {
+          if (err) {
+            return done(err);
+          }
+          assert.equal(Object.keys(json.models.User).length, 2);
+          var user = JSON.parse(json.models.User[ids[1]]);
+          assert.equal(user.name, 'John');
+          assert(user.id === ids[1]);
+          done();
+        });
       });
     });
 
+    it('should persist update', function(done) {
+      User.update({id: ids[1]}, {name: 'John1'},
+        function(err, result) {
+          if (err) {
+            return done(err);
+          }
+          readModels(function(err, json) {
+            if (err) {
+              return done(err);
+            }
+            assert.equal(Object.keys(json.models.User).length, 2);
+            var user = JSON.parse(json.models.User[ids[1]]);
+            assert.equal(user.name, 'John1');
+            assert(user.id === ids[1]);
+            done();
+          });
+        });
     });
 
     // The saved memory.json from previous test should be loaded
-  it('should load from the json file', function (done) {
+    it('should load from the json file', function(done) {
-    var ds = new DataSource({
+      User.find(function(err, users) {
-      connector: 'memory',
-      file: file
-    });
-
-    var User = ds.createModel('User', {
-      name: String,
-      bio: String,
-      approved: Boolean,
-      joinedAt: Date,
-      age: Number
-    });
-
-    User.find(function (err, users) {
         // There should be 2 records
         assert.equal(users.length, 2);
         done(err);
       });
 
     });
+  });
 
-  describe('Query for memory connector', function () {
+  describe('Query for memory connector', function() {
     var ds = new DataSource({
       connector: 'memory'
     });
@@ -109,82 +150,82 @@ describe('Memory connector', function () {
     });
 
     before(seed);
-    it('should allow to find using like', function (done) {
+    it('should allow to find using like', function(done) {
-      User.find({where: {name: {like: '%St%'}}}, function (err, posts) {
+      User.find({where: {name: {like: '%St%'}}}, function(err, posts) {
         should.not.exist(err);
         posts.should.have.property('length', 2);
         done();
       });
     });
 
-    it('should support like for no match', function (done) {
+    it('should support like for no match', function(done) {
-      User.find({where: {name: {like: 'M%XY'}}}, function (err, posts) {
+      User.find({where: {name: {like: 'M%XY'}}}, function(err, posts) {
         should.not.exist(err);
         posts.should.have.property('length', 0);
         done();
       });
     });
 
-    it('should allow to find using nlike', function (done) {
+    it('should allow to find using nlike', function(done) {
-      User.find({where: {name: {nlike: '%St%'}}}, function (err, posts) {
+      User.find({where: {name: {nlike: '%St%'}}}, function(err, posts) {
         should.not.exist(err);
         posts.should.have.property('length', 4);
         done();
       });
     });
 
-    it('should support nlike for no match', function (done) {
+    it('should support nlike for no match', function(done) {
-      User.find({where: {name: {nlike: 'M%XY'}}}, function (err, posts) {
+      User.find({where: {name: {nlike: 'M%XY'}}}, function(err, posts) {
         should.not.exist(err);
         posts.should.have.property('length', 6);
         done();
       });
     });
 
-    it('should throw if the like value is not string or regexp', function (done) {
+    it('should throw if the like value is not string or regexp', function(done) {
-      User.find({where: {name: {like: 123}}}, function (err, posts) {
+      User.find({where: {name: {like: 123}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should throw if the nlike value is not string or regexp', function (done) {
+    it('should throw if the nlike value is not string or regexp', function(done) {
-      User.find({where: {name: {nlike: 123}}}, function (err, posts) {
+      User.find({where: {name: {nlike: 123}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should throw if the inq value is not an array', function (done) {
+    it('should throw if the inq value is not an array', function(done) {
-      User.find({where: {name: {inq: '12'}}}, function (err, posts) {
+      User.find({where: {name: {inq: '12'}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should throw if the nin value is not an array', function (done) {
+    it('should throw if the nin value is not an array', function(done) {
-      User.find({where: {name: {nin: '12'}}}, function (err, posts) {
+      User.find({where: {name: {nin: '12'}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should throw if the between value is not an array', function (done) {
+    it('should throw if the between value is not an array', function(done) {
-      User.find({where: {name: {between: '12'}}}, function (err, posts) {
+      User.find({where: {name: {between: '12'}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should throw if the between value is not an array of length 2', function (done) {
+    it('should throw if the between value is not an array of length 2', function(done) {
-      User.find({where: {name: {between: ['12']}}}, function (err, posts) {
+      User.find({where: {name: {between: ['12']}}}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should support order with multiple fields', function (done) {
+    it('should support order with multiple fields', function(done) {
-      User.find({order: 'vip ASC, seq DESC'}, function (err, posts) {
+      User.find({order: 'vip ASC, seq DESC'}, function(err, posts) {
         should.not.exist(err);
         posts[0].seq.should.be.eql(4);
         posts[1].seq.should.be.eql(3);
@@ -192,8 +233,8 @@ describe('Memory connector', function () {
       });
     });
 
-    it('should sort undefined values to the end when ordered DESC', function (done) {
+    it('should sort undefined values to the end when ordered DESC', function(done) {
-      User.find({order: 'vip ASC, order DESC'}, function (err, posts) {
+      User.find({order: 'vip ASC, order DESC'}, function(err, posts) {
         should.not.exist(err);
 
         posts[4].seq.should.be.eql(1);
@@ -202,15 +243,15 @@ describe('Memory connector', function () {
       });
     });
 
-    it('should throw if order has wrong direction', function (done) {
+    it('should throw if order has wrong direction', function(done) {
-      User.find({order: 'seq ABC'}, function (err, posts) {
+      User.find({order: 'seq ABC'}, function(err, posts) {
         should.exist(err);
         done();
       });
     });
 
-    it('should support neq operator for number', function (done) {
+    it('should support neq operator for number', function(done) {
-      User.find({where: {seq: {neq: 4}}}, function (err, users) {
+      User.find({where: {seq: {neq: 4}}}, function(err, users) {
         should.not.exist(err);
         users.length.should.be.equal(5);
         for (var i = 0; i < users.length; i++) {
@@ -220,8 +261,8 @@ describe('Memory connector', function () {
       });
     });
 
-    it('should support neq operator for string', function (done) {
+    it('should support neq operator for string', function(done) {
-      User.find({where: {role: {neq: 'lead'}}}, function (err, users) {
+      User.find({where: {role: {neq: 'lead'}}}, function(err, users) {
         should.not.exist(err);
         users.length.should.be.equal(4);
         for (var i = 0; i < users.length; i++) {
@@ -233,8 +274,8 @@ describe('Memory connector', function () {
       });
     });
 
-    it('should support neq operator for null', function (done) {
+    it('should support neq operator for null', function(done) {
-      User.find({where: {role: {neq: null}}}, function (err, users) {
+      User.find({where: {role: {neq: null}}}, function(err, users) {
         should.not.exist(err);
         users.length.should.be.equal(2);
         for (var i = 0; i < users.length; i++) {
@@ -279,7 +320,7 @@ describe('Memory connector', function () {
 
   });
 
-  it('should use collection setting', function (done) {
+  it('should use collection setting', function(done) {
     var ds = new DataSource({
       connector: 'memory'
     });
@@ -359,6 +400,17 @@ describe('Memory connector', function () {
     });
   });
 
+  require('./persistence-hooks.suite')(
+    new DataSource({ connector: Memory }),
+    should);
+});
+
+describe('Unoptimized connector', function() {
+  var ds = new DataSource({ connector: Memory });
+  // disable optimized methods
+  ds.connector.updateOrCreate = false;
+
+  require('./persistence-hooks.suite')(ds, should);
 });
 
 

test/model-definition.test.js

@@ -215,6 +215,28 @@ describe('ModelDefinition class', function () {
     done();
   });
 
+  it('should sort id properties by its index', function () {
+    var modelBuilder = new ModelBuilder();
+
+    var User = new ModelDefinition(modelBuilder, 'User', {
+      userId: {type: String, id: 2},
+      userType: {type: String, id: 1},
+      name: "string",
+      bio: ModelBuilder.Text,
+      approved: Boolean,
+      joinedAt: Date,
+      age: "number"
+    });
+
+    var ids = User.ids();
+    assert.ok(Array.isArray(ids));
+    assert.equal(ids.length, 2);
+    assert.equal(ids[0].id, 1);
+    assert.equal(ids[0].name, 'userType');
+    assert.equal(ids[1].id, 2);
+    assert.equal(ids[1].name, 'userId');
+  });
+
   it('should report correct table/column names', function (done) {
     var modelBuilder = new ModelBuilder();
 
@@ -270,6 +292,45 @@ describe('ModelDefinition class', function () {
     assert(grandChild.prototype instanceof child);
   });
 
+  it('should serialize protected properties into JSON', function() {
+    var memory = new DataSource({connector: Memory});
+    var modelBuilder = memory.modelBuilder;
+    var ProtectedModel = memory.createModel('protected', {}, {
+      protected: ['protectedProperty']
+    });
+    var pm = new ProtectedModel({
+      id: 1, foo: 'bar', protectedProperty: 'protected'
+    });
+    var serialized = pm.toJSON();
+    assert.deepEqual(serialized, {
+      id: 1, foo: 'bar', protectedProperty: 'protected'
+    });
+  });
+
+  it('should not serialize protected properties of nested models into JSON', function(done){
+    var memory = new DataSource({connector: Memory});
+    var modelBuilder = memory.modelBuilder;
+    var Parent = memory.createModel('parent');
+    var Child = memory.createModel('child', {}, {protected: ['protectedProperty']});
+    Parent.hasMany(Child);
+    Parent.create({
+      name: 'parent'
+    }, function(err, parent) {
+      parent.children.create({
+        name: 'child',
+        protectedProperty: 'protectedValue'
+      }, function(err, child) {
+        Parent.find({include: 'children'}, function(err, parents) {
+          var serialized = parents[0].toJSON();
+          var child = serialized.children[0];
+          assert.equal(child.name, 'child');
+          assert.notEqual(child.protectedProperty, 'protectedValue');
+          done();
+        });
+      });
+    });
+  });
+
   it('should not serialize hidden properties into JSON', function () {
     var memory = new DataSource({connector: Memory});
     var modelBuilder = memory.modelBuilder;

test/relations.test.js

@@ -727,7 +727,7 @@ describe('relations', function () {
       Job = db.define('Job', {name: String, type: String});
 
       Category.hasMany(Job, {
-        properties: function(inst) {
+        properties: function(inst, target) {
           if (!inst.jobType) return; // skip
           return { type: inst.jobType };
         },
@@ -1718,6 +1718,45 @@ describe('relations', function () {
   
   });
 
+  describe('belongsTo with embed', function () {
+    var Person, Passport;
+
+    it('can be declared with embed and properties', function (done) {
+      Person = db.define('Person', {name: String, age: Number});
+      Passport = db.define('Passport', {name: String, notes: String});
+      Passport.belongsTo(Person, {
+        properties: ['name'],
+        options: { embedsProperties: true, invertProperties: true }
+      });
+      db.automigrate(done);
+    });
+
+    it('should create record with embedded data', function (done) {
+      Person.create({name: 'Fred', age: 36 }, function(err, person) {
+        var p = new Passport({ name: 'Passport', notes: 'Some notes...' });
+        p.person(person);
+        p.personId.should.equal(person.id);
+        var data = p.toObject(true);
+        data.person.id.should.equal(person.id);
+        data.person.name.should.equal('Fred');
+        p.save(function (err) {
+          should.not.exists(err);
+          done();
+        });
+      });
+    });
+
+    it('should find record with embedded data', function (done) {
+      Passport.findOne(function (err, p) {
+        should.not.exists(err);
+        var data = p.toObject(true);
+        data.person.id.should.equal(p.personId);
+        data.person.name.should.equal('Fred');
+        done();
+      });
+    });
+  });
+
   describe('hasOne', function () {
     var Supplier, Account;
     var supplierId, accountId;
@@ -1771,6 +1810,24 @@ describe('relations', function () {
       });
     });
 
+    it('should ignore the foreign key in the update', function(done) {
+      Supplier.create({name: 'Supplier 2'}, function (e, supplier) {
+        var sid = supplier.id;
+        Supplier.findById(supplierId, function(e, supplier) {
+          should.not.exist(e);
+          should.exist(supplier);
+          supplier.account.update({supplierName: 'Supplier A',
+              supplierId: sid},
+            function(err, act) {
+              should.not.exist(e);
+              act.supplierName.should.equal('Supplier A');
+              act.supplierId.should.equal(supplierId);
+              done();
+            });
+        });
+      });
+    });
+    
     it('should get the related item on scope', function(done) {
       Supplier.findById(supplierId, function(e, supplier) {
         should.not.exist(e);

test/validations.test.js

@@ -387,6 +387,30 @@ describe('validations', function () {
   describe('format', function () {
     it('should validate format');
     it('should overwrite default blank message with custom format message');
+
+    it('should skip missing values when allowing null', function () {
+      User.validatesFormatOf('email', { with: /^\S+@\S+\.\S+$/, allowNull: true });
+      var u = new User({});
+      u.isValid().should.be.true;
+    });
+
+    it('should skip null values when allowing null', function () {
+      User.validatesFormatOf('email', { with: /^\S+@\S+\.\S+$/, allowNull: true });
+      var u = new User({ email: null });
+      u.isValid().should.be.true;
+    });
+
+    it('should not skip missing values', function () {
+      User.validatesFormatOf('email', { with: /^\S+@\S+\.\S+$/ });
+      var u = new User({});
+      u.isValid().should.be.false;
+    });
+
+    it('should not skip null values', function () {
+      User.validatesFormatOf('email', { with: /^\S+@\S+\.\S+$/ });
+      var u = new User({ email: null });
+      u.isValid().should.be.false;
+    });
   });
 
   describe('numericality', function () {