Improve error handling in replication

Deprecate `Change.handleError`, it was used inconsistenly for a subset
of possible errors only. Rework all `Change` methods to always report
all errors to the caller via the callback.

Rework `PersistedModel` to report change-tracking errors via the
existing method `PersistedModel.handleChangeError`. This method
can be customized on a per-model basis to provide different error
handling.

The default implementation emits `error` event on the model class,
users can attach an event listener that can provide a custom error
handler.

NOTE: Unhandled `error` events crash the application by default.

common/models/change.js

@@ -9,6 +9,7 @@ var CJSON = {stringify: require('canonical-json')};
 var async = require('async');
 var assert = require('assert');
 var debug = require('debug')('loopback:change');
+var deprecate = require('depd')('loopback');
 
 /**
  * Change list entry.
@@ -73,18 +74,43 @@ module.exports = function(Change) {
    */
 
   Change.rectifyModelChanges = function(modelName, modelIds, callback) {
-    var tasks = [];
     var Change = this;
+    var errors = [];
 
-    modelIds.forEach(function(id) {
+    var tasks = modelIds.map(function(id) {
-      tasks.push(function(cb) {
+      return function(cb) {
         Change.findOrCreateChange(modelName, id, function(err, change) {
-          if (err) return Change.handleError(err, cb);
+          if (err) return next(err);
-          change.rectify(cb);
+          change.rectify(next);
         });
-      });
+
+        function next(err) {
+          if (err) {
+            err.modelName = modelName;
+            err.modelId = id;
+            errors.push(err);
+          }
+          cb();
+        }
+      };
+    });
+
+    async.parallel(tasks, function(err) {
+      if (err) return callback(err);
+      if (errors.length) {
+        var desc = errors
+          .map(function(e) {
+            return '#' + e.modelId + ' - ' + e.toString();
+          })
+          .join('\n');
+
+        var msg = 'Cannot rectify ' + modelName + ' changes:\n' + desc;
+        err = new Error(msg);
+        err.details = { errors: errors };
+        return callback(err);
+      }
+      callback();
     });
-    async.parallel(tasks, callback);
   };
 
   /**
@@ -217,7 +243,7 @@ module.exports = function(Change) {
     var model = this.getModelCtor();
     var id = this.getModelId();
     model.findById(id, function(err, inst) {
-      if (err) return Change.handleError(err, cb);
+      if (err) return cb(err);
       if (inst) {
         cb(null, Change.revisionForInst(inst));
       } else {
@@ -459,6 +485,9 @@ module.exports = function(Change) {
   };
 
   Change.handleError = function(err) {
+    deprecate('Change.handleError is deprecated, ' +
+      'you should pass errors to your callback instead.');
+
     if (!this.settings.ignoreErrors) {
       throw err;
     }

lib/persisted-model.js

@@ -1038,9 +1038,8 @@ PersistedModel.createUpdates = function(deltas, cb) {
           Model.findById(change.modelId, function(err, inst) {
             if (err) return cb(err);
             if (!inst) {
-              console.error('missing data for change:', change);
               return cb &&
-                cb(new Error('missing data for change: ' + change.modelId));
+                cb(new Error('Missing data for change: ' + change.modelId));
             }
             if (inst.toObject) {
               update.data = inst.toObject();
@@ -1343,8 +1342,7 @@ PersistedModel.enableChangeTracking = function() {
   function cleanup() {
     Model.rectifyAllChanges(function(err) {
       if (err) {
-        console.error(Model.modelName + ' Change Cleanup Error:');
+        Model.handleChangeError(err, 'cleanup');
-        console.error(err);
       }
     });
   }
@@ -1359,8 +1357,7 @@ function rectifyOnSave(ctx, next) {
 
   function reportErrorAndNext(err) {
     if (err) {
-      console.error(
+      ctx.Model.handleChangeError(err, 'after save');
-        ctx.Model.modelName + '.rectifyChange(s) after save failed:' + err);
     }
     next();
   }
@@ -1378,8 +1375,7 @@ function rectifyOnDelete(ctx, next) {
 
   function reportErrorAndNext(err) {
     if (err) {
-      console.error(
+      ctx.Model.handleChangeError(err, 'after delete');
-        ctx.Model.modelName + '.rectifyChange(s) after delete failed:' + err);
     }
     next();
   }
@@ -1426,11 +1422,9 @@ PersistedModel.rectifyAllChanges = function(callback) {
  * @param {Error} err Error object; see [Error object](http://docs.strongloop.com/display/LB/Error+object).
  */
 
-PersistedModel.handleChangeError = function(err) {
+PersistedModel.handleChangeError = function(err, operationName) {
-  if (err) {
+  if (!err) return;
-    console.error(Model.modelName + ' Change Tracking Error:');
+  this.emit('error', err, operationName);
-    console.error(err);
-  }
 };
 
 /**