Use options from request for settings

lib/connectors/memory.js

@@ -323,7 +323,7 @@ Memory.prototype.upsertWithWhere = function(model, where, data, options, callbac
   }
 };
 
-Memory.prototype.findOrCreate = function(model, filter, data, callback) {
+Memory.prototype.findOrCreate = function(model, filter, data, options, callback) {
   var self = this;
   var nodes = self._findAllSkippingIncludes(model, filter);
   var found = nodes[0];
@@ -345,7 +345,7 @@ Memory.prototype.findOrCreate = function(model, filter, data, callback) {
     });
   }
 
-  self._models[model].model.include(nodes[0], filter.include, {}, function(err, nodes) {
+  self._models[model].model.include(nodes[0], filter.include, options, function(err, nodes) {
     process.nextTick(function() {
       if (err) return callback(err);
       callback(null, nodes[0], false);

lib/dao.js

@@ -18,6 +18,7 @@ module.exports = DataAccessObject;
 var g = require('strong-globalize')();
 var async = require('async');
 var jutil = require('./jutil');
+var DataAccessObjectUtils = require('./model-utils');
 var ValidationError = require('./validations').ValidationError;
 var Relation = require('./relations.js');
 var Inclusion = require('./include.js');
@@ -229,22 +230,6 @@ DataAccessObject.getConnector = function() {
   return this.getDataSource().connector;
 };
 
-/**
- * Verify if allowExtendedOperators is enabled
- * @options {Object} [options] Optional options to use.
- * @property {Boolean} allowExtendedOperators.
- * @returns {Boolean} Returns `true` if allowExtendedOperators is enabled, else `false`.
- */
-DataAccessObject._allowExtendedOperators = function(options) {
-  return this._getSetting('allowExtendedOperators', options) === true;
-};
-
-// Empty callback function
-function noCallback(err, result) {
-  // NOOP
-  debug('callback is ignored: err=%j, result=%j', err, result);
-}
-
 /**
  * Create an instance of Model with given data and save to the attached data source. Callback is optional.
  * Example:
@@ -391,7 +376,7 @@ DataAccessObject.create = function(data, options, cb) {
     obj.trigger('create', function(createDone) {
       obj.trigger('save', function(saveDone) {
         var _idName = idName(Model);
-        var val = Model._sanitizeData(obj.toObject(true));
+        var val = Model._sanitizeData(obj.toObject(true), options);
         function createCallback(err, id, rev) {
           if (id) {
             obj.__data[_idName] = id;
@@ -620,7 +605,7 @@ DataAccessObject.upsert = function(data, options, cb) {
         }
 
         function callConnector() {
-          update = Model._sanitizeData(update);
+          update = Model._sanitizeData(update, options);
           context = {
             Model: Model,
             where: ctx.where,
@@ -790,7 +775,7 @@ DataAccessObject.upsertWithWhere = function(where, data, options, cb) {
           try {
             ctx.where = Model._sanitizeQuery(ctx.where, options);
             ctx.where = Model._coerce(ctx.where, options);
-            update = Model._sanitizeData(update);
+            update = Model._sanitizeData(update, options);
             update = Model._coerce(update, options);
           } catch (err) {
             return process.nextTick(function() {
@@ -971,7 +956,7 @@ DataAccessObject.replaceOrCreate = function replaceOrCreate(data, options, cb) {
           }, update, options);
 
           function callConnector() {
-            update = Model._sanitizeData(update);
+            update = Model._sanitizeData(update, options);
             context = {
               Model: Model,
               where: where,
@@ -1152,7 +1137,7 @@ DataAccessObject.findOrCreate = function findOrCreate(query, data, options, cb)
       });
     }
 
-    data = Model._sanitizeData(data);
+    data = Model._sanitizeData(data, options);
     var context = {
       Model: Model,
       where: query.where,
@@ -1437,478 +1422,6 @@ DataAccessObject.all = function() {
   return DataAccessObject.find.apply(this, arguments);
 };
 
-/**
- * Get settings via hierarchical determination
- * - method level options
- * - model level settings
- * - datasource level settings
- *
- * @param {String} key The setting key
- */
-DataAccessObject._getSetting = function(key, options) {
-  // Check method level options
-  var val = options && options[key];
-  if (val !== undefined) return val;
-  // Check for settings in model
-  var m = this.definition;
-  if (m && m.settings) {
-    val = m.settings[key];
-    if (val !== undefined) {
-      return m.settings[key];
-    }
-    // Fall back to datasource level
-  }
-
-  // Check for settings in connector
-  var ds = this.getDataSource();
-  if (ds && ds.settings) {
-    return ds.settings[key];
-  }
-
-  return undefined;
-};
-
-var operators = {
-  eq: '=',
-  gt: '>',
-  gte: '>=',
-  lt: '<',
-  lte: '<=',
-  between: 'BETWEEN',
-  inq: 'IN',
-  nin: 'NOT IN',
-  neq: '!=',
-  like: 'LIKE',
-  nlike: 'NOT LIKE',
-  ilike: 'ILIKE',
-  nilike: 'NOT ILIKE',
-  regexp: 'REGEXP',
-};
-
-/*
- * Normalize the filter object and throw errors if invalid values are detected
- * @param {Object} filter The query filter object
- * @options {Object} [options] Optional options to use.
- * @property {Boolean} allowExtendedOperators.
- * @returns {Object} The normalized filter object
- * @private
- */
-DataAccessObject._normalize = function(filter, options) {
-  if (!filter) {
-    return undefined;
-  }
-  var err = null;
-  if ((typeof filter !== 'object') || Array.isArray(filter)) {
-    err = new Error(g.f('The query filter %j is not an {{object}}', filter));
-    err.statusCode = 400;
-    throw err;
-  }
-  if (filter.limit || filter.skip || filter.offset) {
-    var limit = Number(filter.limit || 100);
-    var offset = Number(filter.skip || filter.offset || 0);
-    if (isNaN(limit) || limit <= 0 || Math.ceil(limit) !== limit) {
-      err = new Error(g.f('The {{limit}} parameter %j is not valid',
-        filter.limit));
-      err.statusCode = 400;
-      throw err;
-    }
-    if (isNaN(offset) || offset < 0 || Math.ceil(offset) !== offset) {
-      err = new Error(g.f('The {{offset/skip}} parameter %j is not valid',
-        filter.skip || filter.offset));
-      err.statusCode = 400;
-      throw err;
-    }
-    filter.limit = limit;
-    filter.offset = offset;
-    filter.skip = offset;
-  }
-
-  if (filter.order) {
-    var order = filter.order;
-    if (!Array.isArray(order)) {
-      order = [order];
-    }
-    var fields = [];
-    for (var i = 0, m = order.length; i < m; i++) {
-      if (typeof order[i] === 'string') {
-        // Normalize 'f1 ASC, f2 DESC, f3' to ['f1 ASC', 'f2 DESC', 'f3']
-        var tokens = order[i].split(/(?:\s*,\s*)+/);
-        for (var t = 0, n = tokens.length; t < n; t++) {
-          var token = tokens[t];
-          if (token.length === 0) {
-            // Skip empty token
-            continue;
-          }
-          var parts = token.split(/\s+/);
-          if (parts.length >= 2) {
-            var dir = parts[1].toUpperCase();
-            if (dir === 'ASC' || dir === 'DESC') {
-              token = parts[0] + ' ' + dir;
-            } else {
-              err = new Error(g.f('The {{order}} %j has invalid direction', token));
-              err.statusCode = 400;
-              throw err;
-            }
-          }
-          fields.push(token);
-        }
-      } else {
-        err = new Error(g.f('The order %j is not valid', order[i]));
-        err.statusCode = 400;
-        throw err;
-      }
-    }
-    if (fields.length === 1 && typeof filter.order === 'string') {
-      filter.order = fields[0];
-    } else {
-      filter.order = fields;
-    }
-  }
-
-  // normalize fields as array of included property names
-  if (filter.fields) {
-    filter.fields = fieldsToArray(filter.fields,
-      Object.keys(this.definition.properties), this.settings.strict);
-  }
-
-  filter = this._sanitizeQuery(filter, options);
-  this._coerce(filter.where, options);
-  return filter;
-};
-
-function DateType(arg) {
-  var d = new Date(arg);
-  if (isNaN(d.getTime())) {
-    throw new Error(g.f('Invalid date: %s', arg));
-  }
-  return d;
-}
-
-function BooleanType(arg) {
-  if (typeof arg === 'string') {
-    switch (arg) {
-      case 'true':
-      case '1':
-        return true;
-      case 'false':
-      case '0':
-        return false;
-    }
-  }
-  if (arg == null) {
-    return null;
-  }
-  return Boolean(arg);
-}
-
-function NumberType(val) {
-  var num = Number(val);
-  return !isNaN(num) ? num : val;
-}
-
-function coerceArray(val) {
-  if (Array.isArray(val)) {
-    return val;
-  }
-
-  if (!utils.isPlainObject(val)) {
-    throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices'));
-  }
-
-  // It is an object, check if empty
-  var props = Object.keys(val);
-
-  if (props.length === 0) {
-    throw new Error(g.f('Value is an empty {{object}}'));
-  }
-
-  var arrayVal = new Array(props.length);
-  for (var i = 0; i < arrayVal.length; ++i) {
-    if (!val.hasOwnProperty(i)) {
-      throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices'));
-    }
-
-    arrayVal[i] = val[i];
-  }
-
-  return arrayVal;
-}
-
-function _normalizeAsArray(result) {
-  if (typeof result === 'string') {
-    result = [result];
-  }
-  if (Array.isArray(result)) {
-    return result;
-  } else {
-    // See https://github.com/strongloop/loopback-datasource-juggler/issues/1646
-    // `ModelBaseClass` normalize the properties to an object such as `{secret: true}`
-    var keys = [];
-    for (var k in result) {
-      if (result[k]) keys.push(k);
-    }
-    return keys;
-  }
-}
-
-DataAccessObject._getHiddenProperties = function() {
-  var settings = this.definition.settings || {};
-  var result = settings.hiddenProperties || settings.hidden || [];
-  return _normalizeAsArray(result);
-};
-
-DataAccessObject._getProtectedProperties = function() {
-  var settings = this.definition.settings || {};
-  var result = settings.protectedProperties || settings.protected || [];
-  return _normalizeAsArray(result);
-};
-
-DataAccessObject._sanitizeQuery = function(query, options) {
-  options = options || {};
-
-  // Get settings to normalize `undefined` values
-  var normalizeUndefinedInQuery = this._getSetting('normalizeUndefinedInQuery', options);
-  // Get setting to prohibit hidden/protected properties in query
-  var prohibitHiddenPropertiesInQuery = this._getSetting('prohibitHiddenPropertiesInQuery', options);
-  if (prohibitHiddenPropertiesInQuery == null) {
-    // By default, hidden properties are prohibited in query
-    prohibitHiddenPropertiesInQuery = true;
-  }
-
-  // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651
-  var maxDepthOfQuery = (+this._getSetting('maxDepthOfQuery', options)) || 12;
-
-  var prohibitedKeys = [];
-  // Check violation of keys
-  if (prohibitHiddenPropertiesInQuery) {
-    prohibitedKeys = this._getHiddenProperties();
-    if (options.prohibitProtectedPropertiesInQuery) {
-      prohibitedKeys = prohibitedKeys.concat(this._getProtectedProperties());
-    }
-  }
-  return sanitizeQueryOrData(query,
-    Object.assign({
-      maxDepth: maxDepthOfQuery,
-      prohibitedKeys: prohibitedKeys,
-      normalizeUndefinedInQuery: normalizeUndefinedInQuery,
-    }, options));
-};
-
-DataAccessObject._sanitizeData = function(data, options) {
-  options = options || {};
-  // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651
-  var maxDepthOfQuery = (+this._getSetting('maxDepthOfQuery', options)) || 12;
-  return sanitizeQueryOrData(data,
-    Object.assign({
-      maxDepth: maxDepthOfQuery,
-    }, options));
-};
-
-/*
- * Coerce values based the property types
- * @param {Object} where The where clause
- * @options {Object} [options] Optional options to use.
- * @property {Boolean} allowExtendedOperators.
- * @returns {Object} The coerced where clause
- * @private
- */
-DataAccessObject._coerce = function(where, options) {
-  var self = this;
-  if (where == null) {
-    return where;
-  }
-  options = options || {};
-
-  var err;
-  if (typeof where !== 'object' || Array.isArray(where)) {
-    err = new Error(g.f('The where clause %j is not an {{object}}', where));
-    err.statusCode = 400;
-    throw err;
-  }
-
-  var props = self.definition.properties;
-  for (var p in where) {
-    // Handle logical operators
-    if (p === 'and' || p === 'or' || p === 'nor') {
-      var clauses = where[p];
-      try {
-        clauses = coerceArray(clauses);
-      } catch (e) {
-        err = new Error(g.f('The %s operator has invalid clauses %j: %s', p, clauses, e.message));
-        err.statusCode = 400;
-        throw err;
-      }
-
-      for (var k = 0; k < clauses.length; k++) {
-        self._coerce(clauses[k], options);
-      }
-
-      continue;
-    }
-    var DataType = props[p] && props[p].type;
-    if (!DataType) {
-      continue;
-    }
-    if (Array.isArray(DataType) || DataType === Array) {
-      DataType = DataType[0];
-    }
-    if (DataType === Date) {
-      DataType = DateType;
-    } else if (DataType === Boolean) {
-      DataType = BooleanType;
-    } else if (DataType === Number) {
-      // This fixes a regression in mongodb connector
-      // For numbers, only convert it produces a valid number
-      // LoopBack by default injects a number id. We should fix it based
-      // on the connector's input, for example, MongoDB should use string
-      // while RDBs typically use number
-      DataType = NumberType;
-    }
-
-    if (!DataType) {
-      continue;
-    }
-
-    if (DataType.prototype instanceof BaseModel) {
-      continue;
-    }
-
-    if (DataType === geo.GeoPoint) {
-      // Skip the GeoPoint as the near operator breaks the assumption that
-      // an operation has only one property
-      // We should probably fix it based on
-      // http://docs.mongodb.org/manual/reference/operator/query/near/
-      // The other option is to make operators start with $
-      continue;
-    }
-
-    var val = where[p];
-    if (val === null || val === undefined) {
-      continue;
-    }
-    // Check there is an operator
-    var operator = null;
-    var exp = val;
-    if (val.constructor === Object) {
-      for (var op in operators) {
-        if (op in val) {
-          val = val[op];
-          operator = op;
-          switch (operator) {
-            case 'inq':
-            case 'nin':
-            case 'between':
-              try {
-                val = coerceArray(val);
-              } catch (e) {
-                err = new Error(g.f('The %s property has invalid clause %j: %s', p, where[p], e));
-                err.statusCode = 400;
-                throw err;
-              }
-
-              if (operator === 'between' && val.length !== 2) {
-                err = new Error(g.f(
-                  'The %s property has invalid clause %j: Expected precisely 2 values, received %d',
-                  p,
-                  where[p],
-                  val.length
-                ));
-                err.statusCode = 400;
-                throw err;
-              }
-              break;
-            case 'like':
-            case 'nlike':
-            case 'ilike':
-            case 'nilike':
-              if (!(typeof val === 'string' || val instanceof RegExp)) {
-                err = new Error(g.f(
-                  'The %s property has invalid clause %j: Expected a string or RegExp',
-                  p,
-                  where[p]
-                ));
-                err.statusCode = 400;
-                throw err;
-              }
-              break;
-            case 'regexp':
-              val = utils.toRegExp(val);
-              if (val instanceof Error) {
-                val.statusCode = 400;
-                throw val;
-              }
-              break;
-          }
-          break;
-        }
-      }
-    }
-
-    try {
-      // Coerce val into an array if it resembles an array-like object
-      val = coerceArray(val);
-    } catch (e) {
-      // NOOP when not coercable into an array.
-    }
-
-    var allowExtendedOperators = self._allowExtendedOperators(options);
-    // Coerce the array items
-    if (Array.isArray(val)) {
-      for (var i = 0; i < val.length; i++) {
-        if (val[i] !== null && val[i] !== undefined) {
-          if (!(val[i] instanceof RegExp)) {
-            val[i] = DataType(val[i]);
-          }
-        }
-      }
-    } else {
-      if (val != null) {
-        if (operator === null && val instanceof RegExp) {
-          // Normalize {name: /A/} to {name: {regexp: /A/}}
-          operator = 'regexp';
-        } else if (operator === 'regexp' && val instanceof RegExp) {
-          // Do not coerce regex literals/objects
-        } else if ((operator === 'like' || operator === 'nlike' ||
-            operator === 'ilike' || operator === 'nilike') && val instanceof RegExp) {
-          // Do not coerce RegExp operator value
-        } else if (allowExtendedOperators && typeof val === 'object') {
-          // Do not coerce object values when extended operators are allowed
-        } else {
-          if (!allowExtendedOperators) {
-            var extendedOperators = Object.keys(val).filter(function(k) {
-              return k[0] === '$';
-            });
-            if (extendedOperators.length) {
-              const msg = g.f('Operators "' + extendedOperators.join(', ') + '" are not allowed in query');
-              const err = new Error(msg);
-              err.code = 'OPERATOR_NOT_ALLOWED_IN_QUERY';
-              err.statusCode = 400;
-              err.details = {
-                operators: extendedOperators,
-                where: where,
-              };
-              throw err;
-            }
-          }
-          val = DataType(val);
-        }
-      }
-    }
-    // Rebuild {property: {operator: value}}
-    if (operator && operator !== 'eq') {
-      var value = {};
-      value[operator] = val;
-      if (exp.options) {
-        // Keep options for operators
-        value.options = exp.options;
-      }
-      val = value;
-    }
-    where[p] = val;
-  }
-  return where;
-};
-
 /**
  * Find all instances of Model that match the specified query.
  * Fields used for filter and sort should be declared with `{index: true}` in model definition.
@@ -2614,7 +2127,7 @@ DataAccessObject.prototype.save = function(options, cb) {
     function save() {
       inst.trigger('save', function(saveDone) {
         inst.trigger('update', function(updateDone) {
-          data = Model._sanitizeData(data);
+          data = Model._sanitizeData(data, options);
           function saveCallback(err, unusedData, result) {
             if (err) {
               return cb(err, inst);
@@ -2803,7 +2316,7 @@ DataAccessObject.updateAll = function(where, data, options, cb) {
       // alter configuration of how sanitizeQuery handles undefined values
       where = Model._sanitizeQuery(where, options);
       where = Model._coerce(where, options);
-      data = Model._sanitizeData(data);
+      data = Model._sanitizeData(data, options);
       data = Model._coerce(data, options);
     } catch (err) {
       return process.nextTick(function() {
@@ -3136,7 +2649,7 @@ DataAccessObject.replaceById = function(id, data, options, cb) {
 
     function validateAndCallConnector(err, data) {
       if (err) return cb(err);
-      data = Model._sanitizeData(data);
+      data = Model._sanitizeData(data, options);
       // update instance's properties
       inst.setAttributes(data);
 
@@ -3288,7 +2801,7 @@ function(data, options, cb) {
   if (data instanceof Model) {
     data = data.toObject(false);
   }
-  data = Model._sanitizeData(data);
+  data = Model._sanitizeData(data, options);
 
   // Make sure id(s) cannot be changed
   var idNames = Model.definition.idNames();
@@ -3361,7 +2874,7 @@ function(data, options, cb) {
           inst.trigger('update', function(done) {
             copyData(data, inst);
             var typedData = convertSubsetOfPropertiesByType(inst, data);
-            context.data = Model._sanitizeData(typedData);
+            context.data = Model._sanitizeData(typedData, options);
 
             // Depending on the connector, the database response can
             // contain information about the updated record(s), but also

lib/include.js

@@ -205,8 +205,9 @@ Inclusion.include = function(objects, include, options, cb) {
    */
   function findWithForeignKeysByPage(model, filter, fkName, pageSize, options, cb) {
     try {
-      model._sanitizeQuery(filter.where, {prohibitProtectedPropertiesInQuery: true});
+      const opts = Object.assign({prohibitProtectedPropertiesInQuery: true}, options);
-      model._coerce(filter.where);
+      model._sanitizeQuery(filter.where, opts);
+      model._coerce(filter.where, options);
     } catch (e) {
       return cb(e);
     }

lib/model-utils.js

@@ -0,0 +1,553 @@
+// Copyright IBM Corp. 2013,2016. All Rights Reserved.
+// Node module: loopback-datasource-juggler
+// This file is licensed under the MIT License.
+// License text available at https://opensource.org/licenses/MIT
+
+// Turning on strict for this file breaks lots of test cases;
+// disabling strict for this file
+/* eslint-disable strict */
+
+module.exports = ModelUtils;
+
+/*!
+ * Module dependencies
+ */
+var g = require('strong-globalize')();
+var geo = require('./geo');
+var utils = require('./utils');
+var fieldsToArray = utils.fieldsToArray;
+var sanitizeQueryOrData = utils.sanitizeQuery;
+var BaseModel = require('./model');
+
+/**
+ * A mixin to contain utility methods for DataAccessObject
+ */
+function ModelUtils() {
+}
+
+/**
+ * Verify if allowExtendedOperators is enabled
+ * @options {Object} [options] Optional options to use.
+ * @property {Boolean} allowExtendedOperators.
+ * @returns {Boolean} Returns `true` if allowExtendedOperators is enabled, else `false`.
+ */
+ModelUtils._allowExtendedOperators = function(options) {
+  const flag = this._getSetting('allowExtendedOperators', options);
+  if (flag != null) return !!flag;
+  // Default to `false`
+  return false;
+};
+
+/**
+ * Get settings via hierarchical determination
+ * - method level options
+ * - model level settings
+ * - data source level settings
+ *
+ * @param {String} key The setting key
+ */
+ModelUtils._getSetting = function(key, options) {
+  // Check method level options
+  var val = options && options[key];
+  if (val !== undefined) return val;
+  // Check for settings in model
+  var m = this.definition;
+  if (m && m.settings) {
+    val = m.settings[key];
+    if (val !== undefined) {
+      return m.settings[key];
+    }
+    // Fall back to data source level
+  }
+
+  // Check for settings in connector
+  var ds = this.getDataSource();
+  if (ds && ds.settings) {
+    return ds.settings[key];
+  }
+
+  return undefined;
+};
+
+var operators = {
+  eq: '=',
+  gt: '>',
+  gte: '>=',
+  lt: '<',
+  lte: '<=',
+  between: 'BETWEEN',
+  inq: 'IN',
+  nin: 'NOT IN',
+  neq: '!=',
+  like: 'LIKE',
+  nlike: 'NOT LIKE',
+  ilike: 'ILIKE',
+  nilike: 'NOT ILIKE',
+  regexp: 'REGEXP',
+};
+
+/*
+ * Normalize the filter object and throw errors if invalid values are detected
+ * @param {Object} filter The query filter object
+ * @options {Object} [options] Optional options to use.
+ * @property {Boolean} allowExtendedOperators.
+ * @returns {Object} The normalized filter object
+ * @private
+ */
+ModelUtils._normalize = function(filter, options) {
+  if (!filter) {
+    return undefined;
+  }
+  var err = null;
+  if ((typeof filter !== 'object') || Array.isArray(filter)) {
+    err = new Error(g.f('The query filter %j is not an {{object}}', filter));
+    err.statusCode = 400;
+    throw err;
+  }
+  if (filter.limit || filter.skip || filter.offset) {
+    var limit = Number(filter.limit || 100);
+    var offset = Number(filter.skip || filter.offset || 0);
+    if (isNaN(limit) || limit <= 0 || Math.ceil(limit) !== limit) {
+      err = new Error(g.f('The {{limit}} parameter %j is not valid',
+        filter.limit));
+      err.statusCode = 400;
+      throw err;
+    }
+    if (isNaN(offset) || offset < 0 || Math.ceil(offset) !== offset) {
+      err = new Error(g.f('The {{offset/skip}} parameter %j is not valid',
+        filter.skip || filter.offset));
+      err.statusCode = 400;
+      throw err;
+    }
+    filter.limit = limit;
+    filter.offset = offset;
+    filter.skip = offset;
+  }
+
+  if (filter.order) {
+    var order = filter.order;
+    if (!Array.isArray(order)) {
+      order = [order];
+    }
+    var fields = [];
+    for (var i = 0, m = order.length; i < m; i++) {
+      if (typeof order[i] === 'string') {
+        // Normalize 'f1 ASC, f2 DESC, f3' to ['f1 ASC', 'f2 DESC', 'f3']
+        var tokens = order[i].split(/(?:\s*,\s*)+/);
+        for (var t = 0, n = tokens.length; t < n; t++) {
+          var token = tokens[t];
+          if (token.length === 0) {
+            // Skip empty token
+            continue;
+          }
+          var parts = token.split(/\s+/);
+          if (parts.length >= 2) {
+            var dir = parts[1].toUpperCase();
+            if (dir === 'ASC' || dir === 'DESC') {
+              token = parts[0] + ' ' + dir;
+            } else {
+              err = new Error(g.f('The {{order}} %j has invalid direction', token));
+              err.statusCode = 400;
+              throw err;
+            }
+          }
+          fields.push(token);
+        }
+      } else {
+        err = new Error(g.f('The order %j is not valid', order[i]));
+        err.statusCode = 400;
+        throw err;
+      }
+    }
+    if (fields.length === 1 && typeof filter.order === 'string') {
+      filter.order = fields[0];
+    } else {
+      filter.order = fields;
+    }
+  }
+
+  // normalize fields as array of included property names
+  if (filter.fields) {
+    filter.fields = fieldsToArray(filter.fields,
+      Object.keys(this.definition.properties), this.settings.strict);
+  }
+
+  filter = this._sanitizeQuery(filter, options);
+  this._coerce(filter.where, options);
+  return filter;
+};
+
+function DateType(arg) {
+  var d = new Date(arg);
+  if (isNaN(d.getTime())) {
+    throw new Error(g.f('Invalid date: %s', arg));
+  }
+  return d;
+}
+
+function BooleanType(arg) {
+  if (typeof arg === 'string') {
+    switch (arg) {
+      case 'true':
+      case '1':
+        return true;
+      case 'false':
+      case '0':
+        return false;
+    }
+  }
+  if (arg == null) {
+    return null;
+  }
+  return Boolean(arg);
+}
+
+function NumberType(val) {
+  var num = Number(val);
+  return !isNaN(num) ? num : val;
+}
+
+function coerceArray(val) {
+  if (Array.isArray(val)) {
+    return val;
+  }
+
+  if (!utils.isPlainObject(val)) {
+    throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices'));
+  }
+
+  // It is an object, check if empty
+  var props = Object.keys(val);
+
+  if (props.length === 0) {
+    throw new Error(g.f('Value is an empty {{object}}'));
+  }
+
+  var arrayVal = new Array(props.length);
+  for (var i = 0; i < arrayVal.length; ++i) {
+    if (!val.hasOwnProperty(i)) {
+      throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices'));
+    }
+
+    arrayVal[i] = val[i];
+  }
+
+  return arrayVal;
+}
+
+function _normalizeAsArray(result) {
+  if (typeof result === 'string') {
+    result = [result];
+  }
+  if (Array.isArray(result)) {
+    return result;
+  } else {
+    // See https://github.com/strongloop/loopback-datasource-juggler/issues/1646
+    // `ModelBaseClass` normalize the properties to an object such as `{secret: true}`
+    var keys = [];
+    for (var k in result) {
+      if (result[k]) keys.push(k);
+    }
+    return keys;
+  }
+}
+
+/**
+ * Get an array of hidden property names
+ */
+ModelUtils._getHiddenProperties = function() {
+  var settings = this.definition.settings || {};
+  var result = settings.hiddenProperties || settings.hidden || [];
+  return _normalizeAsArray(result);
+};
+
+/**
+ * Get an array of protected property names
+ */
+ModelUtils._getProtectedProperties = function() {
+  var settings = this.definition.settings || {};
+  var result = settings.protectedProperties || settings.protected || [];
+  return _normalizeAsArray(result);
+};
+
+/**
+ * Get the maximum depth of a query object
+ */
+ModelUtils._getMaxDepthOfQuery = function(options, defaultValue) {
+  options = options || {};
+  // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651
+  var maxDepth = this._getSetting('maxDepthOfQuery', options);
+  if (maxDepth == null) {
+    maxDepth = defaultValue || 32;
+  }
+  return +maxDepth;
+};
+
+/**
+ * Get the maximum depth of a data object
+ */
+ModelUtils._getMaxDepthOfData = function(options, defaultValue) {
+  options = options || {};
+  // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651
+  var maxDepth = this._getSetting('maxDepthOfData', options);
+  if (maxDepth == null) {
+    maxDepth = defaultValue || 64;
+  }
+  return +maxDepth;
+};
+
+/**
+ * Get the prohibitHiddenPropertiesInQuery flag
+ */
+ModelUtils._getProhibitHiddenPropertiesInQuery = function(options, defaultValue) {
+  var flag = this._getSetting('prohibitHiddenPropertiesInQuery', options);
+  if (flag == null) return !!defaultValue;
+  return !!flag;
+};
+
+/**
+ * Sanitize the query object
+ */
+ModelUtils._sanitizeQuery = function(query, options) {
+  options = options || {};
+
+  // Get settings to normalize `undefined` values
+  var normalizeUndefinedInQuery = this._getSetting('normalizeUndefinedInQuery', options);
+  // Get setting to prohibit hidden/protected properties in query
+  var prohibitHiddenPropertiesInQuery = this._getProhibitHiddenPropertiesInQuery(options);
+
+  // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651
+  var maxDepthOfQuery = this._getMaxDepthOfQuery(options);
+
+  var prohibitedKeys = [];
+  // Check violation of keys
+  if (prohibitHiddenPropertiesInQuery) {
+    prohibitedKeys = this._getHiddenProperties();
+    if (options.prohibitProtectedPropertiesInQuery) {
+      prohibitedKeys = prohibitedKeys.concat(this._getProtectedProperties());
+    }
+  }
+  return sanitizeQueryOrData(query,
+    Object.assign({
+      maxDepth: maxDepthOfQuery,
+      prohibitedKeys: prohibitedKeys,
+      normalizeUndefinedInQuery: normalizeUndefinedInQuery,
+    }, options));
+};
+
+/**
+ * Sanitize the data object
+ */
+ModelUtils._sanitizeData = function(data, options) {
+  options = options || {};
+  return sanitizeQueryOrData(data,
+    Object.assign({
+      maxDepth: this._getMaxDepthOfData(options),
+    }, options));
+};
+
+/*
+ * Coerce values based the property types
+ * @param {Object} where The where clause
+ * @options {Object} [options] Optional options to use.
+ * @property {Boolean} allowExtendedOperators.
+ * @returns {Object} The coerced where clause
+ * @private
+ */
+ModelUtils._coerce = function(where, options) {
+  var self = this;
+  if (where == null) {
+    return where;
+  }
+  options = options || {};
+
+  var err;
+  if (typeof where !== 'object' || Array.isArray(where)) {
+    err = new Error(g.f('The where clause %j is not an {{object}}', where));
+    err.statusCode = 400;
+    throw err;
+  }
+
+  var props = self.definition.properties;
+  for (var p in where) {
+    // Handle logical operators
+    if (p === 'and' || p === 'or' || p === 'nor') {
+      var clauses = where[p];
+      try {
+        clauses = coerceArray(clauses);
+      } catch (e) {
+        err = new Error(g.f('The %s operator has invalid clauses %j: %s', p, clauses, e.message));
+        err.statusCode = 400;
+        throw err;
+      }
+
+      for (var k = 0; k < clauses.length; k++) {
+        self._coerce(clauses[k], options);
+      }
+
+      continue;
+    }
+    var DataType = props[p] && props[p].type;
+    if (!DataType) {
+      continue;
+    }
+    if (Array.isArray(DataType) || DataType === Array) {
+      DataType = DataType[0];
+    }
+    if (DataType === Date) {
+      DataType = DateType;
+    } else if (DataType === Boolean) {
+      DataType = BooleanType;
+    } else if (DataType === Number) {
+      // This fixes a regression in mongodb connector
+      // For numbers, only convert it produces a valid number
+      // LoopBack by default injects a number id. We should fix it based
+      // on the connector's input, for example, MongoDB should use string
+      // while RDBs typically use number
+      DataType = NumberType;
+    }
+
+    if (!DataType) {
+      continue;
+    }
+
+    if (DataType.prototype instanceof BaseModel) {
+      continue;
+    }
+
+    if (DataType === geo.GeoPoint) {
+      // Skip the GeoPoint as the near operator breaks the assumption that
+      // an operation has only one property
+      // We should probably fix it based on
+      // http://docs.mongodb.org/manual/reference/operator/query/near/
+      // The other option is to make operators start with $
+      continue;
+    }
+
+    var val = where[p];
+    if (val === null || val === undefined) {
+      continue;
+    }
+    // Check there is an operator
+    var operator = null;
+    var exp = val;
+    if (val.constructor === Object) {
+      for (var op in operators) {
+        if (op in val) {
+          val = val[op];
+          operator = op;
+          switch (operator) {
+            case 'inq':
+            case 'nin':
+            case 'between':
+              try {
+                val = coerceArray(val);
+              } catch (e) {
+                err = new Error(g.f('The %s property has invalid clause %j: %s', p, where[p], e));
+                err.statusCode = 400;
+                throw err;
+              }
+
+              if (operator === 'between' && val.length !== 2) {
+                err = new Error(g.f(
+                  'The %s property has invalid clause %j: Expected precisely 2 values, received %d',
+                  p,
+                  where[p],
+                  val.length
+                ));
+                err.statusCode = 400;
+                throw err;
+              }
+              break;
+            case 'like':
+            case 'nlike':
+            case 'ilike':
+            case 'nilike':
+              if (!(typeof val === 'string' || val instanceof RegExp)) {
+                err = new Error(g.f(
+                  'The %s property has invalid clause %j: Expected a string or RegExp',
+                  p,
+                  where[p]
+                ));
+                err.statusCode = 400;
+                throw err;
+              }
+              break;
+            case 'regexp':
+              val = utils.toRegExp(val);
+              if (val instanceof Error) {
+                val.statusCode = 400;
+                throw val;
+              }
+              break;
+          }
+          break;
+        }
+      }
+    }
+
+    try {
+      // Coerce val into an array if it resembles an array-like object
+      val = coerceArray(val);
+    } catch (e) {
+      // NOOP when not coercable into an array.
+    }
+
+    var allowExtendedOperators = self._allowExtendedOperators(options);
+    // Coerce the array items
+    if (Array.isArray(val)) {
+      for (var i = 0; i < val.length; i++) {
+        if (val[i] !== null && val[i] !== undefined) {
+          if (!(val[i] instanceof RegExp)) {
+            val[i] = DataType(val[i]);
+          }
+        }
+      }
+    } else {
+      if (val != null) {
+        if (operator === null && val instanceof RegExp) {
+          // Normalize {name: /A/} to {name: {regexp: /A/}}
+          operator = 'regexp';
+        } else if (operator === 'regexp' && val instanceof RegExp) {
+          // Do not coerce regex literals/objects
+        } else if ((operator === 'like' || operator === 'nlike' ||
+            operator === 'ilike' || operator === 'nilike') && val instanceof RegExp) {
+          // Do not coerce RegExp operator value
+        } else if (allowExtendedOperators && typeof val === 'object') {
+          // Do not coerce object values when extended operators are allowed
+        } else {
+          if (!allowExtendedOperators) {
+            var extendedOperators = Object.keys(val).filter(function(k) {
+              return k[0] === '$';
+            });
+            if (extendedOperators.length) {
+              const msg = g.f('Operators "' + extendedOperators.join(', ') + '" are not allowed in query');
+              const err = new Error(msg);
+              err.code = 'OPERATOR_NOT_ALLOWED_IN_QUERY';
+              err.statusCode = 400;
+              err.details = {
+                operators: extendedOperators,
+                where: where,
+              };
+              throw err;
+            }
+          }
+          val = DataType(val);
+        }
+      }
+    }
+    // Rebuild {property: {operator: value}}
+    if (operator && operator !== 'eq') {
+      var value = {};
+      value[operator] = val;
+      if (exp.options) {
+        // Keep options for operators
+        value.options = exp.options;
+      }
+      val = value;
+    }
+    where[p] = val;
+  }
+  return where;
+};
+

lib/model.js

@@ -20,6 +20,8 @@ var g = require('strong-globalize')();
 var util = require('util');
 var jutil = require('./jutil');
 var List = require('./list');
+var DataAccessUtils = require('./model-utils');
+var Observer = require('./observer');
 var Hookable = require('./hooks');
 var validations = require('./validations');
 var _extend = util._extend;
@@ -548,28 +550,30 @@ ModelBaseClass.prototype.toObject = function(onlySchema, removeHidden, removePro
   return data;
 };
 
+/**
+ * Convert an array of strings into an object as the map
+ * @param {string[]} arr An array of strings
+ */
+function asObjectMap(arr) {
+  var obj = {};
+  if (Array.isArray(arr)) {
+    for (var i = 0; i < arr.length; i++) {
+      obj[arr[i]] = true;
+    }
+    return obj;
+  }
+  return arr || obj;
+}
 /**
  * Checks if property is protected.
  * @param {String} propertyName Property name
  * @returns  {Boolean} true or false if protected or not.
  */
 ModelBaseClass.isProtectedProperty = function(propertyName) {
-  var Model = this;
+  var settings = (this.definition && this.definition.settings) || {};
-  var settings = Model.definition && Model.definition.settings;
+  var protectedProperties = settings.protectedProperties || settings.protected;
-  var protectedProperties = settings && (settings.protectedProperties || settings.protected);
+  settings.protectedProperties = asObjectMap(protectedProperties);
-  if (Array.isArray(protectedProperties)) {
+  return settings.protectedProperties[propertyName];
-    // Cache the protected properties as an object for quick lookup
-    settings.protectedProperties = {};
-    for (var i = 0; i < protectedProperties.length; i++) {
-      settings.protectedProperties[protectedProperties[i]] = true;
-    }
-    protectedProperties = settings.protectedProperties;
-  }
-  if (protectedProperties) {
-    return protectedProperties[propertyName];
-  } else {
-    return false;
-  }
 };
 
 /**
@@ -578,22 +582,10 @@ ModelBaseClass.isProtectedProperty = function(propertyName) {
  * @returns {Boolean} true or false if hidden or not.
  */
 ModelBaseClass.isHiddenProperty = function(propertyName) {
-  var Model = this;
+  var settings = (this.definition && this.definition.settings) || {};
-  var settings = Model.definition && Model.definition.settings;
+  var hiddenProperties = settings.hiddenProperties || settings.hidden;
-  var hiddenProperties = settings && (settings.hiddenProperties || settings.hidden);
+  settings.hiddenProperties = asObjectMap(hiddenProperties);
-  if (Array.isArray(hiddenProperties)) {
+  return settings.hiddenProperties[propertyName];
-    // Cache the hidden properties as an object for quick lookup
-    settings.hiddenProperties = {};
-    for (var i = 0; i < hiddenProperties.length; i++) {
-      settings.hiddenProperties[hiddenProperties[i]] = true;
-    }
-    hiddenProperties = settings.hiddenProperties;
-  }
-  if (hiddenProperties) {
-    return hiddenProperties[propertyName];
-  } else {
-    return false;
-  }
 };
 
 ModelBaseClass.prototype.toJSON = function() {
@@ -840,13 +832,15 @@ ModelBaseClass.getMergePolicy = function(options) {
  */
 
 ModelBaseClass.getUpdateOnlyProperties = function() {
-  var Model = this;
   const props = this.definition.properties;
   return Object.keys(props).filter(key => props[key].updateOnly);
 };
 
+// Mix in utils
+jutil.mixin(ModelBaseClass, DataAccessUtils);
+
 // Mixin observer
-jutil.mixin(ModelBaseClass, require('./observer'));
+jutil.mixin(ModelBaseClass, Observer);
 
 jutil.mixin(ModelBaseClass, Hookable);
 jutil.mixin(ModelBaseClass, validations.Validatable);

lib/utils.js

@@ -336,7 +336,7 @@ function sanitizeQuery(query, options) {
   const prohibitedKeys = options.prohibitedKeys;
   const offendingKeys = [];
   const normalizeUndefinedInQuery = options.normalizeUndefinedInQuery;
-  const maxDepth = options.maxDepth || 12;
+  const maxDepth = options.maxDepth || Number.MAX_SAFE_INTEGER;
   // WARNING: [rfeng] Use map() will cause mongodb to produce invalid BSON
   // as traverse doesn't transform the ObjectId correctly
   const result = traverse(query).forEach(function(x) {

test/model-definition.test.js

@@ -393,17 +393,17 @@ describe('ModelDefinition class', function() {
       it('should be removed if used in where', function() {
         return Child.find({
           where: {secret: 'guess'},
-        }).then(assertHiddenPropertyIsIgnored);
+        }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
       });
 
       it('should be removed if used in where.and', function() {
         return Child.find({
           where: {and: [{secret: 'guess'}]},
-        }).then(assertHiddenPropertyIsIgnored);
+        }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
       });
 
       it('should be allowed for update', function() {
-        return Child.update({name: 'childA'}, {secret: 'new-secret'}).then(
+        return Child.update({name: 'childA'}, {secret: 'new-secret'}, optionsFromRemoteReq).then(
           function(result) {
             result.count.should.equal(1);
           }
@@ -420,6 +420,15 @@ describe('ModelDefinition class', function() {
         });
       });
 
+      it('should be allowed by default if not remote call', function() {
+        return Child.find({
+          where: {secret: 'guess'},
+        }).then(function(children) {
+          children.length.should.equal(1);
+          children[0].secret.should.equal('guess');
+        });
+      });
+
       it('should be allowed if prohibitHiddenPropertiesInQuery is `false` in options', function() {
         return Child.find({
           where: {secret: 'guess'},
@@ -438,13 +447,13 @@ describe('ModelDefinition class', function() {
       it('should be removed if used in where', function() {
         return Child.find({
           where: {secret: 'guess'},
-        }).then(assertHiddenPropertyIsIgnored);
+        }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
       });
 
       it('should be removed if used in where.and', function() {
         return Child.find({
           where: {and: [{secret: 'guess'}]},
-        }).then(assertHiddenPropertyIsIgnored);
+        }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
       });
     });
 
@@ -474,6 +483,15 @@ describe('ModelDefinition class', function() {
     }
   });
 
+  /**
+   * Mock up for default values set by the remote model
+   */
+  const optionsFromRemoteReq = {
+    prohibitHiddenPropertiesInQuery: true,
+    maxDepthOfQuery: 12,
+    maxDepthOfQuery: 32,
+  };
+
   describe('hidden nested properties', function() {
     var Child;
     beforeEach(givenChildren);
@@ -481,19 +499,19 @@ describe('ModelDefinition class', function() {
     it('should be removed if used in where as a composite key - x.secret', function() {
       return Child.find({
         where: {'x.secret': 'guess'},
-      }).then(assertHiddenPropertyIsIgnored);
+      }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
     });
 
     it('should be removed if used in where as a composite key - secret.y', function() {
       return Child.find({
         where: {'secret.y': 'guess'},
-      }).then(assertHiddenPropertyIsIgnored);
+      }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
     });
 
     it('should be removed if used in where as a composite key - a.secret.b', function() {
       return Child.find({
         where: {'a.secret.b': 'guess'},
-      }).then(assertHiddenPropertyIsIgnored);
+      }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored);
     });
 
     function givenChildren() {
@@ -551,7 +569,7 @@ describe('ModelDefinition class', function() {
             },
           },
         },
-      }).then(assertParentIncludeChildren);
+      }, optionsFromRemoteReq).then(assertParentIncludeChildren);
     });
 
     it('should be rejected if used in include scope.where.and', function() {
@@ -564,7 +582,7 @@ describe('ModelDefinition class', function() {
             },
           },
         },
-      }).then(assertParentIncludeChildren);
+      }, optionsFromRemoteReq).then(assertParentIncludeChildren);
     });
 
     it('should be removed if a hidden property is used in include scope', function() {
@@ -577,7 +595,7 @@ describe('ModelDefinition class', function() {
             },
           },
         },
-      }).then(assertParentIncludeChildren);
+      }, optionsFromRemoteReq).then(assertParentIncludeChildren);
     });
 
     function givenParentAndChild() {
@@ -610,7 +628,7 @@ describe('ModelDefinition class', function() {
             },
           },
         },
-      }).then(assertParentIncludeChildren);
+      }, optionsFromRemoteReq).then(assertParentIncludeChildren);
     });
 
     function givenParentAndChildWithHiddenProperty() {

test/util.test.js

@@ -93,10 +93,13 @@ describe('util.sanitizeQuery', function() {
 
     var q8 = {where: {and: [{and: [{and: [{and: [{and: [{and:
       [{and: [{and: [{and: [{x: 1}]}]}]}]}]}]}]}]}]}};
-    (function() { sanitizeQuery(q8); }).should.throw(
+    (function() { sanitizeQuery(q8, {maxDepth: 12}); }).should.throw(
       /The query object exceeds maximum depth 12/
     );
 
+    // maxDepth is default to maximum integer
+    sanitizeQuery(q8).should.eql(q8);
+
     var q9 = {where: {and: [{and: [{and: [{and: [{x: 1}]}]}]}]}};
     (function() { sanitizeQuery(q8, {maxDepth: 4}); }).should.throw(
       /The query object exceeds maximum depth 4/