From f9131aa18f9018c7ceadfe8e380148cc22bbff5b Mon Sep 17 00:00:00 2001 From: Raymond Feng Date: Mon, 12 Nov 2018 13:54:22 -0800 Subject: [PATCH] Use options from request for settings --- lib/connectors/memory.js | 4 +- lib/dao.js | 509 +------------------------------ lib/include.js | 5 +- lib/model-utils.js | 553 ++++++++++++++++++++++++++++++++++ lib/model.js | 62 ++-- lib/utils.js | 2 +- test/model-definition.test.js | 42 ++- test/util.test.js | 5 +- 8 files changed, 632 insertions(+), 550 deletions(-) create mode 100644 lib/model-utils.js diff --git a/lib/connectors/memory.js b/lib/connectors/memory.js index d8f77561..cbc5f79b 100644 --- a/lib/connectors/memory.js +++ b/lib/connectors/memory.js @@ -323,7 +323,7 @@ Memory.prototype.upsertWithWhere = function(model, where, data, options, callbac } }; -Memory.prototype.findOrCreate = function(model, filter, data, callback) { +Memory.prototype.findOrCreate = function(model, filter, data, options, callback) { var self = this; var nodes = self._findAllSkippingIncludes(model, filter); var found = nodes[0]; @@ -345,7 +345,7 @@ Memory.prototype.findOrCreate = function(model, filter, data, callback) { }); } - self._models[model].model.include(nodes[0], filter.include, {}, function(err, nodes) { + self._models[model].model.include(nodes[0], filter.include, options, function(err, nodes) { process.nextTick(function() { if (err) return callback(err); callback(null, nodes[0], false); diff --git a/lib/dao.js b/lib/dao.js index 9a63f84e..1f484dc2 100644 --- a/lib/dao.js +++ b/lib/dao.js @@ -18,6 +18,7 @@ module.exports = DataAccessObject; var g = require('strong-globalize')(); var async = require('async'); var jutil = require('./jutil'); +var DataAccessObjectUtils = require('./model-utils'); var ValidationError = require('./validations').ValidationError; var Relation = require('./relations.js'); var Inclusion = require('./include.js'); @@ -229,22 +230,6 @@ DataAccessObject.getConnector = function() { return this.getDataSource().connector; }; -/** - * Verify if allowExtendedOperators is enabled - * @options {Object} [options] Optional options to use. - * @property {Boolean} allowExtendedOperators. - * @returns {Boolean} Returns `true` if allowExtendedOperators is enabled, else `false`. - */ -DataAccessObject._allowExtendedOperators = function(options) { - return this._getSetting('allowExtendedOperators', options) === true; -}; - -// Empty callback function -function noCallback(err, result) { - // NOOP - debug('callback is ignored: err=%j, result=%j', err, result); -} - /** * Create an instance of Model with given data and save to the attached data source. Callback is optional. * Example: @@ -391,7 +376,7 @@ DataAccessObject.create = function(data, options, cb) { obj.trigger('create', function(createDone) { obj.trigger('save', function(saveDone) { var _idName = idName(Model); - var val = Model._sanitizeData(obj.toObject(true)); + var val = Model._sanitizeData(obj.toObject(true), options); function createCallback(err, id, rev) { if (id) { obj.__data[_idName] = id; @@ -620,7 +605,7 @@ DataAccessObject.upsert = function(data, options, cb) { } function callConnector() { - update = Model._sanitizeData(update); + update = Model._sanitizeData(update, options); context = { Model: Model, where: ctx.where, @@ -790,7 +775,7 @@ DataAccessObject.upsertWithWhere = function(where, data, options, cb) { try { ctx.where = Model._sanitizeQuery(ctx.where, options); ctx.where = Model._coerce(ctx.where, options); - update = Model._sanitizeData(update); + update = Model._sanitizeData(update, options); update = Model._coerce(update, options); } catch (err) { return process.nextTick(function() { @@ -971,7 +956,7 @@ DataAccessObject.replaceOrCreate = function replaceOrCreate(data, options, cb) { }, update, options); function callConnector() { - update = Model._sanitizeData(update); + update = Model._sanitizeData(update, options); context = { Model: Model, where: where, @@ -1152,7 +1137,7 @@ DataAccessObject.findOrCreate = function findOrCreate(query, data, options, cb) }); } - data = Model._sanitizeData(data); + data = Model._sanitizeData(data, options); var context = { Model: Model, where: query.where, @@ -1437,478 +1422,6 @@ DataAccessObject.all = function() { return DataAccessObject.find.apply(this, arguments); }; -/** - * Get settings via hierarchical determination - * - method level options - * - model level settings - * - datasource level settings - * - * @param {String} key The setting key - */ -DataAccessObject._getSetting = function(key, options) { - // Check method level options - var val = options && options[key]; - if (val !== undefined) return val; - // Check for settings in model - var m = this.definition; - if (m && m.settings) { - val = m.settings[key]; - if (val !== undefined) { - return m.settings[key]; - } - // Fall back to datasource level - } - - // Check for settings in connector - var ds = this.getDataSource(); - if (ds && ds.settings) { - return ds.settings[key]; - } - - return undefined; -}; - -var operators = { - eq: '=', - gt: '>', - gte: '>=', - lt: '<', - lte: '<=', - between: 'BETWEEN', - inq: 'IN', - nin: 'NOT IN', - neq: '!=', - like: 'LIKE', - nlike: 'NOT LIKE', - ilike: 'ILIKE', - nilike: 'NOT ILIKE', - regexp: 'REGEXP', -}; - -/* - * Normalize the filter object and throw errors if invalid values are detected - * @param {Object} filter The query filter object - * @options {Object} [options] Optional options to use. - * @property {Boolean} allowExtendedOperators. - * @returns {Object} The normalized filter object - * @private - */ -DataAccessObject._normalize = function(filter, options) { - if (!filter) { - return undefined; - } - var err = null; - if ((typeof filter !== 'object') || Array.isArray(filter)) { - err = new Error(g.f('The query filter %j is not an {{object}}', filter)); - err.statusCode = 400; - throw err; - } - if (filter.limit || filter.skip || filter.offset) { - var limit = Number(filter.limit || 100); - var offset = Number(filter.skip || filter.offset || 0); - if (isNaN(limit) || limit <= 0 || Math.ceil(limit) !== limit) { - err = new Error(g.f('The {{limit}} parameter %j is not valid', - filter.limit)); - err.statusCode = 400; - throw err; - } - if (isNaN(offset) || offset < 0 || Math.ceil(offset) !== offset) { - err = new Error(g.f('The {{offset/skip}} parameter %j is not valid', - filter.skip || filter.offset)); - err.statusCode = 400; - throw err; - } - filter.limit = limit; - filter.offset = offset; - filter.skip = offset; - } - - if (filter.order) { - var order = filter.order; - if (!Array.isArray(order)) { - order = [order]; - } - var fields = []; - for (var i = 0, m = order.length; i < m; i++) { - if (typeof order[i] === 'string') { - // Normalize 'f1 ASC, f2 DESC, f3' to ['f1 ASC', 'f2 DESC', 'f3'] - var tokens = order[i].split(/(?:\s*,\s*)+/); - for (var t = 0, n = tokens.length; t < n; t++) { - var token = tokens[t]; - if (token.length === 0) { - // Skip empty token - continue; - } - var parts = token.split(/\s+/); - if (parts.length >= 2) { - var dir = parts[1].toUpperCase(); - if (dir === 'ASC' || dir === 'DESC') { - token = parts[0] + ' ' + dir; - } else { - err = new Error(g.f('The {{order}} %j has invalid direction', token)); - err.statusCode = 400; - throw err; - } - } - fields.push(token); - } - } else { - err = new Error(g.f('The order %j is not valid', order[i])); - err.statusCode = 400; - throw err; - } - } - if (fields.length === 1 && typeof filter.order === 'string') { - filter.order = fields[0]; - } else { - filter.order = fields; - } - } - - // normalize fields as array of included property names - if (filter.fields) { - filter.fields = fieldsToArray(filter.fields, - Object.keys(this.definition.properties), this.settings.strict); - } - - filter = this._sanitizeQuery(filter, options); - this._coerce(filter.where, options); - return filter; -}; - -function DateType(arg) { - var d = new Date(arg); - if (isNaN(d.getTime())) { - throw new Error(g.f('Invalid date: %s', arg)); - } - return d; -} - -function BooleanType(arg) { - if (typeof arg === 'string') { - switch (arg) { - case 'true': - case '1': - return true; - case 'false': - case '0': - return false; - } - } - if (arg == null) { - return null; - } - return Boolean(arg); -} - -function NumberType(val) { - var num = Number(val); - return !isNaN(num) ? num : val; -} - -function coerceArray(val) { - if (Array.isArray(val)) { - return val; - } - - if (!utils.isPlainObject(val)) { - throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices')); - } - - // It is an object, check if empty - var props = Object.keys(val); - - if (props.length === 0) { - throw new Error(g.f('Value is an empty {{object}}')); - } - - var arrayVal = new Array(props.length); - for (var i = 0; i < arrayVal.length; ++i) { - if (!val.hasOwnProperty(i)) { - throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices')); - } - - arrayVal[i] = val[i]; - } - - return arrayVal; -} - -function _normalizeAsArray(result) { - if (typeof result === 'string') { - result = [result]; - } - if (Array.isArray(result)) { - return result; - } else { - // See https://github.com/strongloop/loopback-datasource-juggler/issues/1646 - // `ModelBaseClass` normalize the properties to an object such as `{secret: true}` - var keys = []; - for (var k in result) { - if (result[k]) keys.push(k); - } - return keys; - } -} - -DataAccessObject._getHiddenProperties = function() { - var settings = this.definition.settings || {}; - var result = settings.hiddenProperties || settings.hidden || []; - return _normalizeAsArray(result); -}; - -DataAccessObject._getProtectedProperties = function() { - var settings = this.definition.settings || {}; - var result = settings.protectedProperties || settings.protected || []; - return _normalizeAsArray(result); -}; - -DataAccessObject._sanitizeQuery = function(query, options) { - options = options || {}; - - // Get settings to normalize `undefined` values - var normalizeUndefinedInQuery = this._getSetting('normalizeUndefinedInQuery', options); - // Get setting to prohibit hidden/protected properties in query - var prohibitHiddenPropertiesInQuery = this._getSetting('prohibitHiddenPropertiesInQuery', options); - if (prohibitHiddenPropertiesInQuery == null) { - // By default, hidden properties are prohibited in query - prohibitHiddenPropertiesInQuery = true; - } - - // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651 - var maxDepthOfQuery = (+this._getSetting('maxDepthOfQuery', options)) || 12; - - var prohibitedKeys = []; - // Check violation of keys - if (prohibitHiddenPropertiesInQuery) { - prohibitedKeys = this._getHiddenProperties(); - if (options.prohibitProtectedPropertiesInQuery) { - prohibitedKeys = prohibitedKeys.concat(this._getProtectedProperties()); - } - } - return sanitizeQueryOrData(query, - Object.assign({ - maxDepth: maxDepthOfQuery, - prohibitedKeys: prohibitedKeys, - normalizeUndefinedInQuery: normalizeUndefinedInQuery, - }, options)); -}; - -DataAccessObject._sanitizeData = function(data, options) { - options = options || {}; - // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651 - var maxDepthOfQuery = (+this._getSetting('maxDepthOfQuery', options)) || 12; - return sanitizeQueryOrData(data, - Object.assign({ - maxDepth: maxDepthOfQuery, - }, options)); -}; - -/* - * Coerce values based the property types - * @param {Object} where The where clause - * @options {Object} [options] Optional options to use. - * @property {Boolean} allowExtendedOperators. - * @returns {Object} The coerced where clause - * @private - */ -DataAccessObject._coerce = function(where, options) { - var self = this; - if (where == null) { - return where; - } - options = options || {}; - - var err; - if (typeof where !== 'object' || Array.isArray(where)) { - err = new Error(g.f('The where clause %j is not an {{object}}', where)); - err.statusCode = 400; - throw err; - } - - var props = self.definition.properties; - for (var p in where) { - // Handle logical operators - if (p === 'and' || p === 'or' || p === 'nor') { - var clauses = where[p]; - try { - clauses = coerceArray(clauses); - } catch (e) { - err = new Error(g.f('The %s operator has invalid clauses %j: %s', p, clauses, e.message)); - err.statusCode = 400; - throw err; - } - - for (var k = 0; k < clauses.length; k++) { - self._coerce(clauses[k], options); - } - - continue; - } - var DataType = props[p] && props[p].type; - if (!DataType) { - continue; - } - if (Array.isArray(DataType) || DataType === Array) { - DataType = DataType[0]; - } - if (DataType === Date) { - DataType = DateType; - } else if (DataType === Boolean) { - DataType = BooleanType; - } else if (DataType === Number) { - // This fixes a regression in mongodb connector - // For numbers, only convert it produces a valid number - // LoopBack by default injects a number id. We should fix it based - // on the connector's input, for example, MongoDB should use string - // while RDBs typically use number - DataType = NumberType; - } - - if (!DataType) { - continue; - } - - if (DataType.prototype instanceof BaseModel) { - continue; - } - - if (DataType === geo.GeoPoint) { - // Skip the GeoPoint as the near operator breaks the assumption that - // an operation has only one property - // We should probably fix it based on - // http://docs.mongodb.org/manual/reference/operator/query/near/ - // The other option is to make operators start with $ - continue; - } - - var val = where[p]; - if (val === null || val === undefined) { - continue; - } - // Check there is an operator - var operator = null; - var exp = val; - if (val.constructor === Object) { - for (var op in operators) { - if (op in val) { - val = val[op]; - operator = op; - switch (operator) { - case 'inq': - case 'nin': - case 'between': - try { - val = coerceArray(val); - } catch (e) { - err = new Error(g.f('The %s property has invalid clause %j: %s', p, where[p], e)); - err.statusCode = 400; - throw err; - } - - if (operator === 'between' && val.length !== 2) { - err = new Error(g.f( - 'The %s property has invalid clause %j: Expected precisely 2 values, received %d', - p, - where[p], - val.length - )); - err.statusCode = 400; - throw err; - } - break; - case 'like': - case 'nlike': - case 'ilike': - case 'nilike': - if (!(typeof val === 'string' || val instanceof RegExp)) { - err = new Error(g.f( - 'The %s property has invalid clause %j: Expected a string or RegExp', - p, - where[p] - )); - err.statusCode = 400; - throw err; - } - break; - case 'regexp': - val = utils.toRegExp(val); - if (val instanceof Error) { - val.statusCode = 400; - throw val; - } - break; - } - break; - } - } - } - - try { - // Coerce val into an array if it resembles an array-like object - val = coerceArray(val); - } catch (e) { - // NOOP when not coercable into an array. - } - - var allowExtendedOperators = self._allowExtendedOperators(options); - // Coerce the array items - if (Array.isArray(val)) { - for (var i = 0; i < val.length; i++) { - if (val[i] !== null && val[i] !== undefined) { - if (!(val[i] instanceof RegExp)) { - val[i] = DataType(val[i]); - } - } - } - } else { - if (val != null) { - if (operator === null && val instanceof RegExp) { - // Normalize {name: /A/} to {name: {regexp: /A/}} - operator = 'regexp'; - } else if (operator === 'regexp' && val instanceof RegExp) { - // Do not coerce regex literals/objects - } else if ((operator === 'like' || operator === 'nlike' || - operator === 'ilike' || operator === 'nilike') && val instanceof RegExp) { - // Do not coerce RegExp operator value - } else if (allowExtendedOperators && typeof val === 'object') { - // Do not coerce object values when extended operators are allowed - } else { - if (!allowExtendedOperators) { - var extendedOperators = Object.keys(val).filter(function(k) { - return k[0] === '$'; - }); - if (extendedOperators.length) { - const msg = g.f('Operators "' + extendedOperators.join(', ') + '" are not allowed in query'); - const err = new Error(msg); - err.code = 'OPERATOR_NOT_ALLOWED_IN_QUERY'; - err.statusCode = 400; - err.details = { - operators: extendedOperators, - where: where, - }; - throw err; - } - } - val = DataType(val); - } - } - } - // Rebuild {property: {operator: value}} - if (operator && operator !== 'eq') { - var value = {}; - value[operator] = val; - if (exp.options) { - // Keep options for operators - value.options = exp.options; - } - val = value; - } - where[p] = val; - } - return where; -}; - /** * Find all instances of Model that match the specified query. * Fields used for filter and sort should be declared with `{index: true}` in model definition. @@ -2614,7 +2127,7 @@ DataAccessObject.prototype.save = function(options, cb) { function save() { inst.trigger('save', function(saveDone) { inst.trigger('update', function(updateDone) { - data = Model._sanitizeData(data); + data = Model._sanitizeData(data, options); function saveCallback(err, unusedData, result) { if (err) { return cb(err, inst); @@ -2803,7 +2316,7 @@ DataAccessObject.updateAll = function(where, data, options, cb) { // alter configuration of how sanitizeQuery handles undefined values where = Model._sanitizeQuery(where, options); where = Model._coerce(where, options); - data = Model._sanitizeData(data); + data = Model._sanitizeData(data, options); data = Model._coerce(data, options); } catch (err) { return process.nextTick(function() { @@ -3136,7 +2649,7 @@ DataAccessObject.replaceById = function(id, data, options, cb) { function validateAndCallConnector(err, data) { if (err) return cb(err); - data = Model._sanitizeData(data); + data = Model._sanitizeData(data, options); // update instance's properties inst.setAttributes(data); @@ -3288,7 +2801,7 @@ function(data, options, cb) { if (data instanceof Model) { data = data.toObject(false); } - data = Model._sanitizeData(data); + data = Model._sanitizeData(data, options); // Make sure id(s) cannot be changed var idNames = Model.definition.idNames(); @@ -3361,7 +2874,7 @@ function(data, options, cb) { inst.trigger('update', function(done) { copyData(data, inst); var typedData = convertSubsetOfPropertiesByType(inst, data); - context.data = Model._sanitizeData(typedData); + context.data = Model._sanitizeData(typedData, options); // Depending on the connector, the database response can // contain information about the updated record(s), but also diff --git a/lib/include.js b/lib/include.js index 3503efc1..0762b8fa 100644 --- a/lib/include.js +++ b/lib/include.js @@ -205,8 +205,9 @@ Inclusion.include = function(objects, include, options, cb) { */ function findWithForeignKeysByPage(model, filter, fkName, pageSize, options, cb) { try { - model._sanitizeQuery(filter.where, {prohibitProtectedPropertiesInQuery: true}); - model._coerce(filter.where); + const opts = Object.assign({prohibitProtectedPropertiesInQuery: true}, options); + model._sanitizeQuery(filter.where, opts); + model._coerce(filter.where, options); } catch (e) { return cb(e); } diff --git a/lib/model-utils.js b/lib/model-utils.js new file mode 100644 index 00000000..ee36a60d --- /dev/null +++ b/lib/model-utils.js @@ -0,0 +1,553 @@ +// Copyright IBM Corp. 2013,2016. All Rights Reserved. +// Node module: loopback-datasource-juggler +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +// Turning on strict for this file breaks lots of test cases; +// disabling strict for this file +/* eslint-disable strict */ + +module.exports = ModelUtils; + +/*! + * Module dependencies + */ +var g = require('strong-globalize')(); +var geo = require('./geo'); +var utils = require('./utils'); +var fieldsToArray = utils.fieldsToArray; +var sanitizeQueryOrData = utils.sanitizeQuery; +var BaseModel = require('./model'); + +/** + * A mixin to contain utility methods for DataAccessObject + */ +function ModelUtils() { +} + +/** + * Verify if allowExtendedOperators is enabled + * @options {Object} [options] Optional options to use. + * @property {Boolean} allowExtendedOperators. + * @returns {Boolean} Returns `true` if allowExtendedOperators is enabled, else `false`. + */ +ModelUtils._allowExtendedOperators = function(options) { + const flag = this._getSetting('allowExtendedOperators', options); + if (flag != null) return !!flag; + // Default to `false` + return false; +}; + +/** + * Get settings via hierarchical determination + * - method level options + * - model level settings + * - data source level settings + * + * @param {String} key The setting key + */ +ModelUtils._getSetting = function(key, options) { + // Check method level options + var val = options && options[key]; + if (val !== undefined) return val; + // Check for settings in model + var m = this.definition; + if (m && m.settings) { + val = m.settings[key]; + if (val !== undefined) { + return m.settings[key]; + } + // Fall back to data source level + } + + // Check for settings in connector + var ds = this.getDataSource(); + if (ds && ds.settings) { + return ds.settings[key]; + } + + return undefined; +}; + +var operators = { + eq: '=', + gt: '>', + gte: '>=', + lt: '<', + lte: '<=', + between: 'BETWEEN', + inq: 'IN', + nin: 'NOT IN', + neq: '!=', + like: 'LIKE', + nlike: 'NOT LIKE', + ilike: 'ILIKE', + nilike: 'NOT ILIKE', + regexp: 'REGEXP', +}; + +/* + * Normalize the filter object and throw errors if invalid values are detected + * @param {Object} filter The query filter object + * @options {Object} [options] Optional options to use. + * @property {Boolean} allowExtendedOperators. + * @returns {Object} The normalized filter object + * @private + */ +ModelUtils._normalize = function(filter, options) { + if (!filter) { + return undefined; + } + var err = null; + if ((typeof filter !== 'object') || Array.isArray(filter)) { + err = new Error(g.f('The query filter %j is not an {{object}}', filter)); + err.statusCode = 400; + throw err; + } + if (filter.limit || filter.skip || filter.offset) { + var limit = Number(filter.limit || 100); + var offset = Number(filter.skip || filter.offset || 0); + if (isNaN(limit) || limit <= 0 || Math.ceil(limit) !== limit) { + err = new Error(g.f('The {{limit}} parameter %j is not valid', + filter.limit)); + err.statusCode = 400; + throw err; + } + if (isNaN(offset) || offset < 0 || Math.ceil(offset) !== offset) { + err = new Error(g.f('The {{offset/skip}} parameter %j is not valid', + filter.skip || filter.offset)); + err.statusCode = 400; + throw err; + } + filter.limit = limit; + filter.offset = offset; + filter.skip = offset; + } + + if (filter.order) { + var order = filter.order; + if (!Array.isArray(order)) { + order = [order]; + } + var fields = []; + for (var i = 0, m = order.length; i < m; i++) { + if (typeof order[i] === 'string') { + // Normalize 'f1 ASC, f2 DESC, f3' to ['f1 ASC', 'f2 DESC', 'f3'] + var tokens = order[i].split(/(?:\s*,\s*)+/); + for (var t = 0, n = tokens.length; t < n; t++) { + var token = tokens[t]; + if (token.length === 0) { + // Skip empty token + continue; + } + var parts = token.split(/\s+/); + if (parts.length >= 2) { + var dir = parts[1].toUpperCase(); + if (dir === 'ASC' || dir === 'DESC') { + token = parts[0] + ' ' + dir; + } else { + err = new Error(g.f('The {{order}} %j has invalid direction', token)); + err.statusCode = 400; + throw err; + } + } + fields.push(token); + } + } else { + err = new Error(g.f('The order %j is not valid', order[i])); + err.statusCode = 400; + throw err; + } + } + if (fields.length === 1 && typeof filter.order === 'string') { + filter.order = fields[0]; + } else { + filter.order = fields; + } + } + + // normalize fields as array of included property names + if (filter.fields) { + filter.fields = fieldsToArray(filter.fields, + Object.keys(this.definition.properties), this.settings.strict); + } + + filter = this._sanitizeQuery(filter, options); + this._coerce(filter.where, options); + return filter; +}; + +function DateType(arg) { + var d = new Date(arg); + if (isNaN(d.getTime())) { + throw new Error(g.f('Invalid date: %s', arg)); + } + return d; +} + +function BooleanType(arg) { + if (typeof arg === 'string') { + switch (arg) { + case 'true': + case '1': + return true; + case 'false': + case '0': + return false; + } + } + if (arg == null) { + return null; + } + return Boolean(arg); +} + +function NumberType(val) { + var num = Number(val); + return !isNaN(num) ? num : val; +} + +function coerceArray(val) { + if (Array.isArray(val)) { + return val; + } + + if (!utils.isPlainObject(val)) { + throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices')); + } + + // It is an object, check if empty + var props = Object.keys(val); + + if (props.length === 0) { + throw new Error(g.f('Value is an empty {{object}}')); + } + + var arrayVal = new Array(props.length); + for (var i = 0; i < arrayVal.length; ++i) { + if (!val.hasOwnProperty(i)) { + throw new Error(g.f('Value is not an {{array}} or {{object}} with sequential numeric indices')); + } + + arrayVal[i] = val[i]; + } + + return arrayVal; +} + +function _normalizeAsArray(result) { + if (typeof result === 'string') { + result = [result]; + } + if (Array.isArray(result)) { + return result; + } else { + // See https://github.com/strongloop/loopback-datasource-juggler/issues/1646 + // `ModelBaseClass` normalize the properties to an object such as `{secret: true}` + var keys = []; + for (var k in result) { + if (result[k]) keys.push(k); + } + return keys; + } +} + +/** + * Get an array of hidden property names + */ +ModelUtils._getHiddenProperties = function() { + var settings = this.definition.settings || {}; + var result = settings.hiddenProperties || settings.hidden || []; + return _normalizeAsArray(result); +}; + +/** + * Get an array of protected property names + */ +ModelUtils._getProtectedProperties = function() { + var settings = this.definition.settings || {}; + var result = settings.protectedProperties || settings.protected || []; + return _normalizeAsArray(result); +}; + +/** + * Get the maximum depth of a query object + */ +ModelUtils._getMaxDepthOfQuery = function(options, defaultValue) { + options = options || {}; + // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651 + var maxDepth = this._getSetting('maxDepthOfQuery', options); + if (maxDepth == null) { + maxDepth = defaultValue || 32; + } + return +maxDepth; +}; + +/** + * Get the maximum depth of a data object + */ +ModelUtils._getMaxDepthOfData = function(options, defaultValue) { + options = options || {}; + // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651 + var maxDepth = this._getSetting('maxDepthOfData', options); + if (maxDepth == null) { + maxDepth = defaultValue || 64; + } + return +maxDepth; +}; + +/** + * Get the prohibitHiddenPropertiesInQuery flag + */ +ModelUtils._getProhibitHiddenPropertiesInQuery = function(options, defaultValue) { + var flag = this._getSetting('prohibitHiddenPropertiesInQuery', options); + if (flag == null) return !!defaultValue; + return !!flag; +}; + +/** + * Sanitize the query object + */ +ModelUtils._sanitizeQuery = function(query, options) { + options = options || {}; + + // Get settings to normalize `undefined` values + var normalizeUndefinedInQuery = this._getSetting('normalizeUndefinedInQuery', options); + // Get setting to prohibit hidden/protected properties in query + var prohibitHiddenPropertiesInQuery = this._getProhibitHiddenPropertiesInQuery(options); + + // See https://github.com/strongloop/loopback-datasource-juggler/issues/1651 + var maxDepthOfQuery = this._getMaxDepthOfQuery(options); + + var prohibitedKeys = []; + // Check violation of keys + if (prohibitHiddenPropertiesInQuery) { + prohibitedKeys = this._getHiddenProperties(); + if (options.prohibitProtectedPropertiesInQuery) { + prohibitedKeys = prohibitedKeys.concat(this._getProtectedProperties()); + } + } + return sanitizeQueryOrData(query, + Object.assign({ + maxDepth: maxDepthOfQuery, + prohibitedKeys: prohibitedKeys, + normalizeUndefinedInQuery: normalizeUndefinedInQuery, + }, options)); +}; + +/** + * Sanitize the data object + */ +ModelUtils._sanitizeData = function(data, options) { + options = options || {}; + return sanitizeQueryOrData(data, + Object.assign({ + maxDepth: this._getMaxDepthOfData(options), + }, options)); +}; + +/* + * Coerce values based the property types + * @param {Object} where The where clause + * @options {Object} [options] Optional options to use. + * @property {Boolean} allowExtendedOperators. + * @returns {Object} The coerced where clause + * @private + */ +ModelUtils._coerce = function(where, options) { + var self = this; + if (where == null) { + return where; + } + options = options || {}; + + var err; + if (typeof where !== 'object' || Array.isArray(where)) { + err = new Error(g.f('The where clause %j is not an {{object}}', where)); + err.statusCode = 400; + throw err; + } + + var props = self.definition.properties; + for (var p in where) { + // Handle logical operators + if (p === 'and' || p === 'or' || p === 'nor') { + var clauses = where[p]; + try { + clauses = coerceArray(clauses); + } catch (e) { + err = new Error(g.f('The %s operator has invalid clauses %j: %s', p, clauses, e.message)); + err.statusCode = 400; + throw err; + } + + for (var k = 0; k < clauses.length; k++) { + self._coerce(clauses[k], options); + } + + continue; + } + var DataType = props[p] && props[p].type; + if (!DataType) { + continue; + } + if (Array.isArray(DataType) || DataType === Array) { + DataType = DataType[0]; + } + if (DataType === Date) { + DataType = DateType; + } else if (DataType === Boolean) { + DataType = BooleanType; + } else if (DataType === Number) { + // This fixes a regression in mongodb connector + // For numbers, only convert it produces a valid number + // LoopBack by default injects a number id. We should fix it based + // on the connector's input, for example, MongoDB should use string + // while RDBs typically use number + DataType = NumberType; + } + + if (!DataType) { + continue; + } + + if (DataType.prototype instanceof BaseModel) { + continue; + } + + if (DataType === geo.GeoPoint) { + // Skip the GeoPoint as the near operator breaks the assumption that + // an operation has only one property + // We should probably fix it based on + // http://docs.mongodb.org/manual/reference/operator/query/near/ + // The other option is to make operators start with $ + continue; + } + + var val = where[p]; + if (val === null || val === undefined) { + continue; + } + // Check there is an operator + var operator = null; + var exp = val; + if (val.constructor === Object) { + for (var op in operators) { + if (op in val) { + val = val[op]; + operator = op; + switch (operator) { + case 'inq': + case 'nin': + case 'between': + try { + val = coerceArray(val); + } catch (e) { + err = new Error(g.f('The %s property has invalid clause %j: %s', p, where[p], e)); + err.statusCode = 400; + throw err; + } + + if (operator === 'between' && val.length !== 2) { + err = new Error(g.f( + 'The %s property has invalid clause %j: Expected precisely 2 values, received %d', + p, + where[p], + val.length + )); + err.statusCode = 400; + throw err; + } + break; + case 'like': + case 'nlike': + case 'ilike': + case 'nilike': + if (!(typeof val === 'string' || val instanceof RegExp)) { + err = new Error(g.f( + 'The %s property has invalid clause %j: Expected a string or RegExp', + p, + where[p] + )); + err.statusCode = 400; + throw err; + } + break; + case 'regexp': + val = utils.toRegExp(val); + if (val instanceof Error) { + val.statusCode = 400; + throw val; + } + break; + } + break; + } + } + } + + try { + // Coerce val into an array if it resembles an array-like object + val = coerceArray(val); + } catch (e) { + // NOOP when not coercable into an array. + } + + var allowExtendedOperators = self._allowExtendedOperators(options); + // Coerce the array items + if (Array.isArray(val)) { + for (var i = 0; i < val.length; i++) { + if (val[i] !== null && val[i] !== undefined) { + if (!(val[i] instanceof RegExp)) { + val[i] = DataType(val[i]); + } + } + } + } else { + if (val != null) { + if (operator === null && val instanceof RegExp) { + // Normalize {name: /A/} to {name: {regexp: /A/}} + operator = 'regexp'; + } else if (operator === 'regexp' && val instanceof RegExp) { + // Do not coerce regex literals/objects + } else if ((operator === 'like' || operator === 'nlike' || + operator === 'ilike' || operator === 'nilike') && val instanceof RegExp) { + // Do not coerce RegExp operator value + } else if (allowExtendedOperators && typeof val === 'object') { + // Do not coerce object values when extended operators are allowed + } else { + if (!allowExtendedOperators) { + var extendedOperators = Object.keys(val).filter(function(k) { + return k[0] === '$'; + }); + if (extendedOperators.length) { + const msg = g.f('Operators "' + extendedOperators.join(', ') + '" are not allowed in query'); + const err = new Error(msg); + err.code = 'OPERATOR_NOT_ALLOWED_IN_QUERY'; + err.statusCode = 400; + err.details = { + operators: extendedOperators, + where: where, + }; + throw err; + } + } + val = DataType(val); + } + } + } + // Rebuild {property: {operator: value}} + if (operator && operator !== 'eq') { + var value = {}; + value[operator] = val; + if (exp.options) { + // Keep options for operators + value.options = exp.options; + } + val = value; + } + where[p] = val; + } + return where; +}; + diff --git a/lib/model.js b/lib/model.js index 54cc2d40..e431c00b 100644 --- a/lib/model.js +++ b/lib/model.js @@ -20,6 +20,8 @@ var g = require('strong-globalize')(); var util = require('util'); var jutil = require('./jutil'); var List = require('./list'); +var DataAccessUtils = require('./model-utils'); +var Observer = require('./observer'); var Hookable = require('./hooks'); var validations = require('./validations'); var _extend = util._extend; @@ -548,28 +550,30 @@ ModelBaseClass.prototype.toObject = function(onlySchema, removeHidden, removePro return data; }; +/** + * Convert an array of strings into an object as the map + * @param {string[]} arr An array of strings + */ +function asObjectMap(arr) { + var obj = {}; + if (Array.isArray(arr)) { + for (var i = 0; i < arr.length; i++) { + obj[arr[i]] = true; + } + return obj; + } + return arr || obj; +} /** * Checks if property is protected. * @param {String} propertyName Property name * @returns {Boolean} true or false if protected or not. */ ModelBaseClass.isProtectedProperty = function(propertyName) { - var Model = this; - var settings = Model.definition && Model.definition.settings; - var protectedProperties = settings && (settings.protectedProperties || settings.protected); - if (Array.isArray(protectedProperties)) { - // Cache the protected properties as an object for quick lookup - settings.protectedProperties = {}; - for (var i = 0; i < protectedProperties.length; i++) { - settings.protectedProperties[protectedProperties[i]] = true; - } - protectedProperties = settings.protectedProperties; - } - if (protectedProperties) { - return protectedProperties[propertyName]; - } else { - return false; - } + var settings = (this.definition && this.definition.settings) || {}; + var protectedProperties = settings.protectedProperties || settings.protected; + settings.protectedProperties = asObjectMap(protectedProperties); + return settings.protectedProperties[propertyName]; }; /** @@ -578,22 +582,10 @@ ModelBaseClass.isProtectedProperty = function(propertyName) { * @returns {Boolean} true or false if hidden or not. */ ModelBaseClass.isHiddenProperty = function(propertyName) { - var Model = this; - var settings = Model.definition && Model.definition.settings; - var hiddenProperties = settings && (settings.hiddenProperties || settings.hidden); - if (Array.isArray(hiddenProperties)) { - // Cache the hidden properties as an object for quick lookup - settings.hiddenProperties = {}; - for (var i = 0; i < hiddenProperties.length; i++) { - settings.hiddenProperties[hiddenProperties[i]] = true; - } - hiddenProperties = settings.hiddenProperties; - } - if (hiddenProperties) { - return hiddenProperties[propertyName]; - } else { - return false; - } + var settings = (this.definition && this.definition.settings) || {}; + var hiddenProperties = settings.hiddenProperties || settings.hidden; + settings.hiddenProperties = asObjectMap(hiddenProperties); + return settings.hiddenProperties[propertyName]; }; ModelBaseClass.prototype.toJSON = function() { @@ -840,13 +832,15 @@ ModelBaseClass.getMergePolicy = function(options) { */ ModelBaseClass.getUpdateOnlyProperties = function() { - var Model = this; const props = this.definition.properties; return Object.keys(props).filter(key => props[key].updateOnly); }; +// Mix in utils +jutil.mixin(ModelBaseClass, DataAccessUtils); + // Mixin observer -jutil.mixin(ModelBaseClass, require('./observer')); +jutil.mixin(ModelBaseClass, Observer); jutil.mixin(ModelBaseClass, Hookable); jutil.mixin(ModelBaseClass, validations.Validatable); diff --git a/lib/utils.js b/lib/utils.js index 1b4f8eb8..3ddd0d37 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -336,7 +336,7 @@ function sanitizeQuery(query, options) { const prohibitedKeys = options.prohibitedKeys; const offendingKeys = []; const normalizeUndefinedInQuery = options.normalizeUndefinedInQuery; - const maxDepth = options.maxDepth || 12; + const maxDepth = options.maxDepth || Number.MAX_SAFE_INTEGER; // WARNING: [rfeng] Use map() will cause mongodb to produce invalid BSON // as traverse doesn't transform the ObjectId correctly const result = traverse(query).forEach(function(x) { diff --git a/test/model-definition.test.js b/test/model-definition.test.js index 09d1e1a1..ac8de705 100644 --- a/test/model-definition.test.js +++ b/test/model-definition.test.js @@ -393,17 +393,17 @@ describe('ModelDefinition class', function() { it('should be removed if used in where', function() { return Child.find({ where: {secret: 'guess'}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); it('should be removed if used in where.and', function() { return Child.find({ where: {and: [{secret: 'guess'}]}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); it('should be allowed for update', function() { - return Child.update({name: 'childA'}, {secret: 'new-secret'}).then( + return Child.update({name: 'childA'}, {secret: 'new-secret'}, optionsFromRemoteReq).then( function(result) { result.count.should.equal(1); } @@ -420,6 +420,15 @@ describe('ModelDefinition class', function() { }); }); + it('should be allowed by default if not remote call', function() { + return Child.find({ + where: {secret: 'guess'}, + }).then(function(children) { + children.length.should.equal(1); + children[0].secret.should.equal('guess'); + }); + }); + it('should be allowed if prohibitHiddenPropertiesInQuery is `false` in options', function() { return Child.find({ where: {secret: 'guess'}, @@ -438,13 +447,13 @@ describe('ModelDefinition class', function() { it('should be removed if used in where', function() { return Child.find({ where: {secret: 'guess'}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); it('should be removed if used in where.and', function() { return Child.find({ where: {and: [{secret: 'guess'}]}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); }); @@ -474,6 +483,15 @@ describe('ModelDefinition class', function() { } }); + /** + * Mock up for default values set by the remote model + */ + const optionsFromRemoteReq = { + prohibitHiddenPropertiesInQuery: true, + maxDepthOfQuery: 12, + maxDepthOfQuery: 32, + }; + describe('hidden nested properties', function() { var Child; beforeEach(givenChildren); @@ -481,19 +499,19 @@ describe('ModelDefinition class', function() { it('should be removed if used in where as a composite key - x.secret', function() { return Child.find({ where: {'x.secret': 'guess'}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); it('should be removed if used in where as a composite key - secret.y', function() { return Child.find({ where: {'secret.y': 'guess'}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); it('should be removed if used in where as a composite key - a.secret.b', function() { return Child.find({ where: {'a.secret.b': 'guess'}, - }).then(assertHiddenPropertyIsIgnored); + }, optionsFromRemoteReq).then(assertHiddenPropertyIsIgnored); }); function givenChildren() { @@ -551,7 +569,7 @@ describe('ModelDefinition class', function() { }, }, }, - }).then(assertParentIncludeChildren); + }, optionsFromRemoteReq).then(assertParentIncludeChildren); }); it('should be rejected if used in include scope.where.and', function() { @@ -564,7 +582,7 @@ describe('ModelDefinition class', function() { }, }, }, - }).then(assertParentIncludeChildren); + }, optionsFromRemoteReq).then(assertParentIncludeChildren); }); it('should be removed if a hidden property is used in include scope', function() { @@ -577,7 +595,7 @@ describe('ModelDefinition class', function() { }, }, }, - }).then(assertParentIncludeChildren); + }, optionsFromRemoteReq).then(assertParentIncludeChildren); }); function givenParentAndChild() { @@ -610,7 +628,7 @@ describe('ModelDefinition class', function() { }, }, }, - }).then(assertParentIncludeChildren); + }, optionsFromRemoteReq).then(assertParentIncludeChildren); }); function givenParentAndChildWithHiddenProperty() { diff --git a/test/util.test.js b/test/util.test.js index 131d27ab..bc15d23d 100644 --- a/test/util.test.js +++ b/test/util.test.js @@ -93,10 +93,13 @@ describe('util.sanitizeQuery', function() { var q8 = {where: {and: [{and: [{and: [{and: [{and: [{and: [{and: [{and: [{and: [{x: 1}]}]}]}]}]}]}]}]}]}}; - (function() { sanitizeQuery(q8); }).should.throw( + (function() { sanitizeQuery(q8, {maxDepth: 12}); }).should.throw( /The query object exceeds maximum depth 12/ ); + // maxDepth is default to maximum integer + sanitizeQuery(q8).should.eql(q8); + var q9 = {where: {and: [{and: [{and: [{and: [{x: 1}]}]}]}]}}; (function() { sanitizeQuery(q8, {maxDepth: 4}); }).should.throw( /The query object exceeds maximum depth 4/