From 6f88cf193077576f0845c9f7dc7efbac93c18dd4 Mon Sep 17 00:00:00 2001 From: Corentin H Date: Thu, 6 Apr 2017 14:25:04 +0200 Subject: [PATCH] handle deep geo-near queries (#1216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit      a dedicated mongKey is added in geo.nearFilter for mongoDB fixes geo min distance tests as filter now expects an array --- lib/geo.js | 142 +++++++++++++++++++------------- test/basic-querying.test.js | 157 +++++++++++++++++++++++++++++++++++- test/geo.test.js | 8 +- 3 files changed, 242 insertions(+), 65 deletions(-) diff --git a/lib/geo.js b/lib/geo.js index 9bef1222..1a689a86 100644 --- a/lib/geo.js +++ b/lib/geo.js @@ -12,82 +12,109 @@ var assert = require('assert'); */ exports.nearFilter = function nearFilter(where) { - var result = false; + function nearSearch(clause, parentKeys) { + if (typeof clause !== 'object') { + return false; + } + parentKeys = parentKeys || []; - if (where && typeof where === 'object') { - Object.keys(where).forEach(function(key) { - var ex = where[key]; - - if (ex && ex.near) { - result = { - near: ex.near, - maxDistance: ex.maxDistance, - minDistance: ex.minDistance, - unit: ex.unit, - key: key, - }; + Object.keys(clause).forEach(function(clauseKey) { + if (Array.isArray(clause[clauseKey])) { + clause[clauseKey].forEach(function(el, index) { + var ret = nearSearch(el, parentKeys.concat(clauseKey).concat(index)); + if (ret) return ret; + }); + } else { + if (clause[clauseKey].hasOwnProperty('near')) { + var result = clause[clauseKey]; + nearResults.push({ + near: result.near, + maxDistance: result.maxDistance, + minDistance: result.minDistance, + unit: result.unit, + // If key is at root, define a single string, otherwise append it to the full path array + mongoKey: parentKeys.length ? parentKeys.concat(clauseKey) : clauseKey, + key: clauseKey, + }); + } } }); } + var nearResults = []; + nearSearch(where); - return result; + return (!nearResults.length ? false : nearResults); }; /*! - * Filter a set of objects using the given `nearFilter`. + * Filter a set of results using the given filters returned by `nearFilter()`. + * Can support multiple locations, but will include results from all of them. + * + * WARNING: "or" operator with GeoPoint does not work as expected, eg: + * {where: {or: [{location: {near: (29,-90)}},{name:'Sean'}]}} + * Will actually work as if you had used "and". This is because geo filtering + * takes place outside of the SQL query, so the result set of "name = Sean" is + * returned by the database, and then the location filtering happens in the app + * logic. So the "near" operator is always an "and" of the SQL filters, and "or" + * of other GeoPoint filters. + * + * Additionally, since this step occurs after the SQL result set is returned, + * if using GeoPoints with pagination the result set may be smaller than the + * page size. The page size is enforced at the DB level, and then we may + * remove results at the Geo-app level. If we "limit: 25", but 4 of those results + * do not have a matching geopoint field, the request will only return 21 results. + * This may make it erroneously look like a given page is the end of the result set. */ -exports.filter = function(arr, filter) { - var origin = filter.near; - var max = filter.maxDistance > 0 ? filter.maxDistance : false; - var min = filter.minDistance > 0 ? filter.minDistance : false; - var unit = filter.unit; - var key = filter.key; - - // create distance index +exports.filter = function(rawResults, filters) { var distances = {}; - var result = []; + var results = []; - arr.forEach(function(obj) { - var loc = obj[key]; + filters.forEach(function(filter) { + var origin = filter.near; + var max = filter.maxDistance > 0 ? filter.maxDistance : false; + var min = filter.minDistance > 0 ? filter.minDistance : false; + var unit = filter.unit; + var key = filter.key; - // filter out objects without locations - if (!loc) return; + // create distance index + rawResults.forEach(function(result) { + var loc = result[key]; - if (!(loc instanceof GeoPoint)) { - loc = GeoPoint(loc); - } + // filter out results without locations + if (!loc) return; - if (typeof loc.lat !== 'number') return; - if (typeof loc.lng !== 'number') return; + if (!(loc instanceof GeoPoint)) loc = GeoPoint(loc); - var d = GeoPoint.distanceBetween(origin, loc, {type: unit}); + if (typeof loc.lat !== 'number') return; + if (typeof loc.lng !== 'number') return; - if (min && d < min) { - return; - } - if (max && d > max) { - // dont add - } else { - distances[obj.id] = d; - result.push(obj); - } + var d = GeoPoint.distanceBetween(origin, loc, {type: unit}); + + // filter result if distance is either < minDistance or > maxDistance + if ((min && d < min) || (max && d > max)) return; + + distances[result.id] = d; + results.push(result); + }); + + results.sort(function(resA, resB) { + var a = resA[key]; + var b = resB[key]; + + if (a && b) { + var da = distances[resA.id]; + var db = distances[resB.id]; + + if (db === da) return 0; + return da > db ? 1 : -1; + } else { + return 0; + } + }); }); - return result.sort(function(objA, objB) { - var a = objA[key]; - var b = objB[key]; - - if (a && b) { - var da = distances[objA.id]; - var db = distances[objB.id]; - - if (db === da) return 0; - return da > db ? 1 : -1; - } else { - return 0; - } - }); + return results; }; exports.GeoPoint = GeoPoint; @@ -285,4 +312,3 @@ function geoDistance(x1, y1, x2, y2, options) { return 2 * Math.asin(f) * EARTH_RADIUS[type]; } - diff --git a/test/basic-querying.test.js b/test/basic-querying.test.js index d9859eb2..2d973d55 100644 --- a/test/basic-querying.test.js +++ b/test/basic-querying.test.js @@ -38,6 +38,7 @@ describe('basic-querying', function() { name: String, }, ], + addressLoc: {type: 'GeoPoint'}, }); db.automigrate(done); @@ -553,6 +554,147 @@ describe('basic-querying', function() { }); }); + describe('geo queries', function() { + describe('near filter', function() { + it('supports a basic "near" query', function(done) { + User.find({ + where: { + addressLoc: { + near: '29.9,-90.07', + }, + }, + }, function(err, users) { + if (err) done(err); + users.should.have.property('length', 3); + users[0].name.should.equal('John Lennon'); + users[0].addressLoc.should.not.be.null(); + done(); + }); + }); + + it('supports "near" inside a coumpound query with "and"', function(done) { + User.find({ + where: { + and: [ + { + addressLoc: { + near: '29.9,-90.07', + }, + }, + { + vip: true, + }, + ], + }, + }, function(err, users) { + if (err) done(err); + users.should.have.property('length', 2); + users[0].name.should.equal('John Lennon'); + users[0].addressLoc.should.not.be.null(); + users[0].vip.should.be.true(); + done(); + }); + }); + + it('supports "near" inside a complex coumpound query with multiple "and"', function(done) { + User.find({ + where: { + and: [ + { + and: [ + { + addressLoc: { + near: '29.9,-90.07', + }, + }, + { + order: 2, + }, + ], + }, + { + vip: true, + }, + ], + }, + }, function(err, users) { + if (err) done(err); + users.should.have.property('length', 1); + users[0].name.should.equal('John Lennon'); + users[0].addressLoc.should.not.be.null(); + users[0].vip.should.be.true(); + users[0].order.should.equal(2); + done(); + }); + }); + + it('supports multiple "near" queries with "or"', function(done) { + User.find({ + where: { + or: [ + { + addressLoc: { + near: '29.9,-90.04', + maxDistance: 300, + }, + }, + { + addressLoc: { + near: '22.97,-88.03', + maxDistance: 50, + }, + }, + ], + }, + }, function(err, users) { + if (err) done(err); + users.should.have.property('length', 2); + users[0].addressLoc.should.not.be.null(); + users[0].name.should.equal('Paul McCartney'); + users[1].addressLoc.should.not.equal(null); + users[1].name.should.equal('John Lennon'); + done(); + }); + }); + + it('supports multiple "near" queries with "or" ' + + 'inside a coumpound query with "and"', function(done) { + User.find({ + where: { + and: [ + { + or: [ + { + addressLoc: { + near: '29.9,-90.04', + maxDistance: 300, + }, + }, + { + addressLoc: { + near: '22.7,-89.03', + maxDistance: 50, + }, + }, + ], + }, + { + vip: true, + }, + ], + }, + }, function(err, users) { + if (err) done(err); + users.should.have.property('length', 1); + users[0].addressLoc.should.not.be.null(); + users[0].name.should.equal('John Lennon'); + users[0].vip.should.be.true(); + done(); + }); + }); + }); + }); + it('should only include fields as specified', function(done) { var remaining = 0; @@ -590,8 +732,9 @@ describe('basic-querying', function() { } sample({name: true}).expect(['name']); - sample({name: false}).expect(['id', 'seq', 'email', 'role', 'order', 'birthday', 'vip', - 'address', 'friends']); + sample({name: false}).expect([ + 'id', 'seq', 'email', 'role', 'order', 'birthday', 'vip', 'address', 'friends', 'addressLoc', + ]); sample({name: false, id: true}).expect(['id']); sample({id: true}).expect(['id']); sample('id').expect(['id']); @@ -1005,6 +1148,7 @@ function seed(done) { {name: 'George Harrison'}, {name: 'Ringo Starr'}, ], + addressLoc: {lat: 29.97, lng: -90.03}, }, { seq: 1, @@ -1025,8 +1169,15 @@ function seed(done) { {name: 'George Harrison'}, {name: 'Ringo Starr'}, ], + addressLoc: {lat: 22.97, lng: -88.03}, + }, + { + seq: 2, + name: 'George Harrison', + order: 5, + vip: false, + addressLoc: {lat: 22.7, lng: -89.03}, }, - {seq: 2, name: 'George Harrison', order: 5, vip: false}, {seq: 3, name: 'Ringo Starr', order: 6, vip: false}, {seq: 4, name: 'Pete Best', order: 4}, {seq: 5, name: 'Stuart Sutcliffe', order: 3, vip: true}, diff --git a/test/geo.test.js b/test/geo.test.js index 20fc2dd2..a005c372 100644 --- a/test/geo.test.js +++ b/test/geo.test.js @@ -147,8 +147,8 @@ describe('GeoPoint', function() { }, }; var filter = nearFilter(where); - filter.key.should.equal('location'); - filter.should.have.properties({ + filter[0].key.should.equal('location'); + filter[0].should.have.properties({ key: 'location', near: { lat: 40.77492964101182, @@ -187,7 +187,7 @@ describe('GeoPoint', function() { lng: 121.483687, }, }]; - var filter = { + var filter = [{ key: 'location', near: { lat: 30.278562, @@ -195,7 +195,7 @@ describe('GeoPoint', function() { }, unit: 'meters', minDistance: 10000, - }; + }]; var results = geoFilter(points, filter); results.length.should.be.equal(3); });