verdnatura-mirror
/
loopback-datasource-juggler
mirror of https://github.com/loopbackio/loopback-datasource-juggler
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rewrite redis test to make possible filter and sort simultaneously

This commit is contained in:
Anatoliy Chakkaev 2012-01-19 20:18:57 +04:00
parent f21b3f2a43
commit eb4e13922a
2 changed files with 175 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
lib/adapters/redis.js
View File

@ -125,13 +125,16 @@ BridgeToRedis.prototype.possibleIndexes = function (model, filter) {
if (!filter || Object.keys(filter.where || {}).length === 0) return false;
var foundIndex = [];
var noIndex = [];
Object.keys(filter.where).forEach(function (key) {
if (this.indexes[model][key] && typeof filter.where[key] === 'string') {
if (this.indexes[model][key] && (typeof filter.where[key] === 'string' || typeof filter.where[key] === 'number')) {
foundIndex.push('i:' + model + ':' + key + ':' + filter.where[key]);
} else {
noIndex.push(key);
}
}.bind(this));
return foundIndex;
return [foundIndex, noIndex];
};
BridgeToRedis.prototype.all = function all(model, filter, callback) {
@ -145,6 +148,90 @@ BridgeToRedis.prototype.all = function all(model, filter, callback) {
var props = this._models[model].properties;
var allNumeric = true;
// TODO: we need strict mode when filtration only possible when we have indexes
// WHERE
if (filter && filter.where) {
var pi = this.possibleIndexes(model, filter);
var indexes = pi[0];
var noIndexes = pi[1];
if (indexes.length) {
cmd = 'SINTER "' + indexes.join('" "') + '"';
if (noIndexes.length) {
log(model + ': no indexes found for ', noIndexes.join(', '),
'slow sorting and filtering');
}
indexes.push(noIndexes.length ? orderLimitStageBad : orderLimitStage);
client.sinter.apply(client, indexes);
} else {
// filter manually
cmd = 'KEYS ' + model + ':*';
client.keys(model + ':*', orderLimitStageBad);
}
} else {
// no filtering, just sort/limit (if any)
gotKeys('*');
}
// bad case when we trying to filter on non-indexed fields
// in bad case we need retrieve all data and filter/limit/sort manually
function orderLimitStageBad(err, keys) {
log(cmd, t1);
var t2 = Date.now();
if (err) {
return callback(err, []);
}
var query = keys.map(function (key) {
return ['hgetall', key];
});
client.multi(query).exec(function (err, replies) {
log(query, t2);
gotFilteredData(err, replies.filter(applyFilter(filter)));
});
function gotFilteredData(err, nodes) {
if (err) return callback(null);
if (filter.order) {
var allNumeric = true;
var orders = filter.order;
if (typeof filter.order === "string") {
orders = [filter.order];
}
orders.forEach(function (key) {
if (props[key].type.name !== 'Number' && props[key].type.name !== 'Date') {
allNumeric = false;
}
});
if (allNumeric) {
nodes = nodes.sort(numerically.bind(orders));
} else {
nodes = nodes.sort(literally.bind(orders));
}
}
// LIMIT
if (filter && filter.limit) {
var from = (filter.offset || 0), to = from + filter.limit;
callback(null, nodes.slice(from, to));
} else {
callback(null, nodes);
}
}
}
function orderLimitStage(err, keys) {
log(cmd, t1);
var t2 = Date.now();
if (err) {
return callback(err, []);
}
gotKeys(keys);
}
function gotKeys(keys) {
// ORDER
if (filter && filter.order) {
var orders = filter.order;
@ -158,8 +245,9 @@ BridgeToRedis.prototype.all = function all(model, filter, callback) {
sortCmd.push("BY", model + ":*->" + key);
});
}
// LIMIT
if (filter && filter.limit){
if (keys === '*' && filter && filter.limit){
var from = (filter.offset || 0), to = from + filter.limit;
sortCmd.push("LIMIT", from, to);
}
@ -171,40 +259,35 @@ BridgeToRedis.prototype.all = function all(model, filter, callback) {
sortCmd.push('ALPHA');
}
// do we need to sort or to query normally
if (sortCmd.length) {
sortCmd.unshift("s:" + model);
sortCmd.push("GET", "#");
cmd = "SORT " + sortCmd.join(" ");
var ttt = Date.now();
sortCmd.push(function(err, ids){
if (err) {
return callback(err, []);
}
var keys = ids.map(function (i) {
log(cmd, ttt);
var sortedKeys = ids.map(function (i) {
return model + ":" + i;
});
handleKeys(err, keys);
handleKeys(err, intersect(sortedKeys, keys));
});
client.sort.apply(client, sortCmd);
} else {
// Do a normal key lookup with possbible indexes
var indexes = this.possibleIndexes(model, filter);
if (indexes.length) {
cmd = 'SINTER "' + indexes.join('" "') + '"';
indexes.push(handleKeys);
client.sinter.apply(client, indexes, handleKeys);
} else {
// no sorting or filtering: just get all keys
if (keys === '*') {
cmd = 'KEYS ' + model + ':*';
client.keys(model + ':*', handleKeys);
} else {
handleKeys(null, keys);
}
}
}
function handleKeys(err, keys) {
log(cmd, t1);
var t2 = Date.now();
if (err) {
return callback(err, []);
}
var query = keys.map(function (key) {
return ['hgetall', key];
});
@ -214,6 +297,28 @@ BridgeToRedis.prototype.all = function all(model, filter, callback) {
callback(err, filter ? replies.filter(applyFilter(filter)) : replies);
});
}
return;
function numerically(a, b) {
return a[this[0]] - b[this[0]];
}
function literally(a, b) {
return a[this[0]] > b[this[0]];
}
// TODO: find better intersection method
function intersect(sortedKeys, filteredKeys) {
if (filteredKeys === '*') return sortedKeys;
var index = {};
filteredKeys.forEach(function (x) {
index[x] = true;
});
return sortedKeys.filter(function (x) {
return index[x];
});
}
};
function applyFilter(filter) {

26
test/common_test.js
View File

@ -59,7 +59,7 @@ function testOrm(schema) {
Post = schema.define('Post', {
title: { type: String, length: 255, index: true },
content: { type: Text },
date: { type: Date, default: Date.now },
date: { type: Date, default: Date.now, index: true },
published: { type: Boolean, default: false }
});
@ -424,16 +424,17 @@ function testOrm(schema) {
});
it('should handle ORDER clause', function (test) {
var titles = [ 'Title A', 'Title Z', 'Title M', 'Title B' ];
var dates = [ 5, 9, 0, 17 ];
var titles = [ 'Title A', 'Title Z', 'Title M', 'Title B', 'Title C' ];
var dates = [ 5, 9, 0, 17, 9 ];
titles.forEach(function (t, i) {
Post.create({title: t, date: dates[i]}, done);
});
var i = 0;
var i = 0, tests = 0;
function done(err, obj) {
if (++i === titles.length) {
doStringTest();
doFilterAndSortTest();
doNumberTest();
}
}
@ -441,6 +442,7 @@ function testOrm(schema) {
// Post.schema.log = console.log;
function doStringTest() {
tests += 1;
Post.all({order: 'title'}, function (err, posts) {
titles.sort().forEach(function (t, i) {
test.equal(posts[i].title, t);
@ -450,6 +452,7 @@ function testOrm(schema) {
}
function doNumberTest() {
tests += 1;
Post.all({order: 'date'}, function (err, posts) {
dates.sort(numerically).forEach(function (d, i) {
test.equal(posts[i].date, d);
@ -458,9 +461,22 @@ function testOrm(schema) {
});
}
function doFilterAndSortTest() {
tests += 1;
Post.all({where: {date: 9}, order: 'title', limit: 3}, function (err, posts) {
test.equal(posts.length, 2, 'Exactly 2 posts returned by query');
[ 'Title C', 'Title Z' ].forEach(function (t, i) {
if (posts[i]) {
test.equal(posts[i].title, t);
}
});
finished();
});
}
var fin = 0;
function finished() {
if (++fin === 2) {
if (++fin === tests) {
test.done();
}
}