From 2cbc1143c11d98ce41e4f66f9860db31af214195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 19 Oct 2016 12:55:35 +0200 Subject: [PATCH 1/3] Add ModelKeyComposer from kv-redis connector Add two helper methods for composing and parsing key-value keys: - in juggler, we use (modelName, key) tuple - in backends, there is usually a single string key required --- index.js | 4 ++ lib/model-key-composer.js | 76 +++++++++++++++++++++++++++++ lib/utils.js | 20 ++++++++ package.json | 1 + test/model-key-composer.test.js | 86 +++++++++++++++++++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 lib/model-key-composer.js create mode 100644 lib/utils.js create mode 100644 test/model-key-composer.test.js diff --git a/index.js b/index.js index 00bb7fd..71911ab 100644 --- a/index.js +++ b/index.js @@ -12,3 +12,7 @@ exports.SQLConnector = exports.SqlConnector = require('./lib/sql'); exports.ParameterizedSQL = exports.SQLConnector.ParameterizedSQL; exports.Transaction = require('./lib/transaction'); +exports.createPromiseCallback = require('./lib/utils').createPromiseCallback; + +// KeyValue helpers +exports.ModelKeyComposer = require('./lib/model-key-composer'); diff --git a/lib/model-key-composer.js b/lib/model-key-composer.js new file mode 100644 index 0000000..abc073f --- /dev/null +++ b/lib/model-key-composer.js @@ -0,0 +1,76 @@ +// Copyright IBM Corp. 2016. All Rights Reserved. +// Node module: loopback-connector +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +'use strict'; + +var createPromiseCallback = require('./utils').createPromiseCallback; +var debug = require('debug')('loopback:connector:model-key-composer'); +var g = require('strong-globalize')(); + +/** + * Build a single key string from a tuple (modelName, key). + * + * This method is typically used by KeyValue connectors to build a single + * key string for a given modelName+key tuple. + * + * @param {String} modelName + * @param {String} key + * @callback {Function} cb The callback to receive the composed value. + * @param {Error} err + * @param {String} composedKey + * @promise + */ +exports.compose = function composeKeyFromModelNameAndKey(modelName, key, cb) { + cb = cb || createPromiseCallback(); + + // Escape model name to prevent collision + // 'model' + 'foo:bar' --vs-- 'model:foo' + 'bar' + var value = encodeURIComponent(modelName) + ':' + key; + + setImmediate(function() { + cb(null, value); + }); + return cb.promise; +}; + +var PARSE_KEY_REGEX = /^([^:]*):(.*)/; + +/** + * Parse a composed key string into a tuple (modelName, key). + * + * This method is typically used by KeyValue connectors to parse a composed + * key string returned by SCAN/ITERATE method back to the expected + * modelName+tuple key. + * + * @param {String} composed The composed key as returned by `composeKey` + * @callback {Function} cb The callback to receive the parsed result. + * @param {Error} err + * @param {Object} result The result with properties `modelName` and `key`. + * @promise + */ +exports.parse = function(composed, cb) { + cb = cb || createPromiseCallback(); + + var matchResult = composed.match(PARSE_KEY_REGEX); + if (matchResult) { + var result = { + modelName: matchResult[1], + key: matchResult[2], + }; + setImmediate(function() { + cb(null, result); + }); + } else { + debug('Invalid key - missing model-name prefix: %s', composed); + var err = new Error(g.f( + 'Invalid key %j - missing model-name prefix', + composed)); + err.code = 'NO_MODEL_PREFIX'; + setImmediate(function() { + cb(err); + }); + } + return cb.promise; +}; diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..5850905 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,20 @@ +// Copyright IBM Corp. 2012,2016. All Rights Reserved. +// Node module: loopback-datasource-juggler +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +var Promise = require('bluebird'); + +exports.createPromiseCallback = createPromiseCallback; + +function createPromiseCallback() { + var cb; + var promise = new Promise(function(resolve, reject) { + cb = function(err, data) { + if (err) return reject(err); + return resolve(data); + }; + }); + cb.promise = promise; + return cb; +} diff --git a/package.json b/package.json index cbdce02..b1720e7 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "license": "MIT", "dependencies": { "async": "^1.0.0", + "bluebird": "^3.4.6", "debug": "^2.2.0", "strong-globalize": "^2.5.8" }, diff --git a/test/model-key-composer.test.js b/test/model-key-composer.test.js new file mode 100644 index 0000000..4e42649 --- /dev/null +++ b/test/model-key-composer.test.js @@ -0,0 +1,86 @@ +// Copyright IBM Corp. 2016. All Rights Reserved. +// Node module: loopback-connector +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +'use strict'; + +var composer = require('../lib/model-key-composer'); +var expect = require('chai').expect; +var Promise = require('bluebird'); + +describe('ModelKeyComposer', function() { + describe('compose()', function() { + it('honours the key', function() { + return Promise.all([ + composer.compose('Car', 'vin'), + composer.compose('Car', 'name'), + ]).spread(function(key1, key2) { + expect(key1).to.not.equal(key2); + }); + }); + + it('honours the model name', function() { + return Promise.all([ + composer.compose('Product', 'name'), + composer.compose('Category', 'name'), + ]).spread(function(key1, key2) { + expect(key1).to.not.equal(key2); + }); + }); + + it('encodes values', function() { + // This test is based on the knowledge that we are using ':' separator + // when building the composed string + return Promise.all([ + composer.compose('a', 'b:c'), + composer.compose('a:b', 'c'), + ]).spread(function(key1, key2) { + expect(key1).to.not.equal(key2); + }); + }); + + it('supports invocation with a callback', function(done) { + composer.compose('Car', 'vin', done); + }); + }); + + describe('parse()', function() { + it('decodes valid value', function() { + return composer.compose('Car', 'vin') + .then(function(data) { + return composer.parse(data); + }) + .then(function(parsed) { + expect(parsed).to.eql({ + modelName: 'Car', + key: 'vin', + }); + }); + }); + + it('handles invalid values', function() { + return composer.parse('invalid').then( + function onSuccess() { + throw new Error('composer.parse() should have failed'); + }, + function onError(err) { + expect(err).to.have.property('code', 'NO_MODEL_PREFIX'); + }); + }); + + it('supports invocation with a callback', function(done) { + composer.compose('Car', 'vin', function(err, key) { + if (err) return done(err); + composer.parse(key, function(err, parsed) { + if (err) return done(err); + expect(parsed).to.eql({ + modelName: 'Car', + key: 'vin', + }); + done(); + }); + }); + }); + }); +}); From 6fd3ac728536f91a21e8fddfd3f4fb00f8a0e286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 19 Oct 2016 13:14:55 +0200 Subject: [PATCH 2/3] Add BinaryPacker from kv-redis connector Add a helper for encoding JavaScript values into binary Buffers. The implemenetation is based on msgpack5 format and preserves JavaScript objects like Buffers and Dates, as opposed to (binary)JSON. --- index.js | 1 + lib/binary-packer.js | 80 ++++++++++++++++++++++++++++++++++++++ package.json | 1 + test/binary-packer.test.js | 72 ++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+) create mode 100644 lib/binary-packer.js create mode 100644 test/binary-packer.test.js diff --git a/index.js b/index.js index 71911ab..9f52288 100644 --- a/index.js +++ b/index.js @@ -16,3 +16,4 @@ exports.createPromiseCallback = require('./lib/utils').createPromiseCallback; // KeyValue helpers exports.ModelKeyComposer = require('./lib/model-key-composer'); +exports.BinaryPacker = require('./lib/binary-packer'); diff --git a/lib/binary-packer.js b/lib/binary-packer.js new file mode 100644 index 0000000..71d175e --- /dev/null +++ b/lib/binary-packer.js @@ -0,0 +1,80 @@ +// Copyright IBM Corp. 2014,2016. All Rights Reserved. +// Node module: loopback-connector +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +'use strict'; + +var createPromiseCallback = require('./utils').createPromiseCallback; +var msgpack = require('msgpack5'); + +module.exports = BinaryPacker; + +/** + * Create a new Packer instance that can be used to convert between JavaScript + * objects and a binary representation in a Buffer. + * + * Compared to JSON, this encoding preserves the following JavaScript types: + * - Date + */ +function BinaryPacker() { + this._packer = msgpack({ forceFloat64: true }); + this._packer.register(1, Date, encodeDate, decodeDate); +} + +/** + * Encode the provided value to a `Buffer`. + * + * @param {*} value Any value (string, number, object) + * @callback {Function} cb The callback to receive the parsed result. + * @param {Error} err + * @param {Buffer} data The encoded value + * @promise + */ +BinaryPacker.prototype.encode = function(value, cb) { + cb = cb || createPromiseCallback(); + try { + // msgpack5 returns https://www.npmjs.com/package/bl instead of Buffer + // use .slice() to convert to a Buffer + var data = this._packer.encode(value).slice(); + setImmediate(function() { + cb(null, data); + }); + } catch (err) { + setImmediate(function() { + cb(err); + }); + } + return cb.promise; +}; + +/** + * Decode the binary value back to a JavaScript value. + * @param {Buffer} binary The binary input. + * @callback {Function} cb The callback to receive the composed value. + * @param {Error} err + * @param {*} value Decoded value. + * @promise + */ +BinaryPacker.prototype.decode = function(binary, cb) { + cb = cb || createPromiseCallback(); + try { + var value = this._packer.decode(binary); + setImmediate(function() { + cb(null, value); + }); + } catch (err) { + setImmediate(function() { + cb(err); + }); + } + return cb.promise; +}; + +function encodeDate(obj) { + return new Buffer(obj.toISOString(), 'utf8'); +} + +function decodeDate(buf) { + return new Date(buf.toString('utf8')); +} diff --git a/package.json b/package.json index b1720e7..b156c7c 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "async": "^1.0.0", "bluebird": "^3.4.6", "debug": "^2.2.0", + "msgpack5": "^3.4.1", "strong-globalize": "^2.5.8" }, "devDependencies": { diff --git a/test/binary-packer.test.js b/test/binary-packer.test.js new file mode 100644 index 0000000..c2e6c32 --- /dev/null +++ b/test/binary-packer.test.js @@ -0,0 +1,72 @@ +// Copyright IBM Corp. 2016. All Rights Reserved. +// Node module: loopback-connector +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +'use strict'; + +var BinaryPacker = require('../lib/binary-packer'); +var expect = require('chai').expect; + +describe('BinaryPacker', function() { + var packer; + + beforeEach(function createPacker() { + packer = new BinaryPacker(); + }); + + describe('encode()', function() { + it('supports invocation with a callback', function(done) { + packer.encode('a-value', done); + }); + }); + + describe('decode()', function() { + it('supports invocation with a callback', function(done) { + packer.encode('a-value', function(err, binary) { + if (err) return done(err); + packer.decode(binary, function(err, result) { + if (err) return done(err); + expect(result).to.eql('a-value'); + done(); + }); + }); + }); + }); + + describe('roundtrip', function() { + var TEST_CASES = { + String: 'a-value', + Object: { a: 1, b: 2 }, + Buffer: new Buffer([1, 2, 3]), + Date: new Date('2016-08-03T11:53:03.470Z'), + Integer: 12345, + Float: 12.345, + Boolean: false, + }; + + Object.keys(TEST_CASES).forEach(function(tc) { + it('works for ' + tc + ' values', function() { + var value = TEST_CASES[tc]; + return encodeAndDecode(value) + .then(function(result) { + expect(result).to.eql(value); + }); + }); + }); + + it('works for nested properties', function() { + return encodeAndDecode(TEST_CASES) + .then(function(result) { + expect(result).to.eql(TEST_CASES); + }); + }); + + function encodeAndDecode(value) { + return packer.encode(value) + .then(function(binary) { + return packer.decode(binary); + }); + } + }); +}); From 512ff29aa9e31d163c4fb8e6abca04bcc979aae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 24 Oct 2016 10:05:25 +0200 Subject: [PATCH 3/3] Increase delay in tests to stop intermittent fails --- test/transaction.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/transaction.test.js b/test/transaction.test.js index 0f5a4ae..42bb666 100644 --- a/test/transaction.test.js +++ b/test/transaction.test.js @@ -182,7 +182,7 @@ describe('transactions', function() { expect(posts.length).to.be.eql(1); done(); }); - }, 100); + }, 300); done(); });