diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000..5e82b51 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1 @@ +providers-private.json diff --git a/example/app-cloud.js b/example/app-cloud.js index fe9fddc..0060c36 100644 --- a/example/app-cloud.js +++ b/example/app-cloud.js @@ -1,89 +1,57 @@ var StorageService = require('../').StorageService; var path = require('path'); -var providers = require('./providers.json'); +var providers = null; +try { + providers = require('./providers-private.json'); +} catch(err) { + providers = require('./providers.json'); +} -var rs = StorageService({ - provider: 'rackspace', - username: providers.rackspace.username, - apiKey: providers.rackspace.apiKey -}); - -// Container - -rs.getContainers(function (err, containers) { - if (err) { - console.error(err); - return; - } - containers.forEach(function (c) { - console.log('rackspace: ', c.name); - c.getFiles(function (err, files) { - files.forEach(function (f) { - console.log('....', f.name); +function listContainersAndFiles(ss) { + ss.getContainers(function (err, containers) { + if (err) { + console.error(err); + return; + } + console.log('----------- %s (%d) ---------------', ss.provider, containers.length); + containers.forEach(function (c) { + console.log('[%s] %s/', ss.provider, c.name); + c.getFiles(function (err, files) { + files.forEach(function (f) { + console.log('[%s] ... %s', ss.provider, f.name); + }); }); }); }); +} + +var rs = new StorageService({ + provider: 'rackspace', + username: providers.rackspace.username, + apiKey: providers.rackspace.apiKey, + region: providers.rackspace.region }); -/* - client.createContainer(options, function (err, container) { }); - client.destroyContainer(containerName, function (err) { }); - client.getContainer(containerName, function (err, container) { }); +listContainersAndFiles(rs); - // File - - client.upload(options, function (err) { }); - client.download(options, function (err) { }); - client.getFiles(container, function (err, files) { }); - client.getFile(container, file, function (err, server) { }); - client.removeFile(container, file, function (err) { }); - */ - -var s3 = StorageService({ +var s3 = new StorageService({ provider: 'amazon', key: providers.amazon.key, keyId: providers.amazon.keyId }); -s3.getContainers(function (err, containers) { - if (err) { - console.error(err); - return; - } - containers.forEach(function (c) { - console.log('amazon: ', c.name); - c.getFiles(function (err, files) { - files.forEach(function (f) { - console.log('....', f.name); - }); - }); - }); -}); +listContainersAndFiles(s3); + var fs = require('fs'); var path = require('path'); var stream = s3.uploadStream('con1', 'test.jpg'); -var input = fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream); +fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream); var local = StorageService({ provider: 'filesystem', root: path.join(__dirname, 'storage') }); -// Container - -local.getContainers(function (err, containers) { - if (err) { - console.error(err); - return; - } - containers.forEach(function (c) { - console.log('filesystem: ', c.name); - c.getFiles(function (err, files) { - files.forEach(function (f) { - console.log('....', f.name); - }); - }); - }); -}); +listContainersAndFiles(local); diff --git a/example/providers.json b/example/providers.json index 14f76bf..15dd4ac 100644 --- a/example/providers.json +++ b/example/providers.json @@ -1,7 +1,8 @@ { "rackspace": { - "username": "strongloop", - "apiKey": "your-rackspace-api-key" + "username": "your-rackspace-username", + "apiKey": "your-rackspace-api-key", + "region": "DFW" }, "amazon": { "key": "your-amazon-key", diff --git a/index.js b/index.js index e56592e..c1f0429 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,4 @@ var StorageConnector = require('./lib/storage-connector'); -StorageConnector.Container = require('./lib/models/container'); -StorageConnector.File = require('./lib/models/file'); StorageConnector.StorageService = require('./lib/storage-service'); module.exports = StorageConnector; diff --git a/lib/factory.js b/lib/factory.js index dcea329..daf23f1 100644 --- a/lib/factory.js +++ b/lib/factory.js @@ -1,3 +1,65 @@ +var pkgcloud = require('pkgcloud'); + +/*! + * Patch the prototype for a given subclass of Container or File + * @param {Function} cls The subclass + */ +function patchBaseClass(cls) { + var proto = cls.prototype; + var found = false; + // Find the prototype that owns the _setProperties method + while (proto + && proto.constructor !== pkgcloud.storage.Container + && proto.constructor !== pkgcloud.storage.File) { + if (proto.hasOwnProperty('_setProperties')) { + found = true; + break; + } else { + proto = Object.getPrototypeOf(proto); + } + } + if (!found) { + proto = cls.prototype; + } + var m1 = proto._setProperties; + proto._setProperties = function (details) { + // Use an empty object to receive the calculated properties from details + var receiver = {}; + m1.call(receiver, details); + // Apply the calculated properties to this + for (var p in receiver) { + this[p] = receiver[p]; + } + // Keep references to raw and the calculated properties + this._rawMetadata = details; + this._metadata = receiver; // Use _metadata to avoid conflicts + } + + proto.toJSON = function () { + return this._metadata; + }; + + proto.getMetadata = function () { + return this._metadata; + }; + + proto.getRawMetadata = function () { + return this._rawMetadata; + }; + +} +/*! + * Patch the pkgcloud Container/File classes so that the metadata are separately + * stored for JSON serialization + * + * @param {String} provider The name of the storage provider + */ +function patchContainerAndFileClass(provider) { + var storageProvider = getProvider(provider).storage; + + patchBaseClass(storageProvider.Container); + patchBaseClass(storageProvider.File); +} /** * Create a client instance based on the options * @param options @@ -15,7 +77,7 @@ function createClient(options) { // Fall back to pkgcloud handler = require('pkgcloud').storage; } - + patchContainerAndFileClass(provider); return handler.createClient(options); } diff --git a/lib/models/container.js b/lib/models/container.js deleted file mode 100644 index dc81254..0000000 --- a/lib/models/container.js +++ /dev/null @@ -1,8 +0,0 @@ -var loopback = require('loopback'); - -var Container = loopback.createModel('container', { - name: String, - url: String -}, {idInjection: false, strict: false}); - -module.exports = Container; \ No newline at end of file diff --git a/lib/models/file.js b/lib/models/file.js deleted file mode 100644 index fb98ccc..0000000 --- a/lib/models/file.js +++ /dev/null @@ -1,9 +0,0 @@ -var loopback = require('loopback'); - -var File = loopback.createModel('file', { - name: String, - url: String, - container: String -}, {idInjection: false, strict: false}); - -module.exports = File; diff --git a/lib/providers/filesystem/index.js b/lib/providers/filesystem/index.js index 12a7c70..85043eb 100644 --- a/lib/providers/filesystem/index.js +++ b/lib/providers/filesystem/index.js @@ -8,6 +8,8 @@ var fs = require('fs'), File = require('./file').File, Container = require('./container').Container; +module.exports.storage = module.exports; // To make it consistent with pkgcloud + module.exports.File = File; module.exports.Container = Container; module.exports.Client = FileSystemProvider; @@ -51,7 +53,24 @@ function validateName(name, cb) { } } -// Container related functions +/*! + * Populate the metadata from file stat into props + * @param {fs.Stats} stat The file stat instance + * @param {Object} props The metadata object + */ +function populateMetadata(stat, props) { + for (var p in stat) { + switch (p) { + case 'size': + case 'atime': + case 'mtime': + case 'ctime': + props[p] = stat[p]; + break; + } + } +} + FileSystemProvider.prototype.getContainers = function (cb) { var self = this; fs.readdir(self.root, function (err, files) { @@ -68,9 +87,7 @@ FileSystemProvider.prototype.getContainers = function (cb) { if (stat.isDirectory()) { var name = files[index]; var props = {name: name}; - for (var p in stat) { - props[p] = stat[p]; - } + populateMetadata(stat, props); var container = new Container(self, props); containers.push(container); } @@ -84,8 +101,20 @@ FileSystemProvider.prototype.getContainers = function (cb) { FileSystemProvider.prototype.createContainer = function (options, cb) { var self = this; var name = options.name; - validateName(name, cb) && fs.mkdir(path.join(this.root, name), options, function (err) { - cb && cb(err, new Container(self, {name: name})); + var dir = path.join(this.root, name); + validateName(name, cb) && fs.mkdir(dir, options, function (err) { + if(err) { + return cb && cb(err); + } + fs.stat(dir, function (err, stat) { + var container = null; + if (!err) { + var props = {name: name}; + populateMetadata(stat, props); + container = new Container(self, props); + } + cb && cb(err, container); + }); }); }; @@ -116,9 +145,7 @@ FileSystemProvider.prototype.getContainer = function (containerName, cb) { var container = null; if (!err) { var props = {name: containerName}; - for (var p in stat) { - props[p] = stat[p]; - } + populateMetadata(stat, props); container = new Container(self, props); } cb && cb(err, container); @@ -184,9 +211,7 @@ FileSystemProvider.prototype.getFiles = function (container, download, cb) { stats.forEach(function (stat, index) { if (stat.isFile()) { var props = {container: container, name: entries[index]}; - for (var p in stat) { - props[p] = stat[p]; - } + populateMetadata(stat, props); var file = new File(self, props); files.push(file); } @@ -206,9 +231,7 @@ FileSystemProvider.prototype.getFile = function (container, file, cb) { var f = null; if (!err) { var props = {container: container, name: file}; - for (var p in stat) { - props[p] = stat[p]; - } + populateMetadata(stat, props); f = new File(self, props); } cb && cb(err, f); diff --git a/lib/storage-service.js b/lib/storage-service.js index f4b3a98..621722a 100644 --- a/lib/storage-service.js +++ b/lib/storage-service.js @@ -3,9 +3,6 @@ var handler = require('./storage-handler'); var storage = require('pkgcloud').storage; -var Container = require('./models/container'); -var File = require('./models/file'); - module.exports = StorageService; /** @@ -58,7 +55,7 @@ StorageService.prototype.getContainers = function (cb) { cb(err, containers); } else { cb(err, containers.map(function (c) { - return new Container(map(c)); + return map(c); })); } }); @@ -119,8 +116,12 @@ StorageService.prototype.uploadStream = function (container, file, options, cb) options = {}; } options = options || {}; - if (container) options.container = container; - if (file) options.remote = file; + if (container) { + options.container = container; + } + if (file) { + options.remote = file; + } return this.client.upload(options, cb); }; @@ -138,8 +139,12 @@ StorageService.prototype.downloadStream = function (container, file, options, cb options = {}; } options = options || {}; - if (container) options.container = container; - if (file) options.remote = file; + if (container) { + options.container = container; + } + if (file) { + options.remote = file; + } return this.client.download(options, cb); }; @@ -156,7 +161,7 @@ StorageService.prototype.getFiles = function (container, download, cb) { cb(err, files); } else { cb(err, files.map(function (f) { - return new File(map(f)); + return map(f); })); } }); diff --git a/test/fs.test.js b/test/fs.test.js index 27f19f3..e177e0a 100644 --- a/test/fs.test.js +++ b/test/fs.test.js @@ -3,6 +3,17 @@ var FileSystemProvider = require('../lib/providers/filesystem/index.js').Client; var assert = require('assert'); var path = require('path'); +function verifyMetadata(fileOrContainer, name) { + assert(fileOrContainer.getMetadata()); + assert.equal(fileOrContainer.getMetadata().name, name); + assert(fileOrContainer.getMetadata().uid === undefined); + assert(fileOrContainer.getMetadata().gid === undefined); + assert(fileOrContainer.getMetadata().atime); + assert(fileOrContainer.getMetadata().ctime); + assert(fileOrContainer.getMetadata().mtime); + assert.equal(typeof fileOrContainer.getMetadata().size, 'number'); +} + describe('FileSystem based storage provider', function () { describe('container apis', function () { @@ -33,6 +44,7 @@ describe('FileSystem based storage provider', function () { it('should create a new container', function (done) { client.createContainer({name: 'c1'}, function (err, container) { assert(!err); + verifyMetadata(container, 'c1'); done(err, container); }); }); @@ -40,6 +52,7 @@ describe('FileSystem based storage provider', function () { it('should get a container c1', function (done) { client.getContainer('c1', function (err, container) { assert(!err); + verifyMetadata(container, 'c1'); done(err, container); }); }); @@ -114,6 +127,7 @@ describe('FileSystem based storage provider', function () { client.getFile('c1', 'f1.txt', function (err, f) { assert(!err); assert.ok(f); + verifyMetadata(f, 'f1.txt'); done(err, f); }); }); diff --git a/test/storage-service.test.js b/test/storage-service.test.js index f2ea79b..2cdf2cb 100644 --- a/test/storage-service.test.js +++ b/test/storage-service.test.js @@ -20,6 +20,7 @@ describe('Storage service', function () { it('should create a new container', function (done) { storageService.createContainer({name: 'c1'}, function (err, container) { assert(!err); + assert(container.getMetadata()); done(err, container); }); }); @@ -27,6 +28,7 @@ describe('Storage service', function () { it('should get a container c1', function (done) { storageService.getContainer('c1', function (err, container) { assert(!err); + assert(container.getMetadata()); done(err, container); }); }); @@ -97,6 +99,7 @@ describe('Storage service', function () { storageService.getFile('c1', 'f1.txt', function (err, f) { assert(!err); assert.ok(f); + assert(f.getMetadata()); done(err, f); }); }); diff --git a/test/upload-download.test.js b/test/upload-download.test.js index a79aa85..20a0150 100644 --- a/test/upload-download.test.js +++ b/test/upload-download.test.js @@ -17,6 +17,34 @@ var ds = loopback.createDataSource({ var Container = ds.createModel('container'); app.model(Container); +/*! + * Verify that the JSON response has the correct metadata properties. + * Please note the metadata vary by storage providers. This test assumes + * the 'filesystem' provider. + * + * @param {String} containerOrFile The container/file object + * @param {String} [name] The name to be checked if not undefined + */ +function verifyMetadata(containerOrFile, name) { + assert(containerOrFile); + + // Name + if (name) { + assert.equal(containerOrFile.name, name); + } + // No sensitive information + assert(containerOrFile.uid === undefined); + assert(containerOrFile.gid === undefined); + + // Timestamps + assert(containerOrFile.atime); + assert(containerOrFile.ctime); + assert(containerOrFile.mtime); + + // Size + assert.equal(typeof containerOrFile.size, 'number'); +} + describe('storage service', function () { var server = null; before(function (done) { @@ -38,7 +66,7 @@ describe('storage service', function () { .set('Content-Type', 'application/json') .expect('Content-Type', /json/) .expect(200, function (err, res) { - assert.equal(res.body.name, 'test-container'); + verifyMetadata(res.body, 'test-container'); done(); }); }); @@ -50,7 +78,7 @@ describe('storage service', function () { .set('Accept', 'application/json') .expect('Content-Type', /json/) .expect(200, function (err, res) { - assert.equal(res.body.name, 'test-container'); + verifyMetadata(res.body, 'test-container'); done(); }); }); @@ -64,6 +92,9 @@ describe('storage service', function () { .expect(200, function (err, res) { assert(Array.isArray(res.body)); assert.equal(res.body.length, 2); + res.body.forEach(function(c) { + verifyMetadata(c); + }); done(); }); }); @@ -100,6 +131,9 @@ describe('storage service', function () { .expect('Content-Type', /json/) .expect(200, function (err, res) { assert(Array.isArray(res.body)); + res.body.forEach(function(f) { + verifyMetadata(f); + }); done(); }); }); @@ -126,7 +160,7 @@ describe('storage service', function () { .set('Accept', 'application/json') .expect('Content-Type', /json/) .expect(200, function (err, res) { - assert.equal(res.body.name, 'test.jpg'); + verifyMetadata(res.body, 'test.jpg'); done(); }); });