verdnatura-mirror
/
loopback-component-storage
mirror of https://github.com/strongloop/loopback-component-storage.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8 from strongloop/feature/refine-metadata 

Make sure Container/File metadata are cleanly separated from other properties
This commit is contained in:
Raymond Feng 2014-04-04 13:33:06 -07:00
parent d419e16169 af2a7adfb4
commit 160d478940
12 changed files with 205 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
example/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
providers-private.json

96
example/app-cloud.js
View File

@ -1,89 +1,57 @@
var StorageService = require('../').StorageService;
var path = require('path');
var providers = require('./providers.json');
var providers = null;
try {
providers = require('./providers-private.json');
} catch(err) {
providers = require('./providers.json');
}
var rs = StorageService({
provider: 'rackspace',
username: providers.rackspace.username,
apiKey: providers.rackspace.apiKey
});
// Container
rs.getContainers(function (err, containers) {
if (err) {
console.error(err);
return;
}
containers.forEach(function (c) {
console.log('rackspace: ', c.name);
c.getFiles(function (err, files) {
files.forEach(function (f) {
console.log('....', f.name);
function listContainersAndFiles(ss) {
ss.getContainers(function (err, containers) {
if (err) {
console.error(err);
return;
}
console.log('----------- %s (%d) ---------------', ss.provider, containers.length);
containers.forEach(function (c) {
console.log('[%s] %s/', ss.provider, c.name);
c.getFiles(function (err, files) {
files.forEach(function (f) {
console.log('[%s] ... %s', ss.provider, f.name);
});
});
});
});
}
var rs = new StorageService({
provider: 'rackspace',
username: providers.rackspace.username,
apiKey: providers.rackspace.apiKey,
region: providers.rackspace.region
});
/*
client.createContainer(options, function (err, container) { });
client.destroyContainer(containerName, function (err) { });
client.getContainer(containerName, function (err, container) { });
listContainersAndFiles(rs);
// File
client.upload(options, function (err) { });
client.download(options, function (err) { });
client.getFiles(container, function (err, files) { });
client.getFile(container, file, function (err, server) { });
client.removeFile(container, file, function (err) { });
*/
var s3 = StorageService({
var s3 = new StorageService({
provider: 'amazon',
key: providers.amazon.key,
keyId: providers.amazon.keyId
});
s3.getContainers(function (err, containers) {
if (err) {
console.error(err);
return;
}
containers.forEach(function (c) {
console.log('amazon: ', c.name);
c.getFiles(function (err, files) {
files.forEach(function (f) {
console.log('....', f.name);
});
});
});
});
listContainersAndFiles(s3);
var fs = require('fs');
var path = require('path');
var stream = s3.uploadStream('con1', 'test.jpg');
var input = fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream);
fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream);
var local = StorageService({
provider: 'filesystem',
root: path.join(__dirname, 'storage')
});
// Container
local.getContainers(function (err, containers) {
if (err) {
console.error(err);
return;
}
containers.forEach(function (c) {
console.log('filesystem: ', c.name);
c.getFiles(function (err, files) {
files.forEach(function (f) {
console.log('....', f.name);
});
});
});
});
listContainersAndFiles(local);

5
example/providers.json
View File

@ -1,7 +1,8 @@
{
"rackspace": {
"username": "strongloop",
"apiKey": "your-rackspace-api-key"
"username": "your-rackspace-username",
"apiKey": "your-rackspace-api-key",
"region": "DFW"
},
"amazon": {
"key": "your-amazon-key",

2
index.js
View File

@ -1,6 +1,4 @@
var StorageConnector = require('./lib/storage-connector');
StorageConnector.Container = require('./lib/models/container');
StorageConnector.File = require('./lib/models/file');
StorageConnector.StorageService = require('./lib/storage-service');
module.exports = StorageConnector;

64
lib/factory.js
View File

@ -1,3 +1,65 @@
var pkgcloud = require('pkgcloud');
/*!
* Patch the prototype for a given subclass of Container or File
* @param {Function} cls The subclass
*/
function patchBaseClass(cls) {
var proto = cls.prototype;
var found = false;
// Find the prototype that owns the _setProperties method
while (proto
&& proto.constructor !== pkgcloud.storage.Container
&& proto.constructor !== pkgcloud.storage.File) {
if (proto.hasOwnProperty('_setProperties')) {
found = true;
break;
} else {
proto = Object.getPrototypeOf(proto);
}
}
if (!found) {
proto = cls.prototype;
}
var m1 = proto._setProperties;
proto._setProperties = function (details) {
// Use an empty object to receive the calculated properties from details
var receiver = {};
m1.call(receiver, details);
// Apply the calculated properties to this
for (var p in receiver) {
this[p] = receiver[p];
}
// Keep references to raw and the calculated properties
this._rawMetadata = details;
this._metadata = receiver; // Use _metadata to avoid conflicts
}
proto.toJSON = function () {
return this._metadata;
};
proto.getMetadata = function () {
return this._metadata;
};
proto.getRawMetadata = function () {
return this._rawMetadata;
};
}
/*!
* Patch the pkgcloud Container/File classes so that the metadata are separately
* stored for JSON serialization
*
* @param {String} provider The name of the storage provider
*/
function patchContainerAndFileClass(provider) {
var storageProvider = getProvider(provider).storage;
patchBaseClass(storageProvider.Container);
patchBaseClass(storageProvider.File);
}
/**
* Create a client instance based on the options
* @param options
@ -15,7 +77,7 @@ function createClient(options) {
// Fall back to pkgcloud
handler = require('pkgcloud').storage;
}
patchContainerAndFileClass(provider);
return handler.createClient(options);
}

8
lib/models/container.js
View File

@ -1,8 +0,0 @@
var loopback = require('loopback');
var Container = loopback.createModel('container', {
name: String,
url: String
}, {idInjection: false, strict: false});
module.exports = Container;

9
lib/models/file.js
View File

@ -1,9 +0,0 @@
var loopback = require('loopback');
var File = loopback.createModel('file', {
name: String,
url: String,
container: String
}, {idInjection: false, strict: false});
module.exports = File;

53
lib/providers/filesystem/index.js
View File

@ -8,6 +8,8 @@ var fs = require('fs'),
File = require('./file').File,
Container = require('./container').Container;
module.exports.storage = module.exports; // To make it consistent with pkgcloud
module.exports.File = File;
module.exports.Container = Container;
module.exports.Client = FileSystemProvider;
@ -51,7 +53,24 @@ function validateName(name, cb) {
}
}
// Container related functions
/*!
* Populate the metadata from file stat into props
* @param {fs.Stats} stat The file stat instance
* @param {Object} props The metadata object
*/
function populateMetadata(stat, props) {
for (var p in stat) {
switch (p) {
case 'size':
case 'atime':
case 'mtime':
case 'ctime':
props[p] = stat[p];
break;
}
}
}
FileSystemProvider.prototype.getContainers = function (cb) {
var self = this;
fs.readdir(self.root, function (err, files) {
@ -68,9 +87,7 @@ FileSystemProvider.prototype.getContainers = function (cb) {
if (stat.isDirectory()) {
var name = files[index];
var props = {name: name};
for (var p in stat) {
props[p] = stat[p];
}
populateMetadata(stat, props);
var container = new Container(self, props);
containers.push(container);
}
@ -84,8 +101,20 @@ FileSystemProvider.prototype.getContainers = function (cb) {
FileSystemProvider.prototype.createContainer = function (options, cb) {
var self = this;
var name = options.name;
validateName(name, cb) && fs.mkdir(path.join(this.root, name), options, function (err) {
cb && cb(err, new Container(self, {name: name}));
var dir = path.join(this.root, name);
validateName(name, cb) && fs.mkdir(dir, options, function (err) {
if(err) {
return cb && cb(err);
}
fs.stat(dir, function (err, stat) {
var container = null;
if (!err) {
var props = {name: name};
populateMetadata(stat, props);
container = new Container(self, props);
}
cb && cb(err, container);
});
});
};
@ -116,9 +145,7 @@ FileSystemProvider.prototype.getContainer = function (containerName, cb) {
var container = null;
if (!err) {
var props = {name: containerName};
for (var p in stat) {
props[p] = stat[p];
}
populateMetadata(stat, props);
container = new Container(self, props);
}
cb && cb(err, container);
@ -184,9 +211,7 @@ FileSystemProvider.prototype.getFiles = function (container, download, cb) {
stats.forEach(function (stat, index) {
if (stat.isFile()) {
var props = {container: container, name: entries[index]};
for (var p in stat) {
props[p] = stat[p];
}
populateMetadata(stat, props);
var file = new File(self, props);
files.push(file);
}
@ -206,9 +231,7 @@ FileSystemProvider.prototype.getFile = function (container, file, cb) {
var f = null;
if (!err) {
var props = {container: container, name: file};
for (var p in stat) {
props[p] = stat[p];
}
populateMetadata(stat, props);
f = new File(self, props);
}
cb && cb(err, f);

23
lib/storage-service.js
View File

@ -3,9 +3,6 @@ var handler = require('./storage-handler');
var storage = require('pkgcloud').storage;
var Container = require('./models/container');
var File = require('./models/file');
module.exports = StorageService;
/**
@ -58,7 +55,7 @@ StorageService.prototype.getContainers = function (cb) {
cb(err, containers);
} else {
cb(err, containers.map(function (c) {
return new Container(map(c));
return map(c);
}));
}
});
@ -119,8 +116,12 @@ StorageService.prototype.uploadStream = function (container, file, options, cb)
options = {};
}
options = options || {};
if (container) options.container = container;
if (file) options.remote = file;
if (container) {
options.container = container;
}
if (file) {
options.remote = file;
}
return this.client.upload(options, cb);
};
@ -138,8 +139,12 @@ StorageService.prototype.downloadStream = function (container, file, options, cb
options = {};
}
options = options || {};
if (container) options.container = container;
if (file) options.remote = file;
if (container) {
options.container = container;
}
if (file) {
options.remote = file;
}
return this.client.download(options, cb);
};
@ -156,7 +161,7 @@ StorageService.prototype.getFiles = function (container, download, cb) {
cb(err, files);
} else {
cb(err, files.map(function (f) {
return new File(map(f));
return map(f);
}));
}
});

14
test/fs.test.js
View File

@ -3,6 +3,17 @@ var FileSystemProvider = require('../lib/providers/filesystem/index.js').Client;
var assert = require('assert');
var path = require('path');
function verifyMetadata(fileOrContainer, name) {
assert(fileOrContainer.getMetadata());
assert.equal(fileOrContainer.getMetadata().name, name);
assert(fileOrContainer.getMetadata().uid === undefined);
assert(fileOrContainer.getMetadata().gid === undefined);
assert(fileOrContainer.getMetadata().atime);
assert(fileOrContainer.getMetadata().ctime);
assert(fileOrContainer.getMetadata().mtime);
assert.equal(typeof fileOrContainer.getMetadata().size, 'number');
}
describe('FileSystem based storage provider', function () {
describe('container apis', function () {
@ -33,6 +44,7 @@ describe('FileSystem based storage provider', function () {
it('should create a new container', function (done) {
client.createContainer({name: 'c1'}, function (err, container) {
assert(!err);
verifyMetadata(container, 'c1');
done(err, container);
});
});
@ -40,6 +52,7 @@ describe('FileSystem based storage provider', function () {
it('should get a container c1', function (done) {
client.getContainer('c1', function (err, container) {
assert(!err);
verifyMetadata(container, 'c1');
done(err, container);
});
});
@ -114,6 +127,7 @@ describe('FileSystem based storage provider', function () {
client.getFile('c1', 'f1.txt', function (err, f) {
assert(!err);
assert.ok(f);
verifyMetadata(f, 'f1.txt');
done(err, f);
});
});

3
test/storage-service.test.js
View File

@ -20,6 +20,7 @@ describe('Storage service', function () {
it('should create a new container', function (done) {
storageService.createContainer({name: 'c1'}, function (err, container) {
assert(!err);
assert(container.getMetadata());
done(err, container);
});
});
@ -27,6 +28,7 @@ describe('Storage service', function () {
it('should get a container c1', function (done) {
storageService.getContainer('c1', function (err, container) {
assert(!err);
assert(container.getMetadata());
done(err, container);
});
});
@ -97,6 +99,7 @@ describe('Storage service', function () {
storageService.getFile('c1', 'f1.txt', function (err, f) {
assert(!err);
assert.ok(f);
assert(f.getMetadata());
done(err, f);
});
});

40
test/upload-download.test.js
View File

@ -17,6 +17,34 @@ var ds = loopback.createDataSource({
var Container = ds.createModel('container');
app.model(Container);
/*!
* Verify that the JSON response has the correct metadata properties.
* Please note the metadata vary by storage providers. This test assumes
* the 'filesystem' provider.
*
* @param {String} containerOrFile The container/file object
* @param {String} [name] The name to be checked if not undefined
*/
function verifyMetadata(containerOrFile, name) {
assert(containerOrFile);
// Name
if (name) {
assert.equal(containerOrFile.name, name);
}
// No sensitive information
assert(containerOrFile.uid === undefined);
assert(containerOrFile.gid === undefined);
// Timestamps
assert(containerOrFile.atime);
assert(containerOrFile.ctime);
assert(containerOrFile.mtime);
// Size
assert.equal(typeof containerOrFile.size, 'number');
}
describe('storage service', function () {
var server = null;
before(function (done) {
@ -38,7 +66,7 @@ describe('storage service', function () {
.set('Content-Type', 'application/json')
.expect('Content-Type', /json/)
.expect(200, function (err, res) {
assert.equal(res.body.name, 'test-container');
verifyMetadata(res.body, 'test-container');
done();
});
});
@ -50,7 +78,7 @@ describe('storage service', function () {
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect(200, function (err, res) {
assert.equal(res.body.name, 'test-container');
verifyMetadata(res.body, 'test-container');
done();
});
});
@ -64,6 +92,9 @@ describe('storage service', function () {
.expect(200, function (err, res) {
assert(Array.isArray(res.body));
assert.equal(res.body.length, 2);
res.body.forEach(function(c) {
verifyMetadata(c);
});
done();
});
});
@ -100,6 +131,9 @@ describe('storage service', function () {
.expect('Content-Type', /json/)
.expect(200, function (err, res) {
assert(Array.isArray(res.body));
res.body.forEach(function(f) {
verifyMetadata(f);
});
done();
});
});
@ -126,7 +160,7 @@ describe('storage service', function () {
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect(200, function (err, res) {
assert.equal(res.body.name, 'test.jpg');
verifyMetadata(res.body, 'test.jpg');
done();
});
});