Merge pull request #8 from strongloop/feature/refine-metadata

Make sure Container/File metadata are cleanly separated from other properties

example/.gitignore

@@ -0,0 +1 @@
+providers-private.json

example/app-cloud.js

@@ -1,89 +1,57 @@
 var StorageService = require('../').StorageService;
 var path = require('path');
-var providers = require('./providers.json');
+var providers = null;
+try {
+  providers = require('./providers-private.json');
+} catch(err) {
+  providers = require('./providers.json');
+}
 
-var rs = StorageService({
+function listContainersAndFiles(ss) {
-  provider: 'rackspace',
+  ss.getContainers(function (err, containers) {
-  username: providers.rackspace.username,
-  apiKey: providers.rackspace.apiKey
-});
-
-// Container
-
-rs.getContainers(function (err, containers) {
     if (err) {
       console.error(err);
       return;
     }
+    console.log('----------- %s (%d) ---------------', ss.provider, containers.length);
     containers.forEach(function (c) {
-    console.log('rackspace: ', c.name);
+      console.log('[%s] %s/', ss.provider, c.name);
       c.getFiles(function (err, files) {
         files.forEach(function (f) {
-        console.log('....', f.name);
+          console.log('[%s] ... %s', ss.provider, f.name);
         });
       });
     });
+  });
+}
+
+var rs = new StorageService({
+  provider: 'rackspace',
+  username: providers.rackspace.username,
+  apiKey: providers.rackspace.apiKey,
+  region: providers.rackspace.region
 });
 
-/*
+listContainersAndFiles(rs);
- client.createContainer(options, function (err, container) { });
- client.destroyContainer(containerName, function (err) { });
- client.getContainer(containerName, function (err, container) { });
 
- // File
+var s3 = new StorageService({
-
- client.upload(options, function (err) { });
- client.download(options, function (err) { });
- client.getFiles(container, function (err, files) { });
- client.getFile(container, file, function (err, server) { });
- client.removeFile(container, file, function (err) { });
- */
-
-var s3 = StorageService({
   provider: 'amazon',
   key: providers.amazon.key,
   keyId: providers.amazon.keyId
 });
 
-s3.getContainers(function (err, containers) {
+listContainersAndFiles(s3);
-  if (err) {
+
-    console.error(err);
-    return;
-  }
-  containers.forEach(function (c) {
-    console.log('amazon: ', c.name);
-    c.getFiles(function (err, files) {
-      files.forEach(function (f) {
-        console.log('....', f.name);
-      });
-    });
-  });
-});
 
 var fs = require('fs');
 var path = require('path');
 var stream = s3.uploadStream('con1', 'test.jpg');
-var input = fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream);
+fs.createReadStream(path.join(__dirname, 'test.jpg')).pipe(stream);
 
 var local = StorageService({
   provider: 'filesystem',
   root: path.join(__dirname, 'storage')
 });
 
-// Container
+listContainersAndFiles(local);
-
-local.getContainers(function (err, containers) {
-  if (err) {
-    console.error(err);
-    return;
-  }
-  containers.forEach(function (c) {
-    console.log('filesystem: ', c.name);
-    c.getFiles(function (err, files) {
-      files.forEach(function (f) {
-        console.log('....', f.name);
-      });
-    });
-  });
-});
 

example/providers.json

@@ -1,7 +1,8 @@
 {
   "rackspace": {
-    "username": "strongloop",
+    "username": "your-rackspace-username",
-    "apiKey": "your-rackspace-api-key"
+    "apiKey": "your-rackspace-api-key",
+    "region": "DFW"
   },
   "amazon": {
     "key": "your-amazon-key",

index.js

@@ -1,6 +1,4 @@
 var StorageConnector = require('./lib/storage-connector');
-StorageConnector.Container = require('./lib/models/container');
-StorageConnector.File = require('./lib/models/file');
 StorageConnector.StorageService = require('./lib/storage-service');
 
 module.exports = StorageConnector;

lib/factory.js

@@ -1,3 +1,65 @@
+var pkgcloud = require('pkgcloud');
+
+/*!
+ * Patch the prototype for a given subclass of Container or File
+ * @param {Function} cls The subclass
+ */
+function patchBaseClass(cls) {
+  var proto = cls.prototype;
+  var found = false;
+  // Find the prototype that owns the _setProperties method
+  while (proto
+    && proto.constructor !== pkgcloud.storage.Container
+    && proto.constructor !== pkgcloud.storage.File) {
+    if (proto.hasOwnProperty('_setProperties')) {
+      found = true;
+      break;
+    } else {
+      proto = Object.getPrototypeOf(proto);
+    }
+  }
+  if (!found) {
+    proto = cls.prototype;
+  }
+  var m1 = proto._setProperties;
+  proto._setProperties = function (details) {
+    // Use an empty object to receive the calculated properties from details
+    var receiver = {};
+    m1.call(receiver, details);
+    // Apply the calculated properties to this
+    for (var p in receiver) {
+      this[p] = receiver[p];
+    }
+    // Keep references to raw and the calculated properties
+    this._rawMetadata = details;
+    this._metadata = receiver; // Use _metadata to avoid conflicts
+  }
+
+  proto.toJSON = function () {
+    return this._metadata;
+  };
+
+  proto.getMetadata = function () {
+    return this._metadata;
+  };
+
+  proto.getRawMetadata = function () {
+    return this._rawMetadata;
+  };
+
+}
+/*!
+ * Patch the pkgcloud Container/File classes so that the metadata are separately
+ * stored for JSON serialization
+ *
+ * @param {String} provider The name of the storage provider
+ */
+function patchContainerAndFileClass(provider) {
+  var storageProvider = getProvider(provider).storage;
+
+  patchBaseClass(storageProvider.Container);
+  patchBaseClass(storageProvider.File);
+}
 /**
  * Create a client instance based on the options
  * @param options
@@ -15,7 +77,7 @@ function createClient(options) {
     // Fall back to pkgcloud
     handler = require('pkgcloud').storage;
   }
-
+  patchContainerAndFileClass(provider);
   return handler.createClient(options);
 }
 

lib/models/container.js

@@ -1,8 +0,0 @@
-var loopback = require('loopback');
-
-var Container = loopback.createModel('container', {
-  name: String,
-  url: String
-}, {idInjection: false, strict: false});
-
-module.exports = Container;

lib/models/file.js

@@ -1,9 +0,0 @@
-var loopback = require('loopback');
-
-var File = loopback.createModel('file', {
-  name: String,
-  url: String,
-  container: String
-}, {idInjection: false, strict: false});
-
-module.exports = File;

lib/providers/filesystem/index.js

@@ -8,6 +8,8 @@ var fs = require('fs'),
   File = require('./file').File,
   Container = require('./container').Container;
 
+module.exports.storage = module.exports; // To make it consistent with pkgcloud
+
 module.exports.File = File;
 module.exports.Container = Container;
 module.exports.Client = FileSystemProvider;
@@ -51,7 +53,24 @@ function validateName(name, cb) {
   }
 }
 
-// Container related functions
+/*!
+ * Populate the metadata from file stat into props
+ * @param {fs.Stats} stat The file stat instance
+ * @param {Object} props The metadata object
+ */
+function populateMetadata(stat, props) {
+  for (var p in stat) {
+    switch (p) {
+      case 'size':
+      case 'atime':
+      case 'mtime':
+      case 'ctime':
+        props[p] = stat[p];
+        break;
+    }
+  }
+}
+
 FileSystemProvider.prototype.getContainers = function (cb) {
   var self = this;
   fs.readdir(self.root, function (err, files) {
@@ -68,9 +87,7 @@ FileSystemProvider.prototype.getContainers = function (cb) {
           if (stat.isDirectory()) {
             var name = files[index];
             var props = {name: name};
-            for (var p in stat) {
+            populateMetadata(stat, props);
-              props[p] = stat[p];
-            }
             var container = new Container(self, props);
             containers.push(container);
           }
@@ -84,8 +101,20 @@ FileSystemProvider.prototype.getContainers = function (cb) {
 FileSystemProvider.prototype.createContainer = function (options, cb) {
   var self = this;
   var name = options.name;
-  validateName(name, cb) && fs.mkdir(path.join(this.root, name), options, function (err) {
+  var dir = path.join(this.root, name);
-    cb && cb(err, new Container(self, {name: name}));
+  validateName(name, cb) && fs.mkdir(dir, options, function (err) {
+    if(err) {
+      return cb && cb(err);
+    }
+    fs.stat(dir, function (err, stat) {
+      var container = null;
+      if (!err) {
+        var props = {name: name};
+        populateMetadata(stat, props);
+        container = new Container(self, props);
+      }
+      cb && cb(err, container);
+    });
   });
 };
 
@@ -116,9 +145,7 @@ FileSystemProvider.prototype.getContainer = function (containerName, cb) {
     var container = null;
     if (!err) {
       var props = {name: containerName};
-      for (var p in stat) {
+      populateMetadata(stat, props);
-        props[p] = stat[p];
-      }
       container = new Container(self, props);
     }
     cb && cb(err, container);
@@ -184,9 +211,7 @@ FileSystemProvider.prototype.getFiles = function (container, download, cb) {
         stats.forEach(function (stat, index) {
           if (stat.isFile()) {
             var props = {container: container, name: entries[index]};
-            for (var p in stat) {
+            populateMetadata(stat, props);
-              props[p] = stat[p];
-            }
             var file = new File(self, props);
             files.push(file);
           }
@@ -206,9 +231,7 @@ FileSystemProvider.prototype.getFile = function (container, file, cb) {
     var f = null;
     if (!err) {
       var props = {container: container, name: file};
-      for (var p in stat) {
+      populateMetadata(stat, props);
-        props[p] = stat[p];
-      }
       f = new File(self, props);
     }
     cb && cb(err, f);

lib/storage-service.js

@@ -3,9 +3,6 @@ var handler = require('./storage-handler');
 
 var storage = require('pkgcloud').storage;
 
-var Container = require('./models/container');
-var File = require('./models/file');
-
 module.exports = StorageService;
 
 /**
@@ -58,7 +55,7 @@ StorageService.prototype.getContainers = function (cb) {
       cb(err, containers);
     } else {
       cb(err, containers.map(function (c) {
-        return new Container(map(c));
+        return map(c);
       }));
     }
   });
@@ -119,8 +116,12 @@ StorageService.prototype.uploadStream = function (container, file, options, cb)
     options = {};
   }
   options = options || {};
-  if (container) options.container = container;
+  if (container) {
-  if (file) options.remote = file;
+    options.container = container;
+  }
+  if (file) {
+    options.remote = file;
+  }
 
   return this.client.upload(options, cb);
 };
@@ -138,8 +139,12 @@ StorageService.prototype.downloadStream = function (container, file, options, cb
     options = {};
   }
   options = options || {};
-  if (container) options.container = container;
+  if (container) {
-  if (file) options.remote = file;
+    options.container = container;
+  }
+  if (file) {
+    options.remote = file;
+  }
 
   return this.client.download(options, cb);
 };
@@ -156,7 +161,7 @@ StorageService.prototype.getFiles = function (container, download, cb) {
       cb(err, files);
     } else {
       cb(err, files.map(function (f) {
-        return new File(map(f));
+        return map(f);
       }));
     }
   });

test/fs.test.js

@@ -3,6 +3,17 @@ var FileSystemProvider = require('../lib/providers/filesystem/index.js').Client;
 var assert = require('assert');
 var path = require('path');
 
+function verifyMetadata(fileOrContainer, name) {
+  assert(fileOrContainer.getMetadata());
+  assert.equal(fileOrContainer.getMetadata().name, name);
+  assert(fileOrContainer.getMetadata().uid === undefined);
+  assert(fileOrContainer.getMetadata().gid === undefined);
+  assert(fileOrContainer.getMetadata().atime);
+  assert(fileOrContainer.getMetadata().ctime);
+  assert(fileOrContainer.getMetadata().mtime);
+  assert.equal(typeof fileOrContainer.getMetadata().size, 'number');
+}
+
 describe('FileSystem based storage provider', function () {
 
   describe('container apis', function () {
@@ -33,6 +44,7 @@ describe('FileSystem based storage provider', function () {
     it('should create a new container', function (done) {
       client.createContainer({name: 'c1'}, function (err, container) {
         assert(!err);
+        verifyMetadata(container, 'c1');
         done(err, container);
       });
     });
@@ -40,6 +52,7 @@ describe('FileSystem based storage provider', function () {
     it('should get a container c1', function (done) {
       client.getContainer('c1', function (err, container) {
         assert(!err);
+        verifyMetadata(container, 'c1');
         done(err, container);
       });
     });
@@ -114,6 +127,7 @@ describe('FileSystem based storage provider', function () {
       client.getFile('c1', 'f1.txt', function (err, f) {
         assert(!err);
         assert.ok(f);
+        verifyMetadata(f, 'f1.txt');
         done(err, f);
       });
     });

test/storage-service.test.js

@@ -20,6 +20,7 @@ describe('Storage service', function () {
     it('should create a new container', function (done) {
       storageService.createContainer({name: 'c1'}, function (err, container) {
         assert(!err);
+        assert(container.getMetadata());
         done(err, container);
       });
     });
@@ -27,6 +28,7 @@ describe('Storage service', function () {
     it('should get a container c1', function (done) {
       storageService.getContainer('c1', function (err, container) {
         assert(!err);
+        assert(container.getMetadata());
         done(err, container);
       });
     });
@@ -97,6 +99,7 @@ describe('Storage service', function () {
       storageService.getFile('c1', 'f1.txt', function (err, f) {
         assert(!err);
         assert.ok(f);
+        assert(f.getMetadata());
         done(err, f);
       });
     });

test/upload-download.test.js

@@ -17,6 +17,34 @@ var ds = loopback.createDataSource({
 var Container = ds.createModel('container');
 app.model(Container);
 
+/*!
+ * Verify that the JSON response has the correct metadata properties.
+ * Please note the metadata vary by storage providers. This test assumes
+ * the 'filesystem' provider.
+ *
+ * @param {String} containerOrFile The container/file object
+ * @param {String} [name] The name to be checked if not undefined
+ */
+function verifyMetadata(containerOrFile, name) {
+  assert(containerOrFile);
+
+  // Name
+  if (name) {
+    assert.equal(containerOrFile.name, name);
+  }
+  // No sensitive information
+  assert(containerOrFile.uid === undefined);
+  assert(containerOrFile.gid === undefined);
+
+  // Timestamps
+  assert(containerOrFile.atime);
+  assert(containerOrFile.ctime);
+  assert(containerOrFile.mtime);
+
+  // Size
+  assert.equal(typeof containerOrFile.size, 'number');
+}
+
 describe('storage service', function () {
   var server = null;
   before(function (done) {
@@ -38,7 +66,7 @@ describe('storage service', function () {
       .set('Content-Type', 'application/json')
       .expect('Content-Type', /json/)
       .expect(200, function (err, res) {
-        assert.equal(res.body.name, 'test-container');
+        verifyMetadata(res.body, 'test-container');
         done();
       });
   });
@@ -50,7 +78,7 @@ describe('storage service', function () {
       .set('Accept', 'application/json')
       .expect('Content-Type', /json/)
       .expect(200, function (err, res) {
-        assert.equal(res.body.name, 'test-container');
+        verifyMetadata(res.body, 'test-container');
         done();
       });
   });
@@ -64,6 +92,9 @@ describe('storage service', function () {
       .expect(200, function (err, res) {
         assert(Array.isArray(res.body));
         assert.equal(res.body.length, 2);
+        res.body.forEach(function(c) {
+          verifyMetadata(c);
+        });
         done();
       });
   });
@@ -100,6 +131,9 @@ describe('storage service', function () {
       .expect('Content-Type', /json/)
       .expect(200, function (err, res) {
         assert(Array.isArray(res.body));
+        res.body.forEach(function(f) {
+          verifyMetadata(f);
+        });
         done();
       });
   });
@@ -126,7 +160,7 @@ describe('storage service', function () {
       .set('Accept', 'application/json')
       .expect('Content-Type', /json/)
       .expect(200, function (err, res) {
-        assert.equal(res.body.name, 'test.jpg');
+        verifyMetadata(res.body, 'test.jpg');
         done();
       });
   });