From d4972c0eb2024f2df669d0bed0666ba3ddd8afcd Mon Sep 17 00:00:00 2001 From: Karl STEIN Date: Tue, 28 Jul 2015 21:02:04 -1000 Subject: [PATCH] version 0.2.2 - added UploadFS.config.storesPath to set the endpoint URL path - added UploadFS.Store.prototype.onRead method that is called before returning a file from a http request - added UploadFS.Store.prototype.getFileURL method logic directly in this package - added UploadFS.Store.prototype.getURL method to get the store URL - automatically remove the temp file before removing the document - do not retry upload if the error is 404 or 400 --- README.md | 33 +++++++- package.js | 5 +- ufs-server.js | 208 ++++++++++++++++++++++++++++++++++++++++++++++++ ufs-store.js | 42 ++++++++-- ufs-uploader.js | 21 +++-- ufs.js | 166 +++++--------------------------------- 6 files changed, 311 insertions(+), 164 deletions(-) create mode 100644 ufs-server.js diff --git a/README.md b/README.md index 5af9a5f..0c344c6 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Meteor.photosStore = new UploadFS.store.Local({ }); ``` -### Transforming +### Transforming files If you need to modify the file before it is saved to the store, you have to use the **transform** option. ```js @@ -108,6 +108,37 @@ Meteor.photos.allow({ }); ``` +### Configuring endpoint + +Uploaded files will be accessible via a default URL, you can change it, but don't change after having uploaded files because you will break the URL of previous stored files. + +```js +// This path will be appended to the site URL, be sure to not put a "/" as first character +// for example, a PNG file with the _id 12345 in the "photos" store will be available via this URL : +// http://www.yourdomain.com/my/custom/path/photos/12345.png +UploadFS.config.storesPath = 'my/custom/path'; +``` + +### Security + +When returning the file for a HTTP request on the endpoint, you can do some checks to decide whether or not the file should be sent to the client. +This is done by defining the **onRead()** method on the store. + +```js +Meteor.photosStore = new UploadFS.store.Local({ + collection: Meteor.photos, + name: 'photos', + path: '/uploads/photos', + onRead: function (fileId, request, response) { + if (isPrivateFile(fileId, request)) { + throw new Meteor.Error(403, 'the file is private'); + // Because this code is executed before reading and returning the file, + // throwing an exception will simply stops returning the file to the client. + } + } +}); +``` + ### Uploading a file When the store on the server is configured, you can upload a file. diff --git a/package.js b/package.js index f6aa116..2913ef1 100644 --- a/package.js +++ b/package.js @@ -1,6 +1,6 @@ Package.describe({ name: 'jalik:ufs', - version: '0.2.1', + version: '0.2.2', author: 'karl.stein.pro@gmail.com', summary: 'Base package for UploadFS', homepage: 'https://github.com/jalik/jalik-ufs', @@ -13,9 +13,10 @@ Package.onUse(function (api) { api.use(['underscore', 'check']); api.use(['matb33:collection-hooks@0.7.13']); api.use(['minimongo', 'mongo-livedata', 'templating', 'reactive-var'], 'client'); - api.use(['mongo'], 'server'); + api.use(['mongo', 'webapp'], 'server'); api.addFiles(['ufs.js', 'ufs-store.js', 'ufs-filter.js']); api.addFiles(['ufs-uploader.js'], 'client'); + api.addFiles(['ufs-server.js'], 'server'); api.export('UploadFS'); }); diff --git a/ufs-server.js b/ufs-server.js new file mode 100644 index 0000000..6dc0d8b --- /dev/null +++ b/ufs-server.js @@ -0,0 +1,208 @@ +if (Meteor.isServer) { + fs = Npm.require('fs'); + Future = Npm.require('fibers/future'); + mkdirp = Npm.require('mkdirp'); + zlib = Npm.require('zlib'); + + // Create the temporary upload dir + Meteor.startup(function () { + createTempDir(); + }); + + Meteor.methods({ + /** + * Completes the file transfer + * @param fileId + * @param storeName + */ + ufsComplete: function (fileId, storeName) { + check(fileId, String); + check(storeName, String); + + var store = UploadFS.getStore(storeName); + + // Check arguments + if (!store) { + throw new Meteor.Error(404, 'store "' + storeName + '" does not exist'); + } + + // Check that file exists and is owned by current user + if (store.getCollection().find({_id: fileId, userId: this.userId}).count() < 1) { + throw new Meteor.Error(404, 'file "' + fileId + '" does not exist'); + } + + var fut = new Future(); + var tmpFile = UploadFS.getTempFilePath(fileId); + var writeStream = store.getWriteStream(fileId); + var readStream = fs.createReadStream(tmpFile, { + flags: 'r', + encoding: null, + autoClose: true + }); + + readStream.on('error', function (err) { + console.error(err); + store.delete(fileId); + fut.throw(err); + }); + + writeStream.on('error', function (err) { + console.error(err); + store.delete(fileId); + fut.throw(err); + }); + + writeStream.on('finish', Meteor.bindEnvironment(function () { + // Delete the temporary file + Meteor.setTimeout(function () { + fs.unlink(tmpFile); + }, 500); + + // Sets the file URL when file transfer is complete, + // this way, the image will loads entirely. + store.getCollection().update(fileId, { + $set: { + complete: true, + uploading: false, + uploadedAt: new Date(), // todo use UTC date + url: store.getFileURL(fileId) + } + }); + + fut.return(true); + })); + + // Execute transformation + store.transform(readStream, writeStream, fileId); + + return fut.wait(); + }, + + /** + * Saves a chunk of file + * @param chunk + * @param fileId + * @param storeName + * @return {*} + */ + ufsWrite: function (chunk, fileId, storeName) { + check(fileId, String); + check(storeName, String); + + // Check arguments + if (!(chunk instanceof Uint8Array)) { + throw new Meteor.Error(400, 'chunk is not an Uint8Array'); + } + if (chunk.length <= 0) { + throw new Meteor.Error(400, 'chunk is empty'); + } + + var store = UploadFS.getStore(storeName); + if (!store) { + throw new Meteor.Error(404, 'store ' + storeName + ' does not exist'); + } + + // Check that file exists, is not complete and is owned by current user + if (store.getCollection().find({_id: fileId, complete: false, userId: this.userId}).count() < 1) { + throw new Meteor.Error(404, 'file ' + fileId + ' does not exist'); + } + + var fut = new Future(); + var tmpFile = UploadFS.getTempFilePath(fileId); + fs.appendFile(tmpFile, new Buffer(chunk), function (err) { + if (err) { + console.error(err); + fs.unlink(tmpFile); + fut.throw(err); + } else { + fut.return(chunk.length); + } + }); + return fut.wait(); + } + }); + + // Listen HTTP requests to serve files + WebApp.connectHandlers.use(function (req, res, next) { + // Quick check to see if request should be catch + if (req.url.indexOf(UploadFS.config.storesPath) === -1) { + next(); + return; + } + + // Remove store path + var path = req.url.substr(UploadFS.config.storesPath.length + 1); + + // Get store and file + var regExp = new RegExp('^\/([^\/]+)\/([^\/]+)$'); + var match = regExp.exec(path); + + if (match !== null) { + // Get store + var storeName = match[1]; + var store = UploadFS.getStore(storeName); + if (!store) { + res.writeHead(404, {}); + res.end(); + return; + } + + // Get file from database + var fileId = match[2].replace(/\.[^.]+$/, ''); + var file = store.getCollection().findOne(fileId); + if (!file) { + res.writeHead(404, {}); + res.end(); + return; + } + + // Execute callback to do some check (eg. security check) + if (typeof store.onRead === 'function') { + store.onRead.call(store, fileId, req, res); + } + + try { + // Get file stream + var rs = store.getReadStream(fileId); + var accept = req.headers['accept-encoding'] || ''; + + // Compress data if supported by the client + if (accept.match(/\bdeflate\b/)) { + res.writeHead(200, { + 'Content-Encoding': 'deflate', + 'Content-Type': file.type + }); + rs.pipe(zlib.createDeflate()).pipe(res); + + } else if (accept.match(/\bgzip\b/)) { + res.writeHead(200, { + 'Content-Encoding': 'gzip', + 'Content-Type': file.type + }); + rs.pipe(zlib.createGzip()).pipe(res); + + } else { + res.writeHead(200, {}); + rs.pipe(res); + } + } catch (err) { + console.error('Cannot read file ' + fileId); + throw err; + } + + } else { + next(); + } + }); + + function createTempDir() { + var path = UploadFS.config.tmpDir; + mkdirp(path, function (err) { + if (err) { + console.error('ufs: cannot create tmpDir ' + path); + } else { + console.log('ufs: created tmpDir ' + path); + } + }); + } +} \ No newline at end of file diff --git a/ufs-store.js b/ufs-store.js index 69d1ea1..8ea3cc8 100644 --- a/ufs-store.js +++ b/ufs-store.js @@ -11,6 +11,7 @@ UploadFS.Store = function (options) { collection: null, filter: null, name: null, + onRead: null, transform: null }, options); @@ -32,10 +33,16 @@ UploadFS.Store = function (options) { if (UploadFS.getStore(options.name)) { throw new TypeError('name already exists'); } + if (options.onRead && typeof options.onRead !== 'function') { + throw new TypeError('onRead is not a function'); + } if (options.transform && typeof options.transform !== 'function') { throw new TypeError('transform is not a function'); } + // Public attributes + self.onRead = options.onRead; + // Private attributes var collection = options.collection; var filter = options.filter; @@ -85,7 +92,6 @@ UploadFS.Store = function (options) { }; } - // Add file information before insertion collection.before.insert(function (userId, file) { file.complete = false; file.uploading = true; @@ -94,11 +100,16 @@ UploadFS.Store = function (options) { file.userId = userId; }); - // Automatically delete file from store - // when the file is removed from the collection collection.before.remove(function (userId, file) { - if (Meteor.isServer && file.complete) { - self.delete(file._id); + if (Meteor.isServer) { + // Delete the physical file in the store + if (file.complete) { + self.delete(file._id); + } + // Delete the temporary file if uploading + if (file.uploading || !file.complete) { + fs.unlink(UploadFS.getTempFilePath(file._id)); + } } }); @@ -117,7 +128,17 @@ UploadFS.Store = function (options) { * @param fileId */ UploadFS.Store.prototype.getFileURL = function (fileId) { - throw new Error('getFileURL is not implemented'); + var file = this.getCollection().findOne(fileId, { + fields: {extension: 1} + }); + return file && this.getURL() + '/' + fileId + '.' + file.extension; +}; + +/** + * Returns the store URL + */ +UploadFS.Store.prototype.getURL = function () { + return Meteor.absoluteUrl(UploadFS.config.storesPath + '/' + this.getName()); }; if (Meteor.isServer) { @@ -145,4 +166,13 @@ if (Meteor.isServer) { UploadFS.Store.prototype.getWriteStream = function (fileId) { throw new Error('getWriteStream is not implemented'); }; + + /** + * Called when a file is read from the store + * @param fileId + * @param request + * @param response + */ + UploadFS.Store.prototype.onRead = function (fileId, request, response) { + }; } \ No newline at end of file diff --git a/ufs-uploader.js b/ufs-uploader.js index 15a7945..49b41b4 100644 --- a/ufs-uploader.js +++ b/ufs-uploader.js @@ -66,17 +66,15 @@ UploadFS.Uploader = function (options) { * Aborts the current transfer */ self.abort = function () { + uploading.set(false); + complete.set(false); + loaded.set(0); + fileId = null; + offset = 0; + tries = 0; + // Remove the file from database - store.getCollection().remove(fileId, function (err, result) { - if (!err && result) { - uploading.set(false); - complete.set(false); - loaded.set(0); - fileId = null; - offset = 0; - tries = 0; - } - }); + store.getCollection().remove(fileId); }; /** @@ -137,7 +135,8 @@ UploadFS.Uploader = function (options) { Meteor.call('ufsWrite', chunk, fileId, store.getName(), function (err, length) { if (err || !length) { // Retry until max tries is reach - if (tries < self.maxTries) { + // But don't retry if these errors occur + if (tries < self.maxTries && !_.contains([400, 404], err.error)) { tries += 1; // Wait 1 sec before retrying diff --git a/ufs.js b/ufs.js index fc217a4..fd14ee0 100644 --- a/ufs.js +++ b/ufs.js @@ -1,8 +1,27 @@ var stores = {}; UploadFS = { - config: {}, + config: { + /** + * The path where to put uploads before saving to a store + * @type {string} + */ + tmpDir: '/tmp/ufs', + /** + * The path of the URL where files are accessible + * @type {string} + */ + storesPath: 'ufs' + }, store: {}, + /** + * Returns the temporary file path + * @param fileId + * @return {string} + */ + getTempFilePath: function (fileId) { + return UploadFS.config.tmpDir + '/' + fileId; + }, /** * Returns the store by its name * @param name @@ -36,152 +55,11 @@ UploadFS = { var file = files[i]; (function (file) { - reader.onloadend = function (ev) { + reader.onload = function (ev) { callback.call(UploadFS, ev.target.result, file); }; reader.readAsArrayBuffer(file); })(file); } } -}; - - -if (Meteor.isServer) { - var Future = Npm.require('fibers/future'); - var mkdirp = Npm.require('mkdirp'); - var fs = Npm.require('fs'); - - /** - * The path to store uploads before saving to a store - * @type {string} - */ - UploadFS.config.tmpDir = '/tmp/ufs'; - - // Create the temporary upload dir - Meteor.startup(function () { - createTempDir(); - }); - - Meteor.methods({ - /** - * Completes the file transfer - * @param fileId - * @param storeName - */ - ufsComplete: function (fileId, storeName) { - check(fileId, String); - check(storeName, String); - - // Check arguments - if (!stores[storeName]) { - throw new Error('store does not exist'); - } - var store = stores[storeName]; - - // Check that file exists and is owned by current user - if (store.getCollection().find({_id: fileId, userId: this.userId}).count() < 1) { - throw new Error('file does not exist'); - } - - var fut = new Future(); - var tmpFile = UploadFS.config.tmpDir + '/' + fileId; - var writeStream = store.getWriteStream(fileId); - var readStream = fs.createReadStream(tmpFile, { - flags: 'r', - encoding: null, - autoClose: true - }); - - readStream.on('error', function (err) { - console.error(err); - store.delete(fileId); - fut.throw(err); - }); - - writeStream.on('error', function (err) { - console.error(err); - store.delete(fileId); - fut.throw(err); - }); - - writeStream.on('finish', Meteor.bindEnvironment(function () { - // Delete the temporary file - Meteor.setTimeout(function () { - fs.unlink(tmpFile); - }, 500); - - // Sets the file URL when file transfer is complete, - // this way, the image will loads entirely. - store.getCollection().update(fileId, { - $set: { - complete: true, - uploading: false, - uploadedAt: new Date(), - url: store.getFileURL(fileId) - } - }); - - fut.return(true); - })); - - // Execute transformation - store.transform(readStream, writeStream, fileId); - - return fut.wait(); - }, - - /** - * Saves a chunk of file - * @param chunk - * @param fileId - * @param storeName - * @return {*} - */ - ufsWrite: function (chunk, fileId, storeName) { - check(fileId, String); - check(storeName, String); - - // Check arguments - if (!(chunk instanceof Uint8Array)) { - throw new TypeError('chunk is not an Uint8Array'); - } - if (chunk.length <= 0) { - throw new Error('chunk is empty'); - } - if (!stores[storeName]) { - throw new Error('store does not exist'); - } - - var store = stores[storeName]; - - // Check that file exists, is not complete and is owned by current user - if (store.getCollection().find({_id: fileId, complete: false, userId: this.userId}).count() < 1) { - throw new Error('file does not exist'); - } - - var fut = new Future(); - var tmpFile = UploadFS.config.tmpDir + '/' + fileId; - fs.appendFile(tmpFile, new Buffer(chunk), function (err) { - if (err) { - console.error(err); - fs.unlink(tmpFile); - fut.throw(err); - } else { - fut.return(chunk.length); - } - }); - return fut.wait(); - } - }); -} - -function createTempDir() { - var path = UploadFS.config.tmpDir; - mkdirp(path, function (err) { - if (err) { - console.error('ufs: cannot create tmpDir ' + path); - } else { - console.log('ufs: created tmpDir ' + path); - } - }); -} \ No newline at end of file +}; \ No newline at end of file