diff --git a/README.md b/README.md index 37d6b12..4721f05 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,8 @@ store.write(readStream, fileId, function(err, file) { ## Uploading files +### Uploading from a file + When the store on the server is configured, you can upload files to it. Here is the template to upload one or more files : @@ -391,6 +393,25 @@ Template.upload.events({ }); ``` +### Uploading from a URL + +If you want to upload a file directly from a URL, use the `importFromURL(url, fileAttr, storeName, callback)` method. +This method is available both on the client and the server. + +```js +var url = 'https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png'; +var attributes = { name: 'Google Logo', description: 'Logo from www.google.com' }; + +UploadFS.importFromURL(url, attributes, PhotosStore, function (err, fileId) { + if (err) { + displayError(err); + } else { + console.log('Photo saved ', fileId); + } +}); + +``` + ## Displaying images After that, if everything went good, you have you file saved to the store and in database. diff --git a/package.js b/package.js index 1f98083..993f00c 100644 --- a/package.js +++ b/package.js @@ -1,6 +1,6 @@ Package.describe({ name: 'jalik:ufs', - version: '0.5.1', + version: '0.5.2', author: 'karl.stein.pro@gmail.com', summary: 'Base package for UploadFS', homepage: 'https://github.com/jalik/jalik-ufs', @@ -24,6 +24,7 @@ Package.onUse(function (api) { api.addFiles('ufs-store.js'); api.addFiles('ufs-helpers.js', 'client'); api.addFiles('ufs-uploader.js', 'client'); + api.addFiles('ufs-methods.js', 'server'); api.addFiles('ufs-server.js', 'server'); api.export('UploadFS'); diff --git a/ufs-methods.js b/ufs-methods.js new file mode 100644 index 0000000..1c6975f --- /dev/null +++ b/ufs-methods.js @@ -0,0 +1,170 @@ +Meteor.methods({ + + /** + * Completes the file transfer + * @param fileId + * @param storeName + */ + ufsComplete: function (fileId, storeName) { + check(fileId, String); + check(storeName, String); + + // Allow other uploads to run concurrently + this.unblock(); + + var store = UploadFS.getStore(storeName); + if (!store) { + throw new Meteor.Error(404, 'store "' + storeName + '" does not exist'); + } + // Check that file exists and is owned by current user + if (store.getCollection().find({_id: fileId, userId: this.userId}).count() < 1) { + throw new Meteor.Error(404, 'file "' + fileId + '" does not exist'); + } + + var fut = new Future(); + var tmpFile = UploadFS.getTempFilePath(fileId); + + // Get the temp file + var rs = fs.createReadStream(tmpFile, { + flags: 'r', + encoding: null, + autoClose: true + }); + + rs.on('error', Meteor.bindEnvironment(function () { + store.getCollection().remove(fileId); + })); + + // Save file in the store + store.write(rs, fileId, Meteor.bindEnvironment(function (err, file) { + fs.unlink(tmpFile, function (err) { + err && console.error('ufs: cannot delete temp file ' + tmpFile + ' (' + err.message + ')'); + }); + + if (err) { + fut.throw(err); + } else { + fut.return(file); + } + })); + return fut.wait(); + }, + + /** + * Imports a file from the URL + * @param url + * @param file + * @param storeName + * @return {*} + */ + ufsImportURL: function (url, file, storeName) { + check(url, String); + check(file, Object); + check(storeName, String); + + this.unblock(); + + var store = UploadFS.getStore(storeName); + if (!store) { + throw new Meteor.Error(404, 'Store "' + storeName + '" does not exist'); + } + + try { + // Extract file info + if (!file.name) { + file.name = url.replace(/\?.*$/, '').split('/').pop(); + file.extension = file.name.split('.').pop(); + file.type = 'image/' + file.extension; + } + // Check if file is valid + if (store.getFilter() instanceof UploadFS.Filter) { + store.getFilter().check(file); + } + // Create the file + var fileId = store.create(file); + + } catch (err) { + throw new Meteor.Error(500, err.message); + } + + var fut = new Future(); + var proto; + + // Detect protocol to use + if (/http:\/\//i.test(url)) { + proto = http; + } else if (/https:\/\//i.test(url)) { + proto = https; + } + + // Download file + proto.get(url, Meteor.bindEnvironment(function (res) { + // Save the file in the store + store.write(res, fileId, function (err, file) { + if (err) { + fut.throw(err); + } else { + fut.return(fileId); + } + }); + })).on('error', function (err) { + fut.throw(err); + }); + return fut.wait(); + }, + + /** + * Saves a chunk of file + * @param chunk + * @param fileId + * @param storeName + * @param progress + * @return {*} + */ + ufsWrite: function (chunk, fileId, storeName, progress) { + check(fileId, String); + check(storeName, String); + check(progress, Number); + + this.unblock(); + + // Check arguments + if (!(chunk instanceof Uint8Array)) { + throw new Meteor.Error(400, 'chunk is not an Uint8Array'); + } + if (chunk.length <= 0) { + throw new Meteor.Error(400, 'chunk is empty'); + } + + var store = UploadFS.getStore(storeName); + if (!store) { + throw new Meteor.Error(404, 'store ' + storeName + ' does not exist'); + } + + // Check that file exists, is not complete and is owned by current user + if (store.getCollection().find({_id: fileId, complete: false, userId: this.userId}).count() < 1) { + throw new Meteor.Error(404, 'file ' + fileId + ' does not exist'); + } + + var fut = new Future(); + var tmpFile = UploadFS.getTempFilePath(fileId); + + // Save the chunk + fs.appendFile(tmpFile, new Buffer(chunk), Meteor.bindEnvironment(function (err) { + if (err) { + console.error('ufs: cannot write chunk of file "' + fileId + '" (' + err.message + ')'); + fs.unlink(tmpFile, function (err) { + err && console.error('ufs: cannot delete temp file ' + tmpFile + ' (' + err.message + ')'); + }); + fut.throw(err); + } else { + // Update completed state + store.getCollection().update(fileId, { + $set: {progress: progress} + }); + fut.return(chunk.length); + } + })); + return fut.wait(); + } +}); diff --git a/ufs-server.js b/ufs-server.js index 02ca074..e65a168 100644 --- a/ufs-server.js +++ b/ufs-server.js @@ -1,253 +1,147 @@ -if (Meteor.isServer) { - domain = Npm.require('domain'); - fs = Npm.require('fs'); - Future = Npm.require('fibers/future'); - mkdirp = Npm.require('mkdirp'); - stream = Npm.require('stream'); - zlib = Npm.require('zlib'); - - Meteor.startup(function () { - var path = UploadFS.config.tmpDir; - var mode = '0744'; - - fs.stat(path, function (err) { - if (err) { - // Create the temp directory - mkdirp(path, {mode: mode}, function (err) { - if (err) { - console.error('ufs: cannot create temp directory at ' + path + ' (' + err.message + ')'); - } else { - console.log('ufs: temp directory created at ' + path); - } - }); - } else { - // Set directory permissions - fs.chmod(path, mode, function (err) { - err && console.error('ufs: cannot set temp directory permissions ' + mode + ' (' + err.message + ')'); - }); - } - }); - }); - - Meteor.methods({ - /** - * Completes the file transfer - * @param fileId - * @param storeName - */ - ufsComplete: function (fileId, storeName) { - check(fileId, String); - check(storeName, String); - - // Allow other uploads to run concurrently - this.unblock(); - - var store = UploadFS.getStore(storeName); - if (!store) { - throw new Meteor.Error(404, 'store "' + storeName + '" does not exist'); - } - // Check that file exists and is owned by current user - if (store.getCollection().find({_id: fileId, userId: this.userId}).count() < 1) { - throw new Meteor.Error(404, 'file "' + fileId + '" does not exist'); - } - - var fut = new Future(); - var tmpFile = UploadFS.getTempFilePath(fileId); - - // Get the temp file - var rs = fs.createReadStream(tmpFile, { - flags: 'r', - encoding: null, - autoClose: true - }); - - rs.on('error', Meteor.bindEnvironment(function () { - store.getCollection().remove(fileId); - })); - - // Save file in the store - store.write(rs, fileId, Meteor.bindEnvironment(function (err, file) { - fs.unlink(tmpFile, function (err) { - err && console.error('ufs: cannot delete temp file ' + tmpFile + ' (' + err.message + ')'); - }); - +domain = Npm.require('domain'); +fs = Npm.require('fs'); +Future = Npm.require('fibers/future'); +http = Npm.require('http'); +https = Npm.require('https'); +mkdirp = Npm.require('mkdirp'); +stream = Npm.require('stream'); +zlib = Npm.require('zlib'); + +Meteor.startup(function () { + var path = UploadFS.config.tmpDir; + var mode = '0744'; + + fs.stat(path, function (err) { + if (err) { + // Create the temp directory + mkdirp(path, {mode: mode}, function (err) { if (err) { - fut.throw(err); + console.error('ufs: cannot create temp directory at ' + path + ' (' + err.message + ')'); } else { - fut.return(file); + console.log('ufs: temp directory created at ' + path); } - })); - return fut.wait(); - }, - - /** - * Saves a chunk of file - * @param chunk - * @param fileId - * @param storeName - * @param progress - * @return {*} - */ - ufsWrite: function (chunk, fileId, storeName, progress) { - check(fileId, String); - check(storeName, String); - - this.unblock(); - - // Check arguments - if (!(chunk instanceof Uint8Array)) { - throw new Meteor.Error(400, 'chunk is not an Uint8Array'); - } - if (chunk.length <= 0) { - throw new Meteor.Error(400, 'chunk is empty'); - } - - var store = UploadFS.getStore(storeName); - if (!store) { - throw new Meteor.Error(404, 'store ' + storeName + ' does not exist'); - } - - // Check that file exists, is not complete and is owned by current user - if (store.getCollection().find({_id: fileId, complete: false, userId: this.userId}).count() < 1) { - throw new Meteor.Error(404, 'file ' + fileId + ' does not exist'); - } - - var fut = new Future(); - var tmpFile = UploadFS.getTempFilePath(fileId); - - // Save the chunk - fs.appendFile(tmpFile, new Buffer(chunk), Meteor.bindEnvironment(function (err) { - if (err) { - console.error('ufs: cannot write chunk of file "' + fileId + '" (' + err.message + ')'); - fs.unlink(tmpFile, function (err) { - err && console.error('ufs: cannot delete temp file ' + tmpFile + ' (' + err.message + ')'); - }); - fut.throw(err); - } else { - // Update completed state - store.getCollection().update(fileId, { - $set: {progress: progress} - }); - fut.return(chunk.length); - } - })); - return fut.wait(); + }); + } else { + // Set directory permissions + fs.chmod(path, mode, function (err) { + err && console.error('ufs: cannot set temp directory permissions ' + mode + ' (' + err.message + ')'); + }); } }); - - // Create domain to handle errors - // and possibly avoid server crashes. - var d = domain.create(); - - d.on('error', function (err) { - console.error('ufs: ' + err.message); - }); - - // Listen HTTP requests to serve files - WebApp.connectHandlers.use(function (req, res, next) { - // Quick check to see if request should be catch - if (req.url.indexOf(UploadFS.config.storesPath) === -1) { - next(); +}); + +// Create domain to handle errors +// and possibly avoid server crashes. +var d = domain.create(); + +d.on('error', function (err) { + console.error('ufs: ' + err.message); +}); + +// Listen HTTP requests to serve files +WebApp.connectHandlers.use(function (req, res, next) { + // Quick check to see if request should be catch + if (req.url.indexOf(UploadFS.config.storesPath) === -1) { + next(); + return; + } + + // Remove store path + var path = req.url.substr(UploadFS.config.storesPath.length + 1); + + // Get store, file Id and file name + var regExp = new RegExp('^\/([^\/]+)\/([^\/]+)(?:\/([^\/]+))?$'); + var match = regExp.exec(path); + + if (match !== null) { + // Get store + var storeName = match[1]; + var store = UploadFS.getStore(storeName); + + if (!store) { + res.writeHead(404); + res.end(); return; } - // Remove store path - var path = req.url.substr(UploadFS.config.storesPath.length + 1); + if (typeof store.onRead !== 'function') { + console.error('ufs: store "' + storeName + '" onRead is not a function'); + res.writeHead(500); + res.end(); + return; + } - // Get store, file Id and file name - var regExp = new RegExp('^\/([^\/]+)\/([^\/]+)(?:\/([^\/]+))?$'); - var match = regExp.exec(path); + // Remove file extension from file Id + var index = match[2].indexOf('.'); + var fileId = index !== -1 ? match[2].substr(0, index) : match[2]; - if (match !== null) { - // Get store - var storeName = match[1]; - var store = UploadFS.getStore(storeName); + // Get file from database + var file = store.getCollection().findOne(fileId); + if (!file) { + res.writeHead(404); + res.end(); + return; + } - if (!store) { - res.writeHead(404); - res.end(); - return; - } + // Simulate read speed + if (UploadFS.config.simulateReadDelay) { + Meteor._sleepForMs(UploadFS.config.simulateReadDelay); + } - if (typeof store.onRead !== 'function') { - console.error('ufs: store "' + storeName + '" onRead is not a function'); - res.writeHead(500); - res.end(); - return; - } + d.run(function () { + // Check if the file can be accessed + if (store.onRead.call(store, fileId, file, req, res)) { + // Open the file stream + var rs = store.getReadStream(fileId, file); + var ws = new stream.PassThrough(); - // Remove file extension from file Id - var index = match[2].indexOf('.'); - var fileId = index !== -1 ? match[2].substr(0, index) : match[2]; + rs.on('error', function (err) { + store.onReadError.call(store, err, fileId, file); + res.end(); + }); + ws.on('error', function (err) { + store.onReadError.call(store, err, fileId, file); + res.end(); + }); + ws.on('close', function () { + // Close output stream at the end + ws.emit('end'); + }); - // Get file from database - var file = store.getCollection().findOne(fileId); - if (!file) { - res.writeHead(404); + var accept = req.headers['accept-encoding'] || ''; + var headers = { + 'Content-Type': file.type, + 'Content-Length': file.size + }; + + // Transform stream + store.transformRead(rs, ws, fileId, file, req, headers); + + // Compress data using gzip + if (accept.match(/\bgzip\b/)) { + headers['Content-Encoding'] = 'gzip'; + delete headers['Content-Length']; + res.writeHead(200, headers); + ws.pipe(zlib.createGzip()).pipe(res); + } + // Compress data using deflate + else if (accept.match(/\bdeflate\b/)) { + headers['Content-Encoding'] = 'deflate'; + delete headers['Content-Length']; + res.writeHead(200, headers); + ws.pipe(zlib.createDeflate()).pipe(res); + } + // Send raw data + else { + res.writeHead(200, headers); + ws.pipe(res); + } + } else { res.end(); - return; } + }); - // Simulate read speed - if (UploadFS.config.simulateReadDelay) { - Meteor._sleepForMs(UploadFS.config.simulateReadDelay); - } - - d.run(function () { - // Check if the file can be accessed - if (store.onRead.call(store, fileId, file, req, res)) { - // Open the file stream - var rs = store.getReadStream(fileId, file); - var ws = new stream.PassThrough(); - - rs.on('error', function (err) { - store.onReadError.call(store, err, fileId, file); - res.end(); - }); - ws.on('error', function (err) { - store.onReadError.call(store, err, fileId, file); - res.end(); - }); - ws.on('close', function () { - // Close output stream at the end - ws.emit('end'); - }); - - var accept = req.headers['accept-encoding'] || ''; - var headers = { - 'Content-Type': file.type, - 'Content-Length': file.size - }; - - // Transform stream - store.transformRead(rs, ws, fileId, file, req, headers); - - // Compress data using gzip - if (accept.match(/\bgzip\b/)) { - headers['Content-Encoding'] = 'gzip'; - delete headers['Content-Length']; - res.writeHead(200, headers); - ws.pipe(zlib.createGzip()).pipe(res); - } - // Compress data using deflate - else if (accept.match(/\bdeflate\b/)) { - headers['Content-Encoding'] = 'deflate'; - delete headers['Content-Length']; - res.writeHead(200, headers); - ws.pipe(zlib.createDeflate()).pipe(res); - } - // Send raw data - else { - res.writeHead(200, headers); - ws.pipe(res); - } - } else { - res.end(); - } - }); - - } else { - next(); - } - }); -} + } else { + next(); + } +}); diff --git a/ufs.js b/ufs.js index 92b699e..2d76f15 100644 --- a/ufs.js +++ b/ufs.js @@ -28,6 +28,16 @@ UploadFS = { getStores: function () { return stores; }, + /** + * Imports a file from a URL + * @param url + * @param file + * @param store + * @param callback + */ + importFromURL: function (url, file, store, callback) { + Meteor.call('ufsImportURL', url, file, store && store.getName(), callback); + }, /** * Returns file and data as ArrayBuffer for each files in the event * @param event