Created
August 31, 2016 13:00
-
-
Save ermst4r/309865d9a76f6079d01ecdf09687dcbf to your computer and use it in GitHub Desktop.
example 2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var fileType = require('file-type'); | |
| var fs = require('fs'); | |
| var Stream = require('stream').Transform; | |
| var image_dest = sails.config.orangebuddies.settings.image_download_location; | |
| var thumb_dest = sails.config.orangebuddies.settings.image_thumbnail_location; | |
| var max_images_writes = sails.config.orangebuddies.settings.max_write_images; | |
| var product = require('../../services/mongoose_wrapper/product'); | |
| var path = require('path'); | |
| var request = require('request'); | |
| var async = require('async'); | |
| module.exports = { | |
| /** | |
| * detect http or https | |
| * @param image_url | |
| * @returns {string} | |
| */ | |
| detect_ssl : function(image_url) { | |
| var ssl = ''; | |
| if(image_url.indexOf('https') > -1) { | |
| ssl = 'https'; | |
| } else { | |
| ssl = 'http'; | |
| } | |
| return ssl; | |
| }, | |
| /** | |
| * Get the mime type of the url | |
| * @param image_url | |
| * @param chunk | |
| * @returns {*} | |
| */ | |
| get_mime_type:function(image_url,chunk) { | |
| image_url = image_url.substring(0 , image_url.indexOf('?')+1).replace("?",""); | |
| var str = String(image_url).split('.'); | |
| var extensions = ['jpg','gif','png']; | |
| if(extensions.indexOf(str[str.length-1].toLowerCase()) > -1) { | |
| return str[str.length-1]; | |
| } else { | |
| var ext = fileType(chunk); | |
| if(ext != null) { | |
| return ext.ext; | |
| } else { | |
| return "jpg"; | |
| } | |
| } | |
| }, | |
| /** | |
| * Execute the image download | |
| */ | |
| execute_download:function() { | |
| product.getFeedWithoutMainUploadedImage(max_images_writes).then(function(doc){ | |
| var page = 1, | |
| lastPage = (doc.length > max_images_writes ? max_images_writes : doc.length - 1 ); | |
| console.log("Parsing " + lastPage + " entries"); | |
| async.whilst(function () { | |
| return page <= lastPage; | |
| }, | |
| function (next) { | |
| var file_type ; | |
| var fileSavePath = image_dest; | |
| var thumbPath = thumb_dest; | |
| var fileName = ''; | |
| var data = new Stream(); | |
| /** | |
| * Check if we don't habe empty images | |
| * otherwise the app can crash | |
| */ | |
| if(typeof doc[page].image_url === 'undefined' || !doc[page].image_url ) { | |
| console.log("image url is undefined " + doc[page]._id); | |
| product.wrapper().findOneAndUpdate({_id: doc[page]._id}, {image_prepared: true,thumbnail_prepared:true}, | |
| { | |
| upsert: true, | |
| new: true, | |
| }, function (err, place) { | |
| }); | |
| page ++; | |
| next(); | |
| } else { | |
| var options = { | |
| url: doc[page].image_url, | |
| headers: { | |
| 'User-Agent': 'Mozilla/5.0' | |
| } | |
| }; | |
| request(options, function (err, response, body) { | |
| /** | |
| * We got errors, go to next.. | |
| */ | |
| if (err) { | |
| console.log("Skipped " + doc[page].image_url); | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| product.wrapper().findOneAndRemove({_id: doc[page]._id}, function (err, place) { | |
| if(err) { console.log(err);} | |
| page++; | |
| next(); | |
| }); | |
| } else { | |
| // todo fix unwanted else stamtents | |
| // byut notice, one undefined value and you app will crash | |
| // doc values can be undefined somehow.. | |
| page++; | |
| next(); | |
| } | |
| } | |
| }) | |
| .on('error', function (err) { | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| product.wrapper().findOneAndRemove({_id: doc[page]._id}, function (err, place) { | |
| }); | |
| } | |
| page++; | |
| next(); | |
| }).on('data', function (chunk) { | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| file_type = image_service.downloadimage.get_mime_type(doc[page].image_url, chunk); | |
| data.push(chunk); | |
| } | |
| }).on('end', function () { | |
| /** | |
| * And. | |
| * update the image prepare to true so we know that the main image is ready to upload | |
| */ | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| fileSavePath = image_dest + doc[page]._id + "." + file_type; | |
| thumbPath = thumb_dest + doc[page]._id + "." + file_type; | |
| fileName = doc[page]._id + "." + file_type; | |
| } | |
| /** | |
| * Write the file | |
| */ | |
| fs.writeFile(fileSavePath, data.read(), function (wErr, wSucc) { | |
| if (!wErr) { | |
| fs.stat(fileSavePath, function (serr, stats) { | |
| /** | |
| * Check for weird errors | |
| * otherwise next.. | |
| */ | |
| if (serr) { | |
| // error reading remove file | |
| fs.unlink(fileSavePath, function (eerr, ures) { | |
| page++; | |
| next(); | |
| }); | |
| } | |
| /** | |
| * If the stats are defined | |
| * write the image | |
| */ | |
| if (typeof stats !== 'undefined') { | |
| if (stats["size"] < 1000) { | |
| fs.unlink(fileSavePath, function (eerr, ures) { | |
| console.log("File smaller then 1 byte"); | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| product.wrapper().findOneAndRemove({_id: doc[page]._id}, function (err, place) { | |
| }); | |
| } | |
| page++; | |
| next(); | |
| }); | |
| } else { | |
| // make sure the id is not undefined... | |
| if (doc[page] != null || typeof doc[page] !== 'undefined') { | |
| /** | |
| * Upload the original image | |
| */ | |
| image_service.upload_to_s3.upload_images_to_s3(doc[page]._id, fileName, doc[page].feed_id, function (img_err, imag_res) { | |
| if (img_err) { console.log(img_err); page++; next(); } | |
| /** | |
| * Create the thumb | |
| */ | |
| image_service.create_thumbnails.create_thumbs(fileSavePath, thumbPath, 147, 205).then(function (thumb_created) { | |
| /** | |
| * Upload the thumbs | |
| */ | |
| image_service.upload_to_s3.upload_thumbs_to_s3(doc[page]._id, doc[page].feed_id, fileName, function (tuerr, tusucc) { | |
| product.wrapper().findOneAndUpdate({_id: doc[page]._id}, | |
| { | |
| image_url: imag_res, | |
| image_uploaded: true, | |
| thumbnail_uploaded: true, | |
| image_ready:true, | |
| image_prepared:true, | |
| thumbnail_prepared:true, | |
| thumbnail: tusucc | |
| }, | |
| { | |
| upsert: true, | |
| new: true | |
| }, function (err, place) { | |
| if (err) sails.log.error(err); | |
| console.log("image " + imag_res + " thumb: " + tusucc); | |
| // delete files | |
| fs.unlink(fileSavePath, function (err) { }); | |
| fs.unlink(thumb_dest + fileName, function (err) { }); | |
| }); | |
| /** | |
| * Delete the original image | |
| */ | |
| page++; | |
| next(); | |
| }); | |
| }, function (thumb_err) { | |
| console.log("thumb error! or 404 image " + doc[page].image_url); | |
| // Kan ook zijn dat de extensie niet klopt | |
| product.wrapper().findOneAndUpdate({_id: doc[page]._id},{ image_prepared:true, thumbnail_prepared:true,image_prepared:true}, | |
| { | |
| upsert: true, new: true | |
| }, function (err, place) { | |
| if (err) sails.log.error(err); | |
| }); | |
| fs.unlink(fileSavePath, function (err) { | |
| //if (err) sails.log.error(err); | |
| }); | |
| page++; | |
| next(); | |
| }); | |
| }); | |
| } else { | |
| page++; | |
| next(); | |
| } | |
| } | |
| } else { | |
| page++; | |
| next(); | |
| } | |
| }); | |
| } else { | |
| /** | |
| * We got errors writing, go to next.. | |
| */ | |
| page++; | |
| next(); | |
| } | |
| }); | |
| }); // end on end | |
| } | |
| }, | |
| function (err) { | |
| //Jobs.schedule('2 minutes from now','uploadimageJob'); | |
| console.log("Done"); | |
| sails.log.info(max_images_writes + " Images are downloaded!"); | |
| // All things are done! | |
| }); | |
| }); // end mongo | |
| } | |
| }; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment