Created
February 13, 2016 23:52
-
-
Save TheCalorious/5b9ca92f8f52005f3f67 to your computer and use it in GitHub Desktop.
A node app to retreive images from parse
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var Parse = require('parse/node'); | |
var fs = require('fs'); | |
var request = require('request'); | |
var moment = require('moment'); | |
Parse.initialize("<app-id>", "<secret-key>"); | |
/*********************************************** | |
* The idea is to download all images, 100 at | |
* a time. we'll write a recursive function to | |
* do that. the files will be downloaded in the | |
* ./images/ directory. | |
************************************************/ | |
//keep a count of downloaded images | |
var count = 0; | |
var download = function(items){ | |
//if we downloaded all images in the current | |
//list of items, we need to fetch more (next 100). | |
if(count == items.length){ | |
//end case - if # of items parse fetched | |
//is less than 100 we know we've reached | |
//the end of data. | |
if(items.length < 100){ | |
return; | |
} | |
//reset the count | |
count = 0; | |
console.log("got all 100"); | |
//last processed item | |
var lastItem = items[items.length-1]; | |
var newBeforeDate = moment(lastItem.createdAt) | |
//lets get the next 100 starting from the created date | |
//of the last object fetched | |
getItems(newBeforeDate.toDate()); | |
}else{ | |
//if we haven't yet downloaded all images in the | |
//current list of fetched items, download the next one | |
var item = items[count]; | |
var filename = item.get("image")._name; | |
var uri = item.get("image")._url; | |
request.head(uri, function(err, res, body){ | |
if (err){ | |
console.log(err); | |
console.log(item); | |
return; | |
}else { | |
var stream = request(uri); | |
stream.pipe( | |
fs.createWriteStream("images/"+filename) | |
.on('error', function(err){ | |
callback(error, filename); | |
stream.read(); | |
}) | |
).on('close', function() { | |
//call this function again | |
count++; | |
download(items); | |
}); | |
} | |
}); | |
} | |
}; | |
function getItems(beforeDate){ | |
var query = new Parse.Query("Item"); | |
query.notEqualTo("deleted", true); | |
query.descending("createdAt"); | |
query.lessThan("createdAt", beforeDate); | |
query.find({ | |
success: function(item){ | |
//we will get 100 at a time | |
console.log("fetched more items...") | |
download(items); | |
} | |
}); | |
} | |
getItems(new Date()); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment