Skip to content

Instantly share code, notes, and snippets.

@TheCalorious
Created February 13, 2016 23:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save TheCalorious/5b9ca92f8f52005f3f67 to your computer and use it in GitHub Desktop.
Save TheCalorious/5b9ca92f8f52005f3f67 to your computer and use it in GitHub Desktop.
A node app to retreive images from parse
var Parse = require('parse/node');
var fs = require('fs');
var request = require('request');
var moment = require('moment');
Parse.initialize("<app-id>", "<secret-key>");
/***********************************************
* The idea is to download all images, 100 at
* a time. we'll write a recursive function to
* do that. the files will be downloaded in the
* ./images/ directory.
************************************************/
//keep a count of downloaded images
var count = 0;
var download = function(items){
//if we downloaded all images in the current
//list of items, we need to fetch more (next 100).
if(count == items.length){
//end case - if # of items parse fetched
//is less than 100 we know we've reached
//the end of data.
if(items.length < 100){
return;
}
//reset the count
count = 0;
console.log("got all 100");
//last processed item
var lastItem = items[items.length-1];
var newBeforeDate = moment(lastItem.createdAt)
//lets get the next 100 starting from the created date
//of the last object fetched
getItems(newBeforeDate.toDate());
}else{
//if we haven't yet downloaded all images in the
//current list of fetched items, download the next one
var item = items[count];
var filename = item.get("image")._name;
var uri = item.get("image")._url;
request.head(uri, function(err, res, body){
if (err){
console.log(err);
console.log(item);
return;
}else {
var stream = request(uri);
stream.pipe(
fs.createWriteStream("images/"+filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
).on('close', function() {
//call this function again
count++;
download(items);
});
}
});
}
};
function getItems(beforeDate){
var query = new Parse.Query("Item");
query.notEqualTo("deleted", true);
query.descending("createdAt");
query.lessThan("createdAt", beforeDate);
query.find({
success: function(item){
//we will get 100 at a time
console.log("fetched more items...")
download(items);
}
});
}
getItems(new Date());
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment