Skip to content

Instantly share code, notes, and snippets.

@jazahn
Created May 2, 2014 16:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jazahn/9b725b0cd71a6fb0e435 to your computer and use it in GitHub Desktop.
Save jazahn/9b725b0cd71a6fb0e435 to your computer and use it in GitHub Desktop.
youtube data collection
var https = require("https");
var fs = require("fs");
var stream = fs.createWriteStream("youtube.csv");
var nameArray = ["Harvard", "HarvardBSchool", "HarvardKennedySchool", "HarvardEducation", "HarvardLawSchool", "HarvardMedicalSchool", "HarvardPublicHealth", "TheHarvardGSD", "HarvardDivinity", "BerkmanCenter", "AshInstitute", "HarvardBusiness", "Harvardilab", "HarvardExtension", "HarvardCPL", "ArnoldArboretum", "GreenIsTheNewCrimson", "BelferCenter"];
var start = 1;
var url = function(name, start){
return "https://gdata.youtube.com/feeds/api/users/"+name+"/uploads?alt=json&start-index="+start+"&max-results=50&caption=false";
}
var output = '';
var dodata = function(thisName, thisStart){
https.get(url(thisName, thisStart), function(response){
var body = '';
response.on("data", function(chunk) {
body += chunk;
});
response.on("end", function(){
//console.log(body);
var output = '';
var result = JSON.parse(body);
if(result.feed.entry){
for(var i = 0; i < result.feed.entry.length; i++){
if(result.feed.entry[i]){
var title = result.feed.entry[i].title.$t;
//console.log(result.feed.entry[i]);
var duration = result.feed.entry[i]['media$group']['yt$duration'].seconds;
var published = result.feed.entry[i].published.$t;
output += thisName + "\t" + title + "\t" + duration + "\t" + published + "\n";
}
}
stream.write(output);
if(result.feed.entry.length == 50){
console.log("next 50: "+ parseInt(thisStart + 50));
dodata(thisName, thisStart + 50);
} else if(nameArray.length > 0){
name = nameArray.pop();
console.log("popping next ("+name+")")
dodata(name, 1);
} else {
stream.end();
}
} else {
if(nameArray.length > 0){
name = nameArray.pop();
console.log("popping next ("+name+")")
dodata(name, 1);
} else {
stream.end();
}
}
});
});
}
var name = nameArray.pop();
console.log(name);
dodata(nameArray.pop(), start);
console.log(output);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment