Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@shivkumarganesh
Created September 12, 2013 13:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save shivkumarganesh/6537779 to your computer and use it in GitHub Desktop.
Save shivkumarganesh/6537779 to your computer and use it in GitHub Desktop.
This Program helps you to download all the StackExchangeTags from the StackExchange Website. Kinda jumbled up to look but works well. You just need to get the API key and add it with a valid MongoDB url.
npm install -g superagent &&
npm install -g mongoose &&
echo Please include these in your Project directory so as to be referred by StackExchangeTags.js
var siteList={
"items":
[ { "api_site_parameter": "stackoverflow" },
{ "api_site_parameter": "serverfault" },
{ "api_site_parameter": "superuser" },
{ "api_site_parameter": "meta.stackoverflow" },
{ "api_site_parameter": "webapps" },
{ "api_site_parameter": "meta.webapps" },
{ "api_site_parameter": "gaming" },
{ "api_site_parameter": "meta.gaming" },
{ "api_site_parameter": "webmasters" },
{ "api_site_parameter": "meta.webmasters" },
{ "api_site_parameter": "cooking" },
{ "api_site_parameter": "meta.cooking" },
{ "api_site_parameter": "gamedev" },
{ "api_site_parameter": "meta.gamedev" },
{ "api_site_parameter": "photo" },
{ "api_site_parameter": "meta.photo" },
{ "api_site_parameter": "stats" },
{ "api_site_parameter": "meta.stats" },
{ "api_site_parameter": "math" },
{ "api_site_parameter": "meta.math" },
{ "api_site_parameter": "diy" },
{ "api_site_parameter": "meta.diy" },
{ "api_site_parameter": "meta.superuser" },
{ "api_site_parameter": "meta.serverfault" },
{ "api_site_parameter": "gis" },
{ "api_site_parameter": "meta.gis" },
{ "api_site_parameter": "tex" },
{ "api_site_parameter": "meta.tex" },
{ "api_site_parameter": "askubuntu" },
{ "api_site_parameter": "meta.askubuntu" },
{ "api_site_parameter": "money" },
{ "api_site_parameter": "meta.money" },
{ "api_site_parameter": "english" },
{ "api_site_parameter": "meta.english" },
{ "api_site_parameter": "stackapps" },
{ "api_site_parameter": "ux" },
{ "api_site_parameter": "meta.ux" },
{ "api_site_parameter": "unix" },
{ "api_site_parameter": "meta.unix" },
{ "api_site_parameter": "wordpress" },
{ "api_site_parameter": "meta.wordpress" },
{ "api_site_parameter": "cstheory" },
{ "api_site_parameter": "meta.cstheory" },
{ "api_site_parameter": "apple" },
{ "api_site_parameter": "meta.apple" },
{ "api_site_parameter": "rpg" },
{ "api_site_parameter": "meta.rpg" },
{ "api_site_parameter": "bicycles" },
{ "api_site_parameter": "meta.bicycles" },
{ "api_site_parameter": "programmers" },
{ "api_site_parameter": "meta.programmers" },
{ "api_site_parameter": "electronics" },
{ "api_site_parameter": "meta.electronics" },
{ "api_site_parameter": "android" },
{ "api_site_parameter": "meta.android" },
{ "api_site_parameter": "answers.onstartups" },
{ "api_site_parameter": "meta.answers.onstartups" },
{ "api_site_parameter": "boardgames" },
{ "api_site_parameter": "meta.boardgames" },
{ "api_site_parameter": "physics" },
{ "api_site_parameter": "meta.physics" },
{ "api_site_parameter": "homebrew" },
{ "api_site_parameter": "meta.homebrew" },
{ "api_site_parameter": "security" },
{ "api_site_parameter": "meta.security" },
{ "api_site_parameter": "writers" },
{ "api_site_parameter": "meta.writers" },
{ "api_site_parameter": "avp" },
{ "api_site_parameter": "meta.avp" },
{ "api_site_parameter": "graphicdesign" },
{ "api_site_parameter": "meta.graphicdesign" },
{ "api_site_parameter": "dba" },
{ "api_site_parameter": "meta.dba" },
{ "api_site_parameter": "scifi" },
{ "api_site_parameter": "meta.scifi" },
{ "api_site_parameter": "codereview" },
{ "api_site_parameter": "meta.codereview" },
{ "api_site_parameter": "codegolf" },
{ "api_site_parameter": "meta.codegolf" },
{ "api_site_parameter": "quant" },
{ "api_site_parameter": "meta.quant" },
{ "api_site_parameter": "pm" },
{ "api_site_parameter": "meta.pm" },
{ "api_site_parameter": "skeptics" },
{ "api_site_parameter": "meta.skeptics" },
{ "api_site_parameter": "fitness" },
{ "api_site_parameter": "meta.fitness" },
{ "api_site_parameter": "drupal" },
{ "api_site_parameter": "meta.drupal" },
{ "api_site_parameter": "mechanics" },
{ "api_site_parameter": "meta.mechanics" },
{ "api_site_parameter": "parenting" },
{ "api_site_parameter": "meta.parenting" },
{ "api_site_parameter": "sharepoint" },
{ "api_site_parameter": "meta.sharepoint" },
{ "api_site_parameter": "music" },
{ "api_site_parameter": "meta.music" },
{ "api_site_parameter": "sqa" },
{ "api_site_parameter": "meta.sqa" },
{ "api_site_parameter": "judaism" } ,
{ "api_site_parameter": "meta.freelancing" },
{ "api_site_parameter": "blender" },
{ "api_site_parameter": "meta.blender" },
{ "api_site_parameter": "mathoverflow.net" },
{ "api_site_parameter": "meta.mathoverflow.net" },
{ "api_site_parameter": "space" },
{ "api_site_parameter": "meta.space" },
{ "api_site_parameter": "digitalfabrication" },
{ "api_site_parameter": "meta.digitalfabrication" },
{ "api_site_parameter": "sound" },
{ "api_site_parameter": "meta.sound" },
{ "api_site_parameter": "meta.judaism" },
{ "api_site_parameter": "german" },
{ "api_site_parameter": "meta.german" },
{ "api_site_parameter": "japanese" },
{ "api_site_parameter": "meta.japanese" },
{ "api_site_parameter": "philosophy" },
{ "api_site_parameter": "meta.philosophy" },
{ "api_site_parameter": "gardening" },
{ "api_site_parameter": "meta.gardening" },
{ "api_site_parameter": "travel" },
{ "api_site_parameter": "meta.travel" },
{ "api_site_parameter": "productivity" },
{ "api_site_parameter": "meta.productivity" },
{ "api_site_parameter": "crypto" },
{ "api_site_parameter": "meta.crypto" },
{ "api_site_parameter": "dsp" },
{ "api_site_parameter": "meta.dsp" },
{ "api_site_parameter": "french" },
{ "api_site_parameter": "meta.french" },
{ "api_site_parameter": "christianity" },
{ "api_site_parameter": "meta.christianity" },
{ "api_site_parameter": "bitcoin" },
{ "api_site_parameter": "meta.bitcoin" },
{ "api_site_parameter": "linguistics" },
{ "api_site_parameter": "meta.linguistics" },
{ "api_site_parameter": "hermeneutics" },
{ "api_site_parameter": "meta.hermeneutics" },
{ "api_site_parameter": "history" },
{ "api_site_parameter": "meta.history" },
{ "api_site_parameter": "bricks" },
{ "api_site_parameter": "meta.bricks" },
{ "api_site_parameter": "spanish" },
{ "api_site_parameter": "meta.spanish" },
{ "api_site_parameter": "scicomp" },
{ "api_site_parameter": "meta.scicomp" },
{ "api_site_parameter": "movies" },
{ "api_site_parameter": "meta.movies" },
{ "api_site_parameter": "chinese" },
{ "api_site_parameter": "meta.chinese" },
{ "api_site_parameter": "biology" },
{ "api_site_parameter": "meta.biology" },
{ "api_site_parameter": "poker" },
{ "api_site_parameter": "meta.poker" },
{ "api_site_parameter": "mathematica" },
{ "api_site_parameter": "meta.mathematica" },
{ "api_site_parameter": "cogsci" },
{ "api_site_parameter": "meta.cogsci" },
{ "api_site_parameter": "outdoors" },
{ "api_site_parameter": "meta.outdoors" },
{ "api_site_parameter": "martialarts" },
{ "api_site_parameter": "meta.martialarts" },
{ "api_site_parameter": "sports" },
{ "api_site_parameter": "meta.sports" },
{ "api_site_parameter": "academia" },
{ "api_site_parameter": "meta.academia" },
{ "api_site_parameter": "cs" },
{ "api_site_parameter": "meta.cs" },
{ "api_site_parameter": "workplace" },
{ "api_site_parameter": "meta.workplace" },
{ "api_site_parameter": "windowsphone" },
{ "api_site_parameter": "meta.windowsphone" },
{ "api_site_parameter": "chemistry" },
{ "api_site_parameter": "meta.chemistry" },
{ "api_site_parameter": "chess" },
{ "api_site_parameter": "meta.chess" },
{ "api_site_parameter": "raspberrypi" },
{ "api_site_parameter": "meta.raspberrypi" },
{ "api_site_parameter": "russian" },
{ "api_site_parameter": "meta.russian" },
{ "api_site_parameter": "islam" },
{ "api_site_parameter": "meta.islam" },
{ "api_site_parameter": "salesforce" },
{ "api_site_parameter": "meta.salesforce" },
{ "api_site_parameter": "patents" },
{ "api_site_parameter": "meta.patents" },
{ "api_site_parameter": "genealogy" },
{ "api_site_parameter": "meta.genealogy" },
{ "api_site_parameter": "robotics" },
{ "api_site_parameter": "meta.robotics" },
{ "api_site_parameter": "expressionengine" },
{ "api_site_parameter": "meta.expressionengine" },
{ "api_site_parameter": "politics" },
{ "api_site_parameter": "meta.politics" },
{ "api_site_parameter": "anime" },
{ "api_site_parameter": "meta.anime" },
{ "api_site_parameter": "magento" },
{ "api_site_parameter": "meta.magento" },
{ "api_site_parameter": "ell" },
{ "api_site_parameter": "meta.ell" },
{ "api_site_parameter": "sustainability" },
{ "api_site_parameter": "meta.sustainability" },
{ "api_site_parameter": "tridion" },
{ "api_site_parameter": "meta.tridion" },
{ "api_site_parameter": "reverseengineering" },
{ "api_site_parameter": "meta.reverseengineering" },
{ "api_site_parameter": "networkengineering" },
{ "api_site_parameter": "meta.networkengineering" },
{ "api_site_parameter": "opendata" },
{ "api_site_parameter": "meta.opendata" },
{ "api_site_parameter": "freelancing" } ]}
console.log(siteList.items.length);
var request=require('./superagent');
var mongoose = require('./mongoose');
mongoose.connect('mongodb://localhost/StackExchangeTags');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback () {
console.log('Connected');
});
//Schema for Tags
var tagSchema = mongoose.Schema({
name:String,
count:Number,
is_required:Boolean,
is_moderator_only:Boolean,
has_synonyms:Boolean});
var saveSchema = function(Model){
Model.save();
};
var time=0;
var count=0;
var i=1;
var req=function(){
var Tags = mongoose.model(siteList.items[count].api_site_parameter, tagSchema);
request.get('http://api.stackexchange.com/2.1/tags?page='+i+'&pagesize=100&fromdate=1199145600&todate=1377648000&order=asc&sort=popular&site='+siteList.items[count].api_site_parameter+'&key=VA6c0LmO6lmPdYLk3KLp6A((')
.set('Accept', 'application/json')
.end(function(res){
if (res.ok && res.body.items.length!==0) {
console.log('yay got ' + res.body);
var jsonResponse = res.body;
console.log(jsonResponse.items.length);
for(var j=0;j<jsonResponse.items.length;j++)
{
var model = new Tags(jsonResponse.items[j]);
saveSchema(model);
}
i++;
console.log("Number of Iteration ---- > "+i);
if(jsonResponse.backoff !== null && jsonResponse.backoff !== "" && jsonResponse.backoff !== undefined){
time=time+jsonResponse.backoff;
setTimeout(function() {
console.log('BACKOFF IS SET');
req();
}, time);
}
else{
setTimeout(function() {
console.log('BACKOFF IS NOT SET----- :)');
req();
}, time);
}
} else {
count++;
i=1;
console.log('One Site Complete');
setTimeout(function() {
console.log('BACKOFF IS SET');
req();
}, 2000);
}
});
}
req();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment