Skip to content

Instantly share code, notes, and snippets.

@tadfmac
Created Sep 20, 2016
Embed
What would you like to do?
githubのissueを全部jsonで保存する。
//////////////////////////////////////////////////////////////////////////
// github issue downloader
//
// dependencies :
// - [superagent](https://github.com/visionmedia/superagent)
// - [parse-link-header](https://github.com/thlorenz/parse-link-header)
//
// CC 2016 by D.F.Mac. (H.Kodama)
//
// Usage:
// $ node getissues.js <[user-name]or[org-name]/repo-name>
//
// Version:
// v0.0 : 2016.09.20 : WIP
//
//////////////////////////////////////////////////////////////////////////
var ACCESS_TOKEN = 'your access token';
var REPO_PATH = process.argv[2];
var USER_AGENT = 'your user-agent'; // i.e. tadfmac
// dependencies
var parseLH = require('parse-link-header');
var request = require('superagent');
var fs = require('fs');
// querryIssueList : issue Listを全件取得し、queryObjArr[]に保存する
var queryObjArr = [];
function queryIssueList(page){
var url = 'https://api.github.com/repos/'+REPO_PATH+'/issues?access_token='+ACCESS_TOKEN+"&filter=all&state=all&page="+page;
var req = request.get(url);
req.set('user-agent', USER_AGENT);
req.end(function(err,res){
for(var cnt = 0; cnt < res.body.length; cnt ++){
queryObjArr.push(res.body[cnt]);
}
var linkheader = parseLH(res.header.link);
if(linkheader.next != null){
// console.log("next:"+linkheader.next.page);
queryIssueList(linkheader.next.page);
}else{
console.log("query record is:"+queryObjArr.length);
queryComment();
}
});
}
var workrec = 0;
var compage = 1;
function queryComment(){
if(workrec < queryObjArr.length){
if(queryObjArr[workrec].comments > 0){
if(queryObjArr[workrec].commentsArr == null){
queryObjArr[workrec].commentsArr = new Array();
}
}else{
workrec ++;
compage = 1;
queryComment();
return;
}
var issueNum = queryObjArr[workrec].number;
console.log("queryComment:issueNum"+issueNum+" rec:"+workrec);
var url = 'https://api.github.com/repos/'+REPO_PATH+'/issues/'+issueNum+'/comments?access_token='+ACCESS_TOKEN+"&filter=all&state=all&page="+compage;
var req = request.get(url);
req.set('user-agent', USER_AGENT);
req.end(function(err,res){
for(var cnt = 0; cnt < res.body.length; cnt ++){
queryObjArr[workrec].commentsArr.push(res.body[cnt]);
}
var linkheader = parseLH(res.header.link);
if(linkheader != null){
compage = linkheader.next.page;
queryComment();
}else{
workrec ++;
if(workrec < queryObjArr.length){
queryComment();
}else{
console.log("queryComment:complete"+workrec);
saveJsonFile();
}
}
});
}
}
queryIssueList(1); // issue全件取得
// Listを一括保存
function saveJsonFile(){
// 一括ファイル保存
var jsonText = JSON.stringify(queryObjArr, '', ' ');
fs.writeFile('issueList.json', jsonText);
// バラバラのファイルも作る
for(var cnt=0;cnt < queryObjArr.length; cnt++){
var num = queryObjArr[cnt].number;
var numJsonText = JSON.stringify(queryObjArr[cnt], '', ' ');
fs.writeFile(num+'.json', numJsonText);
if(queryObjArr[cnt].comments > 0){
saveCommentJsonFile(cnt,num);
}
}
}
// Commentを一括保存
function saveCommentJsonFile(rec,num){
var jsonText = JSON.stringify(queryObjArr[rec].commentsArr, '', ' ');
fs.writeFile(num+'-comment.json', jsonText);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment