Backup GitHub issues

gistfile1.js

"use strict";

var fs = require("fs"),
    https = require("https"),
    url = require("url"),
    zlib = require("zlib");

/**
 * Process HTTP responses. Handle compressed streams and convert to objects as
 * appropriate.
 *
 * @param {Function} callback Function to pass processed data to.
 */
var processResponse = function(callback) {
    return function(response) {
        // response is a stream, but we may pipe the stream and still need
        // access to response properties
        var stream = response;

        switch (response.headers['content-encoding']) {
        case 'gzip':
            stream = response.pipe(zlib.createGunzip());
            break;

        case 'deflate':
            stream = response.pipe(zlib.createInflate());
            break;
        }
        
        var data = '';
        stream.on('data', function(chunk) {
            data += chunk;
        });

        stream.on('end', function() {
            // we only care about the first part of the content-type string
            if (response.headers['content-type'].split(";")[0] === 'application/json') {
                data = JSON.parse(data);
            }
            callback(data, response);
        });
    };
};

var issuesUrl = "https://api.github.com/repos/USERNAME/REPO/issues";

var parseLinkHeader = function(linkHeader) {
    var links = {};

    linkHeader.split(",").forEach(function(link) {
        var parts = link.split(";");
        var rel = parts[1].split("=")[1].replace(/"/g, "").trim();

        links[rel] = parts[0].replace(/[<>]/g, "").trim();
    });

    return links;
};

var fetch = function(endpoint, callback) {
    var options = url.parse(endpoint);
    options.auth = "mojodna:REDACTED";
    options.headers = {"Accept-Encoding": "gzip,deflate"};

    https.get(options).on("response", processResponse(callback));
};

var processIssues = function(issues, res) {
    issues.forEach(function(issue) {
        if (issue.comments > 0) {
            fetch(issue.url + "/comments", function(comments) {
                // TODO this doesn't paginate comments (ok for now)
                console.log("%d %s (%d)", issue.number, issue.title, comments.length);
                issue.comments = comments;

                fs.writeFile("issue-" + issue.number + ".json", JSON.stringify(issue, null, 2));
            });
        } else {
            console.log("%d %s", issue.number, issue.title);
            issue.comments = null;
            fs.writeFile("issue-" + issue.number + ".json", JSON.stringify(issue, null, 2));
        }
    });

    var links = parseLinkHeader(res.headers.link);

    if (links.next) {
        fetch(links.next, processIssues);
    }
};

fetch(issuesUrl, processIssues);