nodejs - recursively download github api-documentation

webCrawl.js

/*
 * webCrawl.js
 * this program will recursively "wget" api-documentation
 * from https://developer.github.com/v3/index.html to 2-level-depth
 *
 * example usage:
 * $ node webCrawl.js eager # eager mkdirSync before writeFile
 * $ node webCrawl.js lazy  # lazy  mkdirSync after  writeFile failure
 *
 * example output:
eager mode
10353000 total bytes
124 total http.request
121 total fs.fileWriteSync
121 total fs.mkdirSync
6608ms elapsed

lazy mode
10353000 total bytes
128 total http.request
121 total fs.fileWriteSync
17 total fs.mkdirSync
5541ms elapsed
 */
/* jslint utility2:true */
(async function () {
    "use strict";
    let webCrawl;
    let debugInline;
    debugInline = function (...argList) {
    /*
     * this function will both print <argList> to stderr
     * and return <argList>[0]
     */
        console.error("\n\ndebugInline");
        console.error(...argList);
        console.error("\n");
        return argList[0];
    };
    debugInline(debugInline);
    webCrawl = function (url, depthMax, mode) {
    /*
     * this function will recursively web-crawl <url>
     */
        let chunkList;
        let circularSet;
        let cntByteLength;
        let cntFsMkdir;
        let cntFsWrite;
        let cntHttpRequest;
        let cntPromise;
        let fs;
        let fsWriteFileWithMkdirpSync;
        let path;
        let prefixDir;
        let prefixUrl;
        let promise;
        let recurse;
        let res;
        let resolve;
        let resolveAll;
        let timeStart;
        // init function
        fsWriteFileWithMkdirpSync = function (file, data) {
        /*
         * this function will sync write <data> to <file> with "mkdir -p"
         */
            //!! console.error("writing " + file);
            cntFsWrite = (cntFsWrite | 0) + 1;
            if (mode === "eager") {
                // mkdir -p
                cntFsMkdir = (cntFsMkdir | 0) + 1;
                fs.mkdirSync(path.dirname(file), {
                    recursive: true
                });
            }
            // try to write file
            try {
                fs.writeFileSync(file, data);
                return true;
            } catch (ignore) {
                // mkdir -p
                cntFsMkdir = (cntFsMkdir | 0) + 1;
                fs.mkdirSync(path.dirname(file), {
                    recursive: true
                });
                // rewrite file
                fs.writeFileSync(file, data);
                return true;
            }
        };
        recurse = async function (url, depth) {
            let data;
            cntPromise += 1;
            if (depthMax && depthMax < depth) {
                resolve();
                return;
            }
            if (!Array.from([
                prefixDir,
                prefixUrl
            ]).some(function (prefix) {
                if (url.indexOf(prefix) !== 0) {
                    return;
                }
                url = "/" + url.slice(prefix.length) + "/index.html";
                url = url.replace((
                    /\/{2,}/g
                ), "/");
                url = url.replace((
                    /\/index.html\/index.html$/
                ), "/index.html");
                url = url.slice(1);
                return true;
            })) {
                resolve();
                return;
            }
            if (!url || circularSet.has(url)) {
                resolve();
                return;
            }
            circularSet.add(url);
            url = prefixUrl + url;
            //!! console.error("fetching " + url);
            cntHttpRequest = (cntHttpRequest | 0) + 1;
            res = await new Promise(function (resolve) {
                require(
                    url.slice(0, 5) === "https"
                    ? "https"
                    : "http"
                ).request(url, resolve).end();
            });
            if (res.statusCode !== 200) {
                resolve();
                return;
            }
            chunkList = [];
            res.on("data", function (chunk) {
                cntByteLength = (cntByteLength | 0) + chunk.byteLength;
                chunkList.push(chunk);
            });
            data = await new Promise(function (resolve) {
                res.on("end", function () {
                    resolve(Buffer.concat(chunkList).toString());
                });
            });
            fsWriteFileWithMkdirpSync((
                prefixDir.slice(1) + url.slice(prefixUrl.length).replace(
                    "/index.html",
                    ".html"
                )
            ), data);
            // recurse
            data.replace((
                /\bhref="(.*?)["#]/g
            ), function (ignore, match1) {
                recurse(match1, depth + 1);
                return "";
            });
            resolve();
        };
        resolve = function () {
        /*
         * this function will resolve when <cntPromise> === 0
         */
            cntPromise -= 1;
            if (cntPromise !== 0) {
                return;
            }
            console.error();
            console.error(mode + " mode");
            console.error(cntByteLength.toLocaleString() + " total bytes");
            console.error(cntHttpRequest + " total http.request");
            console.error(cntFsWrite + " total fs.fileWriteSync");
            console.error(cntFsMkdir + " total fs.mkdirSync");
            console.error(Date.now() - timeStart + "ms elapsed");
            console.error();
            resolveAll();
        };
        // require module
        fs = require("fs");
        path = require("path");
        // init variable
        circularSet = new Set();
        prefixDir = path.dirname(
            require("url").parse(url).pathname
        ) + "/";
        prefixUrl = url.split("/").slice(0, 3).join("/") + prefixDir;
        promise = new Promise(function (resolve) {
            resolveAll = resolve;
        });
        cntPromise = 0;
        // cleanup download-dir
        require("child_process").spawnSync("rm", [
            "-rf", prefixDir.slice(1) || "undefined"
        ]);
        timeStart = Date.now();
        recurse(url, 0);
        return promise;
    };
    await webCrawl(
        "https://developer.github.com/v3/index.html",
        2,
        process.argv[2]
    );
}());