github project fetcher

dget.d

import std.algorithm, std.exception, std.file, std.json, std.path, std.range, std.zip, std.net.curl;
static import std.stdio;

void main(string[] args)
{
    bool isSnapshot;
    foreach(arg; args[1 .. $])
    {
        auto outdir = arg.replace("/", "_");
        enforce(!outdir.exists, fmt("output folder '%s' already exists", outdir));
        findRepo(arg).findFile(isSnapshot).download()
            .unzip(outdir, isSnapshot);
    }
}

string findRepo(string arg)
{
    import std.regex;
    enum rule = ctRegex!(r"^(?:([^/]*)/)?([^/]*)$");

    auto m = match(arg, rule);
    enforce(!m.empty, fmt("expected 'org/repo' but found '%s'", arg));

    string result = m.captures[1];
    if (!result.empty)
        result ~= "/" ~ m.captures[2];
    else
    {
        auto repo = m.captures[2];
        auto org = find!(o => o.hasRepo(repo))(defaultOrgs);
        enforce(!org.empty, fmt("repo '%s' was not found among '%(%s, %)'",
                                repo, defaultOrgs));

        result = org.front ~ "/" ~ repo;
    }
    std.stdio.writefln("repo: %s", result);
    return result;
}

string findFile(string orgRepo, out bool isSnapshot)
{
    string result;

    auto downloads = fmt("https://api.github.com/repos/%s/downloads", orgRepo)
        .reqJSON().array;
    if (!downloads.empty)
    {
        auto latest = downloads.front.object;
        enforce(latest["content_type"].str == ".zip",
                fmt("can't handle content type '%s' of download '%s'",
                    latest["content_type"].str, latest["html_url"].str));
        result = latest["html_url"].str;
        isSnapshot = false;
        std.stdio.writefln("downloading: '%s'", latest["name"].str);
    }
    else
    {
        result = fmt("https://github.com/%s/zipball/master", orgRepo);
        auto http = HTTP(result);
        http.method = HTTP.Method.head;
        http.perform();
        enforce(http.statusLine().code / 100 == 2,
                fmt("found no download for repo '%s'", orgRepo));
        isSnapshot = true;
        std.stdio.writefln("downloading: 'master zipball'");
    }
    return result;
}

bool hasRepo(string org, string repo)
{
    return fmt("https://api.github.com/orgs/%s/repos?type=public", org)
        .reqJSON().array.canFind!(a => a.object["name"].str == repo)();
}

@property const(string[]) defaultOrgs()
{
    static immutable value = ["D-Programming-Deimos", "D-Programming-Language"];
    // TODO: pick up env var
    return value;
}

JSONValue reqJSON(string url)
{
    return get(url).parseJSON();
}

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

ubyte[] download(string url)
{
    // doesn't work because it already timeouts after 2 minutes
    // return get!(HTTP, ubyte)();

    import core.time, std.array, std.conv;

    auto buf = appender!(ubyte[])();
    size_t contentLength;

    auto http = HTTP(url);
    http.method = HTTP.Method.get;
    http.onReceiveHeader((k, v)
    {
        if (k == "content-length")
            contentLength = to!size_t(v);
    });
    http.onReceive((data)
    {
        buf.put(data);
        std.stdio.writef("%sk/%sk\r", buf.data.length/1024,
                         contentLength ? to!string(contentLength/1024) : "?");
        std.stdio.stdout.flush();
        return data.length;
    });
    http.dataTimeout = dur!"msecs"(0);
    http.perform();
    immutable sc = http.statusLine().code;
    enforce(sc / 100 == 2 || sc == 302,
            fmt("HTTP request returned status code %s", sc));
    std.stdio.writeln("done                    ");
    return buf.data;
}

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

void unzip(ubyte[] data, string outdir, bool isSnapshot)
{
    import std.string;

    scope archive = new ZipArchive(data);
    std.stdio.writeln("unpacking:");
    string prefix;
    mkdir(outdir);
    if (isSnapshot)
    {
        foreach(name, _; archive.directory)
        {
            prefix = name[0 .. $ - name.find("/").length + 1];
            break;
        }
    }
    foreach(name, am; archive.directory)
    {
        if (!am.expandedSize) continue;

        string path = buildPath(outdir, chompPrefix(name, prefix));
        std.stdio.writeln(path);
        auto dir = dirName(path);
        if (!dir.empty && !dir.exists)
            mkdirRecurse(dir);
        archive.expand(am);
        write(path, am.expandedData);
    }
}

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

string fmt(Args...)(string fmt, auto ref Args args)
{
    import std.array, std.format;
    auto app = appender!string();
    formattedWrite(app, fmt, args);
    return app.data;
}