public
Created

github project fetcher

  • Download Gist
dget.d
D
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
import std.algorithm, std.exception, std.file, std.json, std.path, std.range, std.zip, std.net.curl;
static import std.stdio;
 
void main(string[] args)
{
bool isSnapshot;
foreach(arg; args[1 .. $])
{
auto outdir = arg.replace("/", "_");
enforce(!outdir.exists, fmt("output folder '%s' already exists", outdir));
findRepo(arg).findFile(isSnapshot).download()
.unzip(outdir, isSnapshot);
}
}
 
string findRepo(string arg)
{
import std.regex;
enum rule = ctRegex!(r"^(?:([^/]*)/)?([^/]*)$");
 
auto m = match(arg, rule);
enforce(!m.empty, fmt("expected 'org/repo' but found '%s'", arg));
 
string result = m.captures[1];
if (!result.empty)
result ~= "/" ~ m.captures[2];
else
{
auto repo = m.captures[2];
auto org = find!(o => o.hasRepo(repo))(defaultOrgs);
enforce(!org.empty, fmt("repo '%s' was not found among '%(%s, %)'",
repo, defaultOrgs));
 
result = org.front ~ "/" ~ repo;
}
std.stdio.writefln("repo: %s", result);
return result;
}
 
string findFile(string orgRepo, out bool isSnapshot)
{
string result;
 
auto downloads = fmt("https://api.github.com/repos/%s/downloads", orgRepo)
.reqJSON().array;
if (!downloads.empty)
{
auto latest = downloads.front.object;
enforce(latest["content_type"].str == ".zip",
fmt("can't handle content type '%s' of download '%s'",
latest["content_type"].str, latest["html_url"].str));
result = latest["html_url"].str;
isSnapshot = false;
std.stdio.writefln("downloading: '%s'", latest["name"].str);
}
else
{
result = fmt("https://github.com/%s/zipball/master", orgRepo);
auto http = HTTP(result);
http.method = HTTP.Method.head;
http.perform();
enforce(http.statusLine().code / 100 == 2,
fmt("found no download for repo '%s'", orgRepo));
isSnapshot = true;
std.stdio.writefln("downloading: 'master zipball'");
}
return result;
}
 
bool hasRepo(string org, string repo)
{
return fmt("https://api.github.com/orgs/%s/repos?type=public", org)
.reqJSON().array.canFind!(a => a.object["name"].str == repo)();
}
 
@property const(string[]) defaultOrgs()
{
static immutable value = ["D-Programming-Deimos", "D-Programming-Language"];
// TODO: pick up env var
return value;
}
 
JSONValue reqJSON(string url)
{
return get(url).parseJSON();
}
 
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
ubyte[] download(string url)
{
// doesn't work because it already timeouts after 2 minutes
// return get!(HTTP, ubyte)();
 
import core.time, std.array, std.conv;
 
auto buf = appender!(ubyte[])();
size_t contentLength;
 
auto http = HTTP(url);
http.method = HTTP.Method.get;
http.onReceiveHeader((k, v)
{
if (k == "content-length")
contentLength = to!size_t(v);
});
http.onReceive((data)
{
buf.put(data);
std.stdio.writef("%sk/%sk\r", buf.data.length/1024,
contentLength ? to!string(contentLength/1024) : "?");
std.stdio.stdout.flush();
return data.length;
});
http.dataTimeout = dur!"msecs"(0);
http.perform();
immutable sc = http.statusLine().code;
enforce(sc / 100 == 2 || sc == 302,
fmt("HTTP request returned status code %s", sc));
std.stdio.writeln("done ");
return buf.data;
}
 
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
void unzip(ubyte[] data, string outdir, bool isSnapshot)
{
import std.string;
 
scope archive = new ZipArchive(data);
std.stdio.writeln("unpacking:");
string prefix;
mkdir(outdir);
if (isSnapshot)
{
foreach(name, _; archive.directory)
{
prefix = name[0 .. $ - name.find("/").length + 1];
break;
}
}
foreach(name, am; archive.directory)
{
if (!am.expandedSize) continue;
 
string path = buildPath(outdir, chompPrefix(name, prefix));
std.stdio.writeln(path);
auto dir = dirName(path);
if (!dir.empty && !dir.exists)
mkdirRecurse(dir);
archive.expand(am);
write(path, am.expandedData);
}
}
 
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
string fmt(Args...)(string fmt, auto ref Args args)
{
import std.array, std.format;
auto app = appender!string();
formattedWrite(app, fmt, args);
return app.data;
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.