Skip to content

@MartinNowak /dget.d
Created

Embed URL

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
github project fetcher
import std.algorithm, std.exception, std.file, std.json, std.path, std.range, std.zip, std.net.curl;
static import std.stdio;
void main(string[] args)
{
bool isSnapshot;
foreach(arg; args[1 .. $])
{
auto outdir = arg.replace("/", "_");
enforce(!outdir.exists, fmt("output folder '%s' already exists", outdir));
findRepo(arg).findFile(isSnapshot).download()
.unzip(outdir, isSnapshot);
}
}
string findRepo(string arg)
{
import std.regex;
enum rule = ctRegex!(r"^(?:([^/]*)/)?([^/]*)$");
auto m = match(arg, rule);
enforce(!m.empty, fmt("expected 'org/repo' but found '%s'", arg));
string result = m.captures[1];
if (!result.empty)
result ~= "/" ~ m.captures[2];
else
{
auto repo = m.captures[2];
auto org = find!(o => o.hasRepo(repo))(defaultOrgs);
enforce(!org.empty, fmt("repo '%s' was not found among '%(%s, %)'",
repo, defaultOrgs));
result = org.front ~ "/" ~ repo;
}
std.stdio.writefln("repo: %s", result);
return result;
}
string findFile(string orgRepo, out bool isSnapshot)
{
string result;
auto downloads = fmt("https://api.github.com/repos/%s/downloads", orgRepo)
.reqJSON().array;
if (!downloads.empty)
{
auto latest = downloads.front.object;
enforce(latest["content_type"].str == ".zip",
fmt("can't handle content type '%s' of download '%s'",
latest["content_type"].str, latest["html_url"].str));
result = latest["html_url"].str;
isSnapshot = false;
std.stdio.writefln("downloading: '%s'", latest["name"].str);
}
else
{
result = fmt("https://github.com/%s/zipball/master", orgRepo);
auto http = HTTP(result);
http.method = HTTP.Method.head;
http.perform();
enforce(http.statusLine().code / 100 == 2,
fmt("found no download for repo '%s'", orgRepo));
isSnapshot = true;
std.stdio.writefln("downloading: 'master zipball'");
}
return result;
}
bool hasRepo(string org, string repo)
{
return fmt("https://api.github.com/orgs/%s/repos?type=public", org)
.reqJSON().array.canFind!(a => a.object["name"].str == repo)();
}
@property const(string[]) defaultOrgs()
{
static immutable value = ["D-Programming-Deimos", "D-Programming-Language"];
// TODO: pick up env var
return value;
}
JSONValue reqJSON(string url)
{
return get(url).parseJSON();
}
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
ubyte[] download(string url)
{
// doesn't work because it already timeouts after 2 minutes
// return get!(HTTP, ubyte)();
import core.time, std.array, std.conv;
auto buf = appender!(ubyte[])();
size_t contentLength;
auto http = HTTP(url);
http.method = HTTP.Method.get;
http.onReceiveHeader((k, v)
{
if (k == "content-length")
contentLength = to!size_t(v);
});
http.onReceive((data)
{
buf.put(data);
std.stdio.writef("%sk/%sk\r", buf.data.length/1024,
contentLength ? to!string(contentLength/1024) : "?");
std.stdio.stdout.flush();
return data.length;
});
http.dataTimeout = dur!"msecs"(0);
http.perform();
immutable sc = http.statusLine().code;
enforce(sc / 100 == 2 || sc == 302,
fmt("HTTP request returned status code %s", sc));
std.stdio.writeln("done ");
return buf.data;
}
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
void unzip(ubyte[] data, string outdir, bool isSnapshot)
{
import std.string;
scope archive = new ZipArchive(data);
std.stdio.writeln("unpacking:");
string prefix;
mkdir(outdir);
if (isSnapshot)
{
foreach(name, _; archive.directory)
{
prefix = name[0 .. $ - name.find("/").length + 1];
break;
}
}
foreach(name, am; archive.directory)
{
if (!am.expandedSize) continue;
string path = buildPath(outdir, chompPrefix(name, prefix));
std.stdio.writeln(path);
auto dir = dirName(path);
if (!dir.empty && !dir.exists)
mkdirRecurse(dir);
archive.expand(am);
write(path, am.expandedData);
}
}
//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
string fmt(Args...)(string fmt, auto ref Args args)
{
import std.array, std.format;
auto app = appender!string();
formattedWrite(app, fmt, args);
return app.data;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.