downloader.d

// Written in the D programming language.

/**
 * High peformance downloader
 *
 * Implemented according to <a href="http://yusukebe.com/archives/20120229/072808.html">this implementation</a>.
 *
 * Example:
 * -----
 * dmd -L-lcurl -run downloader.d
 * -----
 */

import core.atomic : atomicOp;

import std.array       : join;
import std.conv        : to;
import std.getopt      : getopt;
import std.file        : exists, mkdir;
import std.json        : parseJSON, toJSON;
import std.path        : buildPath, setExtension;
import std.parallelism : parallel;
import std.digest.md   : md5Of, toHexString;
import std.net.curl    : get, download;
import std.regex       : match;
import std.string      : toLower;
import std.stdio       : writeln, writefln;
import std.uri         : encode;

@trusted
string buildQuery(in string[string] params)
{
    string[] queries;

    foreach (k, v; params)
        queries ~= encode(k) ~ "=" ~ encode(v);

    return queries.join("&");
}

void main(string[] args)
{
    immutable AppURI = "http://api.bing.net/json.aspx";
    immutable AppId  = "Your AppId";
    string storeDir  = "./images";
    string query;
    size_t page;

    getopt(args,
           "query", &query,
           "dir",   &storeDir,
           "page",  &page);

    if (!storeDir.exists)
        mkdir(storeDir);

    writefln("Configuration: query = %s, dir = %s, offset = %s", query, storeDir, page);

    for (shared(size_t) downloadCount; ; page++)
    {
        immutable uri = AppURI ~ '?' ~ buildQuery(["AppId":        AppId,
                                                   "Version":      "2.2",
                                                   "Market":       "ja-JP",
                                                   "Sources":      "Image",
                                                   "Image.Count":  "50",
                                                   "Image.Offset": to!string(page * 50),
                                                   "Adult":        "off", 
                                                   "Query":        query]);
        auto response = get(uri);
        auto result   = parseJSON(response).object["SearchResponse"];
        if ("Errors" in result.object)
        {
            writeln("Error: reason = ", toJSON(&result.object["Errors"]));
            break;
        }

        foreach (ref entry; parallel(result.object["Image"].object["Results"].array))
        {
            immutable downloadUri = entry.object["MediaUrl"].str;
            if (!downloadUri.match(".jpe?g$"))
                continue;

            immutable path = buildPath(storeDir, md5Of(downloadUri).toHexString().setExtension("jpg"));
            if (path.exists)
                continue;

            download(downloadUri, path);

            atomicOp!"+="(downloadCount, 1);
            writeln(downloadCount, " : downloaded... ", downloadUri);
        }
    }

    writeln("done!");
}