dataminer.php

<?php

/**
 * Mines data from the battle net API.
 *
 * Should be used as a console command.
 */
class DataMiner
{
    private $base_uri;
    private $api_key;
    private $output_dir;
    private $buffer = [];

    /**
     * @param string $base_uri
     * @param string $api_key
     * @param string $output_dir
     */
    public function __construct($base_uri, $api_key, $output_dir)
    {
        $this->base_uri   = $base_uri;
        $this->api_key    = $api_key;
        $this->output_dir = $output_dir;
    }

    public function fetchItems(array $item_ids, callable $callback)
    {
        // Create a collection of Request URI's
        $url_collection = [];
        foreach ($item_ids as $id) {
            $url_collection[$id] = $this->base_uri . '/wow/item/' . $id . '?apikey=' . $this->api_key;
        }

        $this->go($url_collection, $callback);
    }

    public function fetchAchievements(array $ids, callable $callback)
    {
        // Create a collection of Request URI's
        $url_collection = [];
        foreach ($ids as $id) {
            $url_collection[$id] = $this->base_uri . '/wow/achievement/' . $id . '?apikey=' . $this->api_key;
        }

        $this->go($url_collection, $callback);
    }

    private function go($urls, callable $callback)
    {
        // Set-up master thread & curl options.
        $handles = [];
        $options = [
            CURLOPT_TIMEOUT        => 2,
            CURLOPT_CONNECTTIMEOUT => 2,
            CURLOPT_SSL_VERIFYHOST => false,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_HEADER         => false,
            CURLOPT_RETURNTRANSFER => true
        ];

        // Create handles
        foreach ($urls as $url) {
            $ch = curl_init($url);
            curl_setopt_array($ch, $options);
            $handles[] = $ch;
        }

        // Execute!
        while (count($handles) > 0) {
            $handles = $this->exec($handles);
            sleep(1); // Wait 1 second before running the next batch.
            $this->processBuffer($callback);
        }
    }

    private function processBuffer(callable $callback)
    {
        foreach ($this->buffer as $id => $data) {

            // Do stuff...
            $callback($data);

            // Clean memory.
            unset($this->buffer[$id]);
        }
    }

    private function exec(array $handles)
    {
        $chunks  = array_splice($handles, 0, 99); // 100 requests per batch.
        $master  = curl_multi_init();
        $running = false;

        // Add handles to master request.
        foreach ($chunks as $handle) {
            curl_multi_add_handle($master, $handle);
        }

        // Do all the work...
        do {
            curl_multi_exec($master, $running);
            curl_multi_select($master);
        } while ($running > 0);

        // Get results & clean up curl resources.
        foreach ($chunks as $handle) {
            $res = json_decode(curl_multi_getcontent($handle), true);
            if (isset($res['status']) && $res['status'] == 'nok') {
                echo 'E';
                continue;
            }

            if (isset($res['id']) && (int)$res['id'] > 1000) {
                $this->buffer[$res['id']] = $res;
                echo '+';
            } else {
                print_r($res);
                exit;
                echo '.';
            }
            @curl_close($handle);
        }
        @curl_multi_close($master);
        echo "\n";

        return $handles;
    }
}


$dm = new DataMiner('https://eu.api.battle.net', '', __DIR__ . '/items.csv');
$dm->fetchAchievements(range(1000, 2500), function ($data) {
    print_r($data);
});

// print_r(json_decode('[' . trim(file_get_contents(__DIR__ . '/test.json'), ',') . ']', true));