Simple Apify Script for scraping 24hr kline data from Binance

binance_price_scraper.js

const Apify = require('apify');
const axios = require('axios');

/**
 * Converts List of Lists of kline data from Binance to a list of
 * dictionaries.
 * @param  {[List[List[String]]} data     The raw data returned from binance
 * @param  {[String]}            exchange The name of the exchange to scrape
 * @return {[Dict[String, String]}
 */
function convertListToJSON(data, exchange) {
    var ls = []
    for (var row of data) {
        ls.push({
            "open_time": row[0],
            "open": row[1],
            "high": row[2],
            "low": row[3],
            "close": row[4],
            "volume": row[5],
            "close_time": row[6],
            "quote_asset_volume": row[7],
            "number_of_trades": row[8],
            "taker_buy_base_asset_volume": row[9],
            "taker_buy_quote_asset_volume": row[10],
            "ignore": row[11],
            "exchange": exchange
        });
    }
    return ls;
}

// Simple promise to sleep for a set amount of time.  It helps with not overhitting
// endpoints.
const sleep = (milliseconds) => {
  return new Promise(resolve => setTimeout(resolve, milliseconds))
}

Apify.main(async () => {
    // Get the input from the Apify Console in this case it will b json of the form:
    //{ "exchanges": ["EXCHANGE1", "EXCHANGE2", ETC]}
    const input = await Apify.getInput();
  
    // Initialize an Apify Key-Value store Dataset called binance-kline
    const dataset = await Apify.openDataset('binance-kline');

    // Step through each exchange in our input and pull the kline data from the last 24hrs
    for (var exchange of input["exchanges"]){
        let url = 'https://api.binance.com/api/v1/klines?&symbol=' + exchange + '&interval=3m';
        var nothing = await axios.get(url)
            .then(async response => {
                console.log(exchange);
                
                // Save the data into the key-value store
                await dataset.pushData(convertListToJSON(response.data, exchange));
            })
            .catch(error => {
                console.log(error);
            });
        
            // Sleep for 200ms to be nice and avoid over hitting the endpoint.
            sleep(200).then(() => {});
    }
});