fetch_candles.js

/*************************************************************************************/
/* Import Dependencies */
/*************************************************************************************/

const { mapSeries } = require('async');
const axios = require('axios');
const fs = require('fs');
const { chain, flatten, sortBy, sortedUniqBy } = require('lodash');
const moment = require('moment');


/*************************************************************************************/
/* Define Constants */
/*************************************************************************************/

const START = moment.utc('2017-08-01 00:00:00');
const END = moment.utc('2017-08-04 00:00:00');
const GRANULARITY = 60; // every 60 seconds


/*************************************************************************************/
/* Fetch Candles
/*************************************************************************************/

const ranges = [];
let start = START, end;
while (!end || end.valueOf() < END.valueOf()) {
  end = start.clone().add(200 * GRANULARITY, 'seconds');
  ranges.push({ start, end });
  start = end;
}

const fetch = ({ start, end }, callback) => {
  console.log(`Fetching ${ GRANULARITY }-second candles from ${ start.toISOString() } (${ start.valueOf() }) to ${ end.toISOString() } (${ end.valueOf() })`);
  axios({
    url: 'https://api.gdax.com/products/BTC-USD/candles',
    params: {
      start: start.toISOString(),
      end: end.toISOString(),
      granularity: GRANULARITY
    },
  }).then((res) => {
    setTimeout(() => callback(null, res.data), 1000); // Pause for a bit to avoid 3-requests-per-second rate limit
  }).catch(callback);
};

const finished = (err, fetchResults) => {
  if (err) {
    let errorMessage = `ERROR: ${ err.message }`;
    if (err.response && err.response.data && err.response.data.message) {
      errorMessage += ` - ${ err.response.data.message }`;
    }
    console.error(errorMessage);
  } else {
    const baseFilename = `${ START.toISOString() }_${ END.toISOString() }_${ GRANULARITY }s`;
    const jsonPath = `${ __dirname }/${ baseFilename }.json`;
    const csvPath = `${ __dirname }/${ baseFilename }.csv`;
    const candles = chain(fetchResults)
      .flatten()
      .sortBy((candle) => candle[0])
      .sortedUniqBy((candle) => candle[0]);
    const jsonContent = JSON.stringify(candles);
    const csvHeader ='Time,Low,High,Open,Close,Volume,Time (UTC)\n';
    const csvContent = candles.reduce((content, candle) => {
      return content + candle.concat(moment(candle[0] * 1000).utc()).join(',') + "\n"; }
    , csvHeader);
    fs.writeFileSync(jsonPath, jsonContent);
    fs.writeFileSync(csvPath, csvContent);
  }
};

mapSeries(ranges, fetch, finished);