convert-stock-data.js

const fs = require('fs');
const readline = require('readline');

const roundTime = (date, period) => {
  const year = date.getFullYear();
  const month = date.getMonth() + 1;
  const day = date.getDate();

  if (period === '1d') {
    return `${year}-${month}-${day}`;
  } else if (period === '1h') {
    return `${year}-${month}-${day} ${date.getHours()}:00`;
  } else if (period === '5m') {
    const fiveMinutes = Math.floor(date.getMinutes() / 5) * 5;
    const fiveMinuteString = fiveMinutes.toString().padStart(2, '0');
    return `${year}-${month}-${day} ${date.getHours()}:${fiveMinuteString}`;
  }

  console.error(period, 'is not valid');
};

class CsvConverter {
  constructor(options) {
    this.inputFile = options.inputFile;
    this.timeFrame = options.timeFrame;

    this.dataObject = {};
    this.lineCount = 0;
    this.lastDay = null;
    this.dayVolume = 0;
    this.open = 0;
    this.high = -Infinity;
    this.low = Infinity;
    this.close = 0;

    this.processingStartTime = Date.now();
  }

  saveArrayToCsv(arr) {
    let csvString = `${Object.keys(arr[0]).join(',')}\n`; // create headers from object keys

    arr.forEach(row => {
      // this relies on the object keys being in the same order for each item in the array.
      // which I think is sensible
      csvString += `${Object.values(row).join(',')}\n`;
    });

    fs.writeFileSync(this.inputFile.replace(/.csv$/, '_summary.csv'), csvString);
  }

  start() {
    const rl = readline.createInterface({
      input: fs.createReadStream(this.inputFile),
      crlfDelay: Infinity
    });

    rl.on('line', line => this.handleLine(line));
    rl.on('close', () => this.handleClose());
  }

  handleLine(line) {
    this.lineCount++;

    const transaction = line.split(',');

    const thisPrice = Number(transaction[1]);
    const thisVolume = Number(transaction[2]);
    const date = new Date(Number(transaction[0]) * 1000);
    const dateTimeString = roundTime(date, this.timeFrame);

    if (this.lastDay && dateTimeString !== this.lastDay) {
      // this is a new day, wrap up the averages for the previous day.
      this.dataObject[this.lastDay].open = this.open;
      this.dataObject[this.lastDay].high = this.high;
      this.dataObject[this.lastDay].low = this.low;
      this.dataObject[this.lastDay].close = this.close;
      this.dataObject[this.lastDay].volume = this.dayVolume;

      this.dataObject[dateTimeString] = {};
      this.dayVolume = thisVolume;
      this.open = thisPrice;
      this.high = -Infinity;
      this.low = Infinity;

      this.lastDay = dateTimeString;
    } else {
      if (!this.lastDay) {
        // for the very first row
        this.open = thisPrice;
        this.dataObject[dateTimeString] = {};
        this.lastDay = dateTimeString;
      }

      this.dayVolume += thisVolume;
      this.close = thisPrice;
      this.high = Math.max(thisPrice, this.high);
      this.low = Math.min(thisPrice, this.low);
    }
  }

  handleClose() {
    this.dataObject[this.lastDay].open = this.open;
    this.dataObject[this.lastDay].high = this.high;
    this.dataObject[this.lastDay].low = this.low;
    this.dataObject[this.lastDay].close = this.close;
    this.dataObject[this.lastDay].volume = this.dayVolume;

    const dataArray = Object.entries(this.dataObject).map(([date, val]) => ({
      date,
      ...val,
    }));

    this.saveArrayToCsv(dataArray);

    console.log(`Processed ${this.lineCount.toLocaleString()} lines in ${Date.now() - this.processingStartTime}ms`);

    process.exit(0);
  }
}

const csvConverter = new CsvConverter({
  inputFile: './bitfinexUSD.csv',
  timeFrame: '1d', // '1d' | '1h' | '5m'
});

csvConverter.start();