Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
const fs = require('fs');
const readline = require('readline');
const roundTime = (date, period) => {
const year = date.getFullYear();
const month = date.getMonth() + 1;
const day = date.getDate();
if (period === '1d') {
return `${year}-${month}-${day}`;
} else if (period === '1h') {
return `${year}-${month}-${day} ${date.getHours()}:00`;
} else if (period === '5m') {
const fiveMinutes = Math.floor(date.getMinutes() / 5) * 5;
const fiveMinuteString = fiveMinutes.toString().padStart(2, '0');
return `${year}-${month}-${day} ${date.getHours()}:${fiveMinuteString}`;
}
console.error(period, 'is not valid');
};
class CsvConverter {
constructor(options) {
this.inputFile = options.inputFile;
this.timeFrame = options.timeFrame;
this.dataObject = {};
this.lineCount = 0;
this.lastDay = null;
this.dayVolume = 0;
this.open = 0;
this.high = -Infinity;
this.low = Infinity;
this.close = 0;
this.processingStartTime = Date.now();
}
saveArrayToCsv(arr) {
let csvString = `${Object.keys(arr[0]).join(',')}\n`; // create headers from object keys
arr.forEach(row => {
// this relies on the object keys being in the same order for each item in the array.
// which I think is sensible
csvString += `${Object.values(row).join(',')}\n`;
});
fs.writeFileSync(this.inputFile.replace(/.csv$/, '_summary.csv'), csvString);
}
start() {
const rl = readline.createInterface({
input: fs.createReadStream(this.inputFile),
crlfDelay: Infinity
});
rl.on('line', line => this.handleLine(line));
rl.on('close', () => this.handleClose());
}
handleLine(line) {
this.lineCount++;
const transaction = line.split(',');
const thisPrice = Number(transaction[1]);
const thisVolume = Number(transaction[2]);
const date = new Date(Number(transaction[0]) * 1000);
const dateTimeString = roundTime(date, this.timeFrame);
if (this.lastDay && dateTimeString !== this.lastDay) {
// this is a new day, wrap up the averages for the previous day.
this.dataObject[this.lastDay].open = this.open;
this.dataObject[this.lastDay].high = this.high;
this.dataObject[this.lastDay].low = this.low;
this.dataObject[this.lastDay].close = this.close;
this.dataObject[this.lastDay].volume = this.dayVolume;
this.dataObject[dateTimeString] = {};
this.dayVolume = thisVolume;
this.open = thisPrice;
this.high = -Infinity;
this.low = Infinity;
this.lastDay = dateTimeString;
} else {
if (!this.lastDay) {
// for the very first row
this.open = thisPrice;
this.dataObject[dateTimeString] = {};
this.lastDay = dateTimeString;
}
this.dayVolume += thisVolume;
this.close = thisPrice;
this.high = Math.max(thisPrice, this.high);
this.low = Math.min(thisPrice, this.low);
}
}
handleClose() {
this.dataObject[this.lastDay].open = this.open;
this.dataObject[this.lastDay].high = this.high;
this.dataObject[this.lastDay].low = this.low;
this.dataObject[this.lastDay].close = this.close;
this.dataObject[this.lastDay].volume = this.dayVolume;
const dataArray = Object.entries(this.dataObject).map(([date, val]) => ({
date,
...val,
}));
this.saveArrayToCsv(dataArray);
console.log(`Processed ${this.lineCount.toLocaleString()} lines in ${Date.now() - this.processingStartTime}ms`);
process.exit(0);
}
}
const csvConverter = new CsvConverter({
inputFile: './bitfinexUSD.csv',
timeFrame: '1d', // '1d' | '1h' | '5m'
});
csvConverter.start();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.