exportEthBlocks.js

//exportEthBlocks.js
//Node.js script that gets block header data from geth via web3 RPC calls and exports to a CSV file
//Warning: it could take a few days to process the entire blockchain
//Automatically picks up were it left off
//CSV file format is:
//number,timestamp,difficulty,totalDifficulty,gasLimit,gasUsed,size,num tx
var fs = require('fs'),
	web3 = require('web3'),
    file = 'ethBlocks.csv',
    notifyInterval = 30, //seconds
    rpcHost = 'http://localhost:8545',
    startBlock = 0, //0 starts at the genesis block
    endBlock = -1; //-1 runs until current block is reached

if((endBlock != -1 && startBlock > endBlock) || endBlock < -1 || startBlock < 0) {
	console.log('Something\'s fucky.');
	process.exit();
}
	
console.log('Reading file...');
buff = fs.readFileSync(file, 'utf-8');
lines = buff.trim().split('\n');
lastLine = lines.slice(-1)[0];
fields = lastLine.split(',');
lastBlock = fields[0];

web3.setProvider(new web3.providers.HttpProvider(rpcHost));
while(1) {
	try {
		currentBlock = web3.eth.blockNumber;
	} catch(e) {
		console.log('RPC error, retrying in 5 seconds...');
		timer = process.hrtime();
		while(process.hrtime(timer)[0] < 5) {} //wait 5 sec
		continue
	}
	break;
}
console.log("Got latest block: " + currentBlock);
if(endBlock == -1)
	endBlock = currentBlock;
if(endBlock <= lastBlock) {
	console.log('endBlock already in file');
	process.exit();
}
if(startBlock <= lastBlock) {
	startBlock = Number(lastBlock) + 1;
	console.log('Resuming from last block in file: ' + lastBlock);
}
if(startBlock > lastBlock+1)
	console.log('Warning: gap between startBlock and last block in file');

function additionalBlockTime(_remainingTime, _secondsPerBlock) {
	var _addedTime = _remainingTime/10 * _secondsPerBlock; //assuming 10 sec blocks
	if(_addedTime < 10)
		return _addedTime;
	else 
		return additionalBlockTime(_addedTime, _secondsPerBlock);
}

startTime = process.hrtime();
blockAtStart = startBlock;
for(var i = startBlock; i <= endBlock; i++) {
	
	while(1) {
		try {
			block = web3.eth.getBlock(i);
			currentBlock = web3.eth.blockNumber;
		} catch(e) {
			console.log('RPC error, retrying in 5 seconds...');
			timer = process.hrtime();
			while(process.hrtime(timer)[0] < 5) {} //wait 5 sec
			continue
		}
		break;
	}

	fs.appendFileSync(file,
		block.number + ',' + block.timestamp + ',' + block.difficulty + ',' + block.totalDifficulty + ',' + block.gasLimit + ',' + block.gasUsed + ',' + block.size + ',' + block.transactions.length + '\n');
	
	diff = process.hrtime(startTime);
	elapsedTime = diff[0] + diff[1]/1e9;
	if(elapsedTime >= notifyInterval) {
		startTime = process.hrtime();
		secondsPerBlock = elapsedTime/(i-blockAtStart);
		blockAtStart = i;
		remainingTime = secondsPerBlock * (currentBlock - i);
		remainingTime += additionalBlockTime(remainingTime, secondsPerBlock);
		console.log('Processed to block ' + block.number + ' at ' + (1/secondsPerBlock).toFixed(1) + ' blocks/s. ETA ' + (remainingTime/3600).toFixed(1) + ' hr');
	}
}

console.log("FUCKING DONE");
process.exit();