Skip to content

Instantly share code, notes, and snippets.

@toddgeist
Created December 30, 2014 03:56
Show Gist options
  • Save toddgeist/9766463ea17810eb2401 to your computer and use it in GitHub Desktop.
Save toddgeist/9766463ea17810eb2401 to your computer and use it in GitHub Desktop.
creating a massive CSV with node streams
/**
* Created by toddgeist on 12/28/14.
*/
var faker = require('faker');
var stringify = require('csv-stringify');
var uuid =require('uuid');
var moment = require('moment');
var fs = require('fs');
var Readable = require('stream').Readable;
var start = moment();
var birthDateStart = moment().subtract(100, 'years');
var birthDateEnd = moment().subtract(18, 'years');
var rows = 10000000;
var fileName = 'data.csv';
var columns = [
'uuid',
'firstName',
'lastName',
'email',
'mobilePhone',
'workPhone',
'address',
'city',
'state',
'postalCode',
'birthDate',
'company',
// 'slogan',
// 'notes',
'userName'
].join(',') + '\n';
var person = function(){
return [
uuid.v4(),
faker.name.firstName(),
faker.name.lastName(),
faker.internet.email(),
faker.phone.phoneNumber(),
faker.phone.phoneNumber(),
faker.address.streetAddress(),
faker.address.city(),
faker.address.state(),
faker.address.zipCode(),
moment( faker.date.between(birthDateStart, birthDateEnd)).format('M/d/YYYY'),
faker.company.companyName(),
// faker.company.catchPhrase(),
// faker.lorem.paragraphs(),
faker.internet.userName()
].join(',') + '\n';
}
rs = new Readable;
var writable = fs.createWriteStream(__dirname + '/data.csv');
var i = 0;
rs._read = function(){
i++;
if( i===1) {
rs.push(columns)
}else if(i > rows){
rs.push(null)
}else{
rs.push(person())
}
};
rs.pipe(writable);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment