Skip to content

Instantly share code, notes, and snippets.

@georgeh
Last active June 9, 2021 15:38
Show Gist options
  • Save georgeh/c02660e14771a5d78d7e9f09439726b3 to your computer and use it in GitHub Desktop.
Save georgeh/c02660e14771a5d78d7e9f09439726b3 to your computer and use it in GitHub Desktop.
CSV logging server
/**
* Usage: node server.js foo.csv
*
* Test with: curl -d "{\"Foo\":1,\"Bar\":\"baz\"}" -X POST http://localhost:8000/
*
*/
const http = require('http');
const fs = require('fs');
const fsPromises = require('fs/promises');
const PORT = 8000;
const MAX_REQUEST_SIZE = 1e6;
const outFilename = process.argv[2];
if ( ! outFilename ) {
console.error( `Usage: ${process.argv[0]} ${process.argv[1]} OUTPUT_FILENAME`);
process.exit();
}
const escapeField = ( val ) => {
if ( typeof val === 'string' && val.includes( ',' ) ) {
return '"' + val.replace( '"', '""' ) + '"';
}
return val;
}
// Use the stream API to read headers so we don't read in giant files just
// to get the first line
const getHeaders = async ( filename ) => {
return new Promise( ( resolve, reject ) => {
let headerLine = '';
const stream = fs.createReadStream( filename, { encoding: 'utf8', start: 0 } );
stream.on( 'error', ( err ) => {
reject( err );
} );
function onData( chunk ) {
headerLine += chunk;
if ( headerLine.includes('\n') ) {
const headerCSV = headerLine.split('\n')[0];
const headers = headerCSV.split(',').slice(1);
resolve( headers );
stream.removeListener( 'data', onData );
stream.destroy();
}
}
stream.on( 'data', onData );
} );
}
( async () => {
let headersWritten = false;
let fields = [];
try {
fields = await getHeaders( outFilename );
console.log( 'Using existing fields: ' + fields.join(', ') + '\n');
headersWritten = true;
} catch ( err ) {
console.error( err );
console.log( 'Creating new file ' + outFilename + '\n');
headersWritten = false;
}
let filehandle = await fsPromises.open( outFilename, 'a' );
process.on('SIGINT', async () => {
try {
await filehandle.close();
} catch ( err ) {
console.error( err );
}
process.exit();
});
// kill -HUP to rotate CSV files
process.on('SIGHUP', async () => {
await filehandle.close();
filehandle = await fsPromises.open( outFilename, 'a' );
await filehandle.write( ['Date', ...fields ].join(',') + '\n' );
});
const server = http.createServer(async ( req, res ) => {
if ( req.method === 'POST' ) {
let body = '';
req.on( 'data', ( data ) => {
body += data;
if ( body.length > MAX_REQUEST_SIZE ) {
req.destroy();
}
} );
req.on( 'end', async () => {
try {
const postData = JSON.parse( body );
if ( ! headersWritten ) {
fields = Object.keys( postData );
await filehandle.write( ['Date', ...fields ].join(',') + '\n' );
console.log( 'Using fields: ' + fields.join(', ') + '\n');
headersWritten = true;
}
const row = fields.map( field => escapeField( postData[field] ) );
await filehandle.write( [(new Date()).toISOString(), ...row].join(',') + '\n' );
res.writeHead( 200 );
res.end( http.STATUS_CODES[200] + '\n' );
} catch ( err ) {
console.error( body );
console.error( err );
res.writeHead( 500 );
res.end( err + '\n' );
}
} );
}
});
server.listen( PORT );
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment