process-gzipped-csvs.js

'use strict'
//  by Charlie Hoey <me@charliehoey.com>
//
//  script for processing a folder full of gzipped CSV files into a single filtered CSV file.
//  specifically this calculates x/y/z positions and distances for stars that contain a parallax measurement
// ......
var csv = require('fast-csv')
var fs = require('fs')
const zlib = require('zlib');
var glob = require('glob');
var r; // this is probably not needed anymore
var first = true;
var headers = false;
var fileList = false;
const DEG2RAD = Math.PI / 180;

function processFile(file) {
  if(file === undefined) return;
  fs.createReadStream(file)
    .pipe(zlib.createGunzip())
    .pipe(csv())
    .on('data', d => {
      // assume the first row is a list of headers, we can use this to do easy lookups of rows later
      if(!headers) {
        headers = d;
      } else {
        // only process rows that have a measurement for parallax
        if(d[headers.indexOf('parallax')] !== '') {
          // build an array of values for each of these keys, converting parallax to distance in parsecs
          let rec = ['source_id', 'ra', 'dec', 'parallax', 'phot_g_mean_mag'].map( k => {
            if(k !== 'parallax') {
              return d[headers.indexOf(k)];
            } else {
              return 1 / d[headers.indexOf(k)];
            }
          });
          // add three more fields for the x,y and z position of each star
          rec.push( Math.cos(rec[1] * DEG2RAD) * Math.cos(rec[2] * DEG2RAD) * rec[3] ); // x coord + dist
          rec.push( Math.sin(rec[1] * DEG2RAD) * Math.cos(rec[2] * DEG2RAD) * rec[3] ); // y coord + dist
          rec.push( Math.sin(rec[2] * DEG2RAD) * rec[3] ); // z coord + dist
          // print a csv list to stdout, which should probably be going to a file like "node process-gzipped-csvs.js >> final.csv"
          console.log(rec.join(','));
        }
      }
    })
    .on('end', () => {
      processFile(fileList.pop())
    });
}

glob('/YOUR/PATH/HERE/csv/*.gz', {}, (err, files) => {
  fileList = files;
  processFile(fileList.pop())
});