Combine Flightscope Mevo CSVs from dated directories

mevo-parser.js

const csv = require('csv-parser');
const fs = require('fs');
// Written by Michael Sitver
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
var shots = []
var files = []
var headers = [
    { id: "ballSpeed", title: 'Ball Speed (mph)'},
    {id: "clubSpeed", title: "Club Speed (mph)"},
    {id: "smash", title: "Smash"},
    {id: "carryDistance", title: "Carry Distance (yds)"},
    {id: "launchAngle", title: "Launch Angle V (°)"},
    {id: "spin", title: "Spin (rpm)"},
    {id: "height", title: "Height (ft)"},
    {id: "time", title: "Time (s)"},
    {id: "club", title: "Club"},
    {id: "date", title: "Date"},
    {id: "srcFile", title: "src file"}
  ]
var yearsParsed = 0
const csvWriter = createCsvWriter({
  path: '../Data/Aggregate/test-out.csv',
  header: headers
});
/*
Plan:
- Load files
- Add date column and source column
- Combine files and give it the filename of aggregate-to-${today}
{ 'Ball Speed (mph)': '123',
  'Club Speed (mph)': '85',
  Smash: '1.45',
  'Carry Distance (yds)': '189.4',
  'Launch Angle V (°)': '18.1',
  'Spin (rpm)': '3114',
  'Height (ft)': '90',
  'Time (s)': '6',
  Club: '3 Hybrid' }
*/

function finishFile(){
    // Sanitize data, save to sanitized array, then turn into a CSV.
    var cleanedShots = []
    for(i = 0; i < shots.length; i++){
    cleanedShots[cleanedShots.length] = { ballSpeed: shots[i]['Ball Speed (mph)'],
   clubSpeed: shots[i]['Club Speed (mph)'],
   smash: shots[i]['Smash'],
   carryDistance: shots[i]['Carry Distance (yds)'],
   launchAngle: shots[i]['Launch Angle V (°)'],
    spin: shots[i]['Spin (rpm)'],
    height: shots[i]['Height (ft)'],
    time: shots[i]['Time (s)'],
    club: shots[i]['Club'],
date: shots[i]['date'],
srcFile: shots[i]['srcFile']
}

    if(i == shots.length - 1){
    csvWriter
  .writeRecords(cleanedShots)
  .then(()=> console.log('The CSV file was written successfully'));
  fs.writeFile('../Data/Aggregate/test-data.json', {headers: headers, shots: cleanedShots}, function (err) {
    if (err) throw err;
    console.log('Saved!');
  });
    }
    }
}

function addFile(num){
    // Add file to aggregate file
    console.log(files[num])
    var filepath = files[num]
    fs.createReadStream(files[num])
      .pipe(csv())
      .on('data', (row) => {
        //console.log(row);
        shots[shots.length] = row
        shots[shots.length - 1].srcFile = filepath
        var fileDate = filepath.split('../Data/').join('').split('/')
        fileDate.pop()
        var fullDate = fileDate[1] + "/" + fileDate[2] + "/" + fileDate[0]
        //console.log(fullDate)
        shots[shots.length - 1].date = fullDate
      })
      .on('end', () => {
        console.log('CSV file successfully processed');
        if(num < files.length - 1){
            addFile(num + 1)
        }else{
            finishFile()
        }
        
      });
    }
function parseDay(year, month, day){
    fs.readdir('../Data/' + year + "/" + month + "/" + day, function(dayErr, dayItems) {
        if(dayItems.length){
            for(var a = 0; a < dayItems.length; a++){
                files[files.length] = '../Data/' + year + "/" + month + "/" + day + "/" + dayItems[a]
                console.log("Path: " + '../Data/' + year + "/" + month + "/" + day + "/" + dayItems[a])
            }
        }
    });
}
 function parseMonth(year, month, callback){
    fs.readdir('../Data/' + year + "/" + month, function(err, days) {
     
        for (var k=0; k < days.length; k++) {
        parseDay(year, month, days[k])   
        }
    });
 }  
    

function aggregate(){


    fs.readdir('../Data/2019', function(err, months) {
       for(var a = 0; a < months.length; a++){
           parseMonth('2019', months[a])
           if(a == months.length - 1){
               yearsParsed += 1
           }
       }
    });
    fs.readdir('../Data/2020', function(err, months) {
        for(var a = 0; a < months.length; a++){
            parseMonth('2020', months[a])
            if(a == months.length - 1){
                yearsParsed += 1
            }
        }
     });
var parsing = setInterval(function(){
    if(yearsParsed >= 2){
        clearInterval(parsing)
        addFile(0)
    }
}, 200)

}
aggregate()