Last active
August 13, 2018 22:43
-
-
Save ChrisWhealy/e1c4b2477d2f703074d6b40f11f9fdb5 to your computer and use it in GitHub Desktop.
Read a CSV file (with optional header line) and transform it into an object array
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const compose = fn1 => fn2 => val => fn2(fn1(val)) | |
// Remove only the start and end double quotes in a string | |
const stripQuotes = str => str.replace(/(^")|("$)/g,"") | |
// Remove commas that might occur inside names | |
const removeCommas = str => str.replace(/,/g,"") | |
const cleanTxt = compose(stripQuotes)(removeCommas) | |
// Reject null entries | |
const notNull = el => el !== null | |
/*********************************************************************************************************************** | |
* Read a CSV file (with an optional header line) and transform it into an array of objects. | |
* The properties of the object are supplied in the array prop_list. The number and order of the names in prop_list | |
* must match the number and order of the columns in the CSV file | |
* | |
* To understand how the sequence of processing works in the following function, read the comments in numerical order | |
* where 1) is at the end! | |
*/ | |
const csv_to_object_array = (filename, prop_list, hasHdr) => | |
((lineArray => // 2) Arg "lineArray" is the CSV file now as an array of lines | |
(hasHdr ? (lineArray.slice(1)) : lineArray) // 3) Optionally remove the header line | |
.map( // 4) For each line in the CSV file... | |
line => // "line" is the arg to the mapper function | |
(line.length > 0) // 5) Check for an empty line | |
? line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/) // 6) Split line using positive lookahead to ignore quoted commas | |
.reduce( // 7) Transform array of fields into new object using a... | |
(acc, el, idx) => // 8) Reducer function | |
(_ => acc) // 11) Inner fn ignores its arg value and returns the accumulator | |
(acc[prop_list[idx]] = cleanTxt(el)) // 10) Add new prop/value to acc and pass to inner, anonymous fn | |
, {} // 9) Accumulator starts as an empty array | |
) // | |
: null // 12) Empty lines become null (will be filtered out next) | |
) // | |
.filter(notNull)) // 13) Finally, filter accumulator to remove null entries | |
) // | |
((fs.readFileSync(filename, "utf-8")).split(/\r?\n/)) // 1) Read CSV file, split into lines & pass to anonymous inner fn | |
// Define the names of the columns in the CSV file. These names become the property names of the generated object | |
const airports_props = [ | |
"id", "name", "city", "country", "iata", "icao", "lat", "lng" | |
, "elevation", "tz", "dst", "olson_tz_name", "type", "source" | |
] | |
// Assuming you have already downloaded airports.dat from OpenFlights.org, convert this CSV files to an object array | |
var openflights_airports = csv_to_object_array("airports.dat", airports_props, false) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment