Created
April 28, 2021 19:14
-
-
Save derekmc/23eeb7da2d500028b993c9abd05bd8c2 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const CSVFOLDER = "csv"; | |
const LOGFOLDER = "log"; | |
const LOGEXTENSION = ".csvlog"; | |
const DEFAULTDB = "csvdb"; | |
if(require.main === module){ | |
Test(); | |
} | |
// updates file: | |
// tablename, a, b, c, d, e, f, g... | |
// national debt simulator. | |
function TableSet({ dbname: DEFAULTDB, autoparse }){ | |
const folder = dbname; | |
const tables = {}; // this is the internal object | |
const tableset = {}; // this is the return object | |
const pendingupdates = []; // the changes which haven't been flushed to disk yet. | |
let logfile = ""; | |
tableset.update = (tablename, row) =>{ | |
if(!(tablename in tables)){ | |
tables[tablename] = Table(); } | |
let table = tables[tablename]; | |
} | |
tableset.flush = ()=> { | |
} | |
return tableset; | |
} | |
function Table(headers, options){ | |
if(!options) options = {}; | |
if(!headers) headers = []; | |
let ids = {}; | |
checkType(headers, "array"); | |
checkType(options, "object"); | |
// TODO let savedelay = ; // the minimum time between saving | |
let autoparse = options['autoparse']? true : false; | |
//let filename = options['filename']? options['filename'] : null; | |
//let folder = options['folder']? options['folder'] : '.'; | |
let autoid = options['autoid']? true : false; // generate an id, instead of using first entry. | |
//fs.mkdirSync(folder, {recursive: true}); | |
// checkType(autoid, "boolean"); | |
// checkType(autoparse, "boolean"); | |
//if(filename) checkType(filename, "string"); | |
// store autoids for live table no matter what, | |
// the autoid prop, just determines whether that is saved. | |
let currentAutoId = 1; | |
let columnquoted = []; | |
for(let i=0; i<headers.length; ++i){ | |
let header = headers[i].trim(); | |
let n = header.length - 1; | |
columnquoted[i] = (header[0] == "'" && header[n] == "'" || header[0] == '"' && header[n] == '"'); | |
} | |
let table = { | |
headers: headers, | |
rows: [], | |
ids: {}, | |
}; | |
function checkType(value, type){ | |
let t = typeof value; | |
if(type == "array" || type == "object"){ | |
t = Array.isArray(value)? "array" : t; } | |
if(t != type){ | |
throw new Error(`csvtable: expected "${type}" got ${value}, with type "${t}".`); } | |
} | |
function checkRow(row){ | |
if(!Array.isArray(row)){ | |
throw new Error("csvtable: row must be an array."); } | |
if(row.length == 0){ | |
throw new Error("csvtable: row must not be empty."); } | |
} | |
function getRowId(key){ | |
if(!table.ids.hasOwnProperty(key)){ | |
return -1; | |
} | |
return table.ids[key]; | |
} | |
table.addRow = (row)=>{ | |
checkRow(row); | |
let key = autoid? currentAutoId: row[0]; | |
if(!autoid && table.hasOwnProperty(key)){ | |
return false; | |
} | |
let rowid = currentAutoId++; | |
table.ids[key] = rowid; | |
table.rows[rowid] = row; | |
return rowid; | |
} | |
table.setProp = (name, value)=>{ | |
if(name == "autoid"){ | |
checkType(value, 'boolean'); | |
autoid = value; | |
} | |
if(name == "autoparse"){ | |
checkType(value, 'boolean'); | |
autoparse = value; | |
} | |
if(name == "filename"){ | |
checkType(value, 'string'); | |
filename = value; | |
} | |
} | |
table.getRow = (key)=>{ | |
let rowid = getRowId(key); | |
if(rowid == -1) return null; | |
return table.rows[rowid]; | |
return table.rows[key]; | |
} | |
// id is optional, and only used if autoid is true. | |
table.setRow = (row, id)=>{ | |
checkRow(row); | |
let rowid = autoid? id : getRowId(row[0]); | |
if(rowid==null || rowid == undefined || rowid < 0){ | |
rowid = currentAutoId++; | |
} | |
table.rows[rowid] = row; | |
} | |
table.deleteRow = (key)=>{ | |
let rowid = getRowId(key); | |
if(rowid < 0) return false; | |
if(!table.rows.hasOwnProperty(key)){ | |
return false; | |
} | |
delete table.ids[key]; | |
delete table.rows[rowid]; | |
return true; | |
} | |
table.save = (after)=>{ | |
if(!filename){ | |
return after(false); | |
} | |
const stream = fs.createWriteStream(folder + "/" + filename, {encoding: 'utf8'}); | |
if(after){ | |
if(typeof after != "function"){ | |
throw new Error("csvtable table.save(after): after must be a function."); | |
} | |
stream.on('finish', ()=>after(true)); | |
} | |
const quotecolumn = (x,i)=> columnquoted[i]?'"' + x + '"' : x; | |
let headerstr = table.headers.map(quotecolumn).join(","); | |
if(autoid) headerstr = "AutoId," + headerstr | |
stream.write(headerstr + "\n"); | |
//console.log(table); | |
for(const key in table.ids){ | |
const rowid = table.ids[key]; | |
const row = table.rows[rowid]; | |
//console.log(key, rowid, row); | |
let rowstr = rowToString(row); | |
if(autoid) rowstr = `${rowid},${rowstr}`; | |
stream.write(rowstr + "\n"); | |
} | |
} | |
// TODO a field is default quoted if the header is quoted. | |
table.load = (after)=>{ | |
if(!after) after = (success)=> console.log(`Table load ${success? "succeeded" : "failed"}`); | |
if(typeof after != "function"){ | |
throw new Error("csvtable table.load(after): after must be a function."); | |
} | |
if(!filename){ | |
return after(false); | |
} | |
let stream = fs.createReadStream(folder + "/" + filename, {encoding: 'utf8'}); | |
let line = ""; | |
let linenumber = 0; | |
let n = line.length; | |
//console.log("stream", stream); | |
stream.on('readable', function(){ | |
let chunk; | |
while((chunk = stream.read()) != null){ | |
readChunk(chunk); | |
} | |
readChunk("\n"); | |
}) | |
//stream.on('data', readChunk); | |
stream.on('finish', ()=>{ readChunk("\n"); after(true); }); | |
stream.on('error', (err)=>{ | |
console.error(`failed to load file '${folder + "/" + filename}`, err); | |
after(false); | |
}) | |
console.log(`Reading table ${folder + "/" + filename}`); | |
//readChunk('hey'); | |
//readTable(stream); | |
//async function readTable(tableStream){ | |
function readChunk(chunk){ | |
// console.log('chunk', chunk.toString()); | |
line += chunk; | |
let nextline = ""; | |
for(let i=n; i<line.length; ++i){ | |
if(line[i] == "\n"){ | |
nextline = line.substring(i+1); | |
line = line.substring(0, i); | |
let row = []; | |
if(i > 0){ | |
row = parseRow(line); | |
if(linenumber == 0){ | |
if(!headers || headers.length == 0){ | |
headers = row.slice(autoid? 1 : 0); | |
} else { | |
fileheaders = row.slice(autoid? 1 : 0); | |
let match = headers.length == fileheaders.length; | |
for(let i=0; match && i<headers.length; ++i){ | |
if(headers[i].trim().toLowerCase() != fileheaders[i].trim().toLowerCase()){ | |
match = false; | |
} | |
} | |
if(!match){ | |
console.warn(`Table '${tablename}' expected headers do not match fileheaders.\n ${headers}\n ${fileheaders}`) | |
} | |
} | |
} else { | |
if(autoid){ | |
let id = parseInt(row[0]); | |
row = row.slice(1); | |
table.ids[id] = id; | |
table.rows[id] = row; | |
currentAutoId = Math.max(currentAutoId, id + 1); | |
} else { | |
if(!table.addRow(row)){ | |
console.warn(`Duplicate row key: ${row[0]}, for row ${JSON.stringify(row)}.`); | |
} | |
} | |
} | |
++linenumber; | |
} | |
line = nextline; | |
i = 0; | |
n = line.length; | |
} | |
} | |
} | |
} | |
return table; | |
function rowToString(row){ | |
let s = ""; | |
for(let i=0; i<row.length; ++i){ | |
let entry = (typeof row[i] == "bigint")? row[i] + "n" : "" + row[i]; | |
let special = entry.indexOf('"') >= 0 || entry.indexOf("\n") >= 0 || entry.indexOf(",") >= 0 || entry.indexOf("'") >= 0; | |
if(columnquoted[i] || special){ | |
try{ | |
entry = JSON.stringify(entry); | |
} catch(e){ | |
console.warn("csvtable internal function rowToString: row entry could not be stringified with 'JSON.stringify'. Falling back on manual escaping.", str); | |
entry = "\"" + entry.replace(/\\/g, '\\\\').replace(/\"/g, '\\"') + "\""; | |
} | |
} | |
s += entry; | |
if(i < row.length - 1) s += ","; | |
} | |
return s; | |
} | |
function parseRow(s){ | |
let options = { | |
autoparse: autoparse, | |
} | |
return CSVParseRow(s, options); | |
} | |
} | |
function CSVParseRow(s, options){ | |
// console.log("parsing line", s); | |
if(!options) options = {}; | |
let row = []; | |
let j = 0; | |
let fullquote = true; | |
let quotechar = null; | |
for(let i=0; i<s.length; ++i){ | |
let c = s[i]; | |
// trim leading whitespace | |
while(i == j && (s[i] == " " || s[i] == "\t")){ | |
++i; ++j; | |
} | |
if(c == "\"" || c == "\'"){ | |
quotechar = c; | |
if(i > j) fullquote = false; | |
while(s[++i] != quotechar){ | |
if(s[i] == "\\") ++i; } | |
} | |
if(c == "," || i == s.length-1){ | |
if(i==s.length-1){ | |
++i; | |
} | |
let entry; | |
// back up to trim whitespace | |
while(s[i-1] == " " || s[i-1] == "\t") --i; | |
if(fullquote && s[i-1] == "\""){ | |
let str = ""; | |
try{ | |
str = s.substring(j, i); | |
entry = JSON.parse(str) | |
} catch(e){ | |
console.warn("csvtable internal function 'CSVParseRow': row entry could not be parsed with 'JSON.parse'. Falling back on manual unescaping.", str); | |
str = s.substring(j+1, i-1); | |
entry = str.replace(/(?:\\([\\\"\']))/g, '$1'); | |
} | |
} else { | |
entry = s.substring(j, i); | |
} | |
if(options.autoparse){ | |
if(entry == "true") entry = true; | |
else if(entry == "false") entry = false; | |
else if(entry == "null") entry = null; | |
else if(entry == "undefined") entry = undefined; | |
else if(entry.match(FloatRegex)) entry = parseFloat(entry); | |
else if(entry.match(BigIntRegex)) entry = BigInt(entry); | |
else if(entry.match(HexRegex)) entry = parseInt(entry, 16); | |
} | |
row.push(entry); | |
fullquote = true; | |
// we backed up to trim whitespace, go forward to comma again. | |
while(i<s.length && (s[i] == " " || s[i] == "\t")) ++i; | |
j = i+1; | |
} | |
if(c == "\n"){ | |
throw new Error("csvtable, internal function 'CSVParseRow' encountered unexpected newline character."); | |
} | |
} | |
// console.log("parsed row", row); | |
return row; | |
} | |
function Test(){ | |
// combine gui and text user interfaces | |
// web command | |
console.log('csv database'); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment