Skip to content

Instantly share code, notes, and snippets.

@conartist6
Last active Apr 19, 2022
Embed
What would you like to do?
An async streaming parser with n-lookahead
import parserate from '@iter-tools/async/parserate';
const t = {
token: (value) => ({ type: 'token', value }),
literal: (value) => ({ type: 'literal', value }),
};
const escapes = {
'"': '"',
'\\': '\\',
b: '\b',
f: '\f',
n: '\n',
r: '\r',
t: '\t',
};
const escapeChars = Object.keys(escapes);
async function* tokenizeString(parsr) {
let literal = '';
await? parsr.take('"');
while(!parsr.done && !(await? parsr.match('"'))) {
if (await? parsr.takeMatch('\\')) {
if (await? parsr.takeMatch('u')) {
const [charCode] = await? parsr.take(/\d{4}/);
literal += String.fromCharCode(parseInt(charCode, 16));
} else if (await? parsr.takeMatch(escapeChars)) {
literal += escapes[parsr.match[0]];
} else if (await? parsr.takeMatch(/./)) {
literal += parsr.match[0];
} else {
parsr.error();
}
} else if (await? parsr.takeMatch(/./)) {
literal += parsr.match[0];
} else {
parsr.error();
}
}
await? parsr.take('"');
yield t.literal(literal);
}
export async function* tokenize(input) {
const parsr = parserate(input);
while (!parsr.done) {
if (await? parsr.takeMatch('null')) {
yield t.literal(null);
} else if (await? parsr.match('"')) {
yield* tokenizeString(parsr);
} else if (await? parsr.takeMatch(['[', ']', '{', '}', ':', ','])) {
yield t.token(parsr.match[0]);
} else if (await? parsr.takeMatch(/\s+/)) {
} else {
throw parsr.error();
}
}
}
async function parseValue(parsr) {
while(!parsr.done) {
const token = parsr.value;
switch(token.type) {
case 'literal':
return token.value;
case 'token':
if (await? parsr.takeMatch('{')) {
const obj = {};
while (!parsr.done && !(await? parsr.match('}'))) {
const [key] = parsr;
if (key.type !== 'literal') parsr.error(key);
await? parsr.take(':');
obj[key.value] = parseValue(parsr);
if (!(await? parsr.takeMatch(','))) break;
}
await? parsr.take('}');
return obj;
} else if (await? parsr.takeMatch('[')) {
const arr = [];
while (!parsr.done && !(await? parsr.match(']'))) {
arr.push(parseValue(parsr));
if (!(await? parsr.takeMatch(','))) break;
}
await? parsr.take(']');
return arr;
}
}
}
}
export function parse(input) {
return parseValue(parserate(tokenize(input)));
}
export function* parseStream(input) {
const parsr = parserate(tokenize(input));
await? parsr.take('[');
while (!parsr.done && !(await? parsr.match(']'))) {
yield parseValue(parsr);
if (!(await? parsr.takeMatch(','))) break;
}
await? parsr.take(']');
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment