Skip to content

Instantly share code, notes, and snippets.

@conartist6
Created March 20, 2022 21:29
Embed
What would you like to do?
Peekerate json tokenizer
import peekerate from '@iter-tools/peekerate';
export function* tokenize(input) {
const peekr = peekerate(input);
while (!peekr.done) {
const char = peekr.value;
if (char === '"') {
// yields three tokens: " content "
yield* tokenizeString(peekr);
} else if ('{}[]:,'.includes(char)) {
yield t.token(char);
} else {
throw new SyntaxError();
}
peekr.advance();
}
}
function* tokenizeString(peekr) {
let escaped = false;
let literal = '';
peekr.advance(); // "
while(!peekr.done) {
const char = peekr.value;
if (escaped) {
if (escapes.hasOwnProperty(char)) {
literal += escapes[char];
} else if (char === 'u') {
// TODO handle unicode escapes : (
} else {
literal += char;
}
} else if (char === '\\') {
escaped = true;
} else if (char === '"') {
yield t.literal(literal);
yield '"';
return;
} else {
literal += char;
}
peekr.advance();
}
peekr.advance(); // "
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment