The Parser constructor does all the tokenizing

tokenizer.js

class Parser {
    constructor(text) {
        this.nextTokenIndex = 0;
        this.tokenList = [];
        const reToken = /[0-9]+(\.[0-9]*)?([eE][\+\-]?[0-9]+)?|[A-Za-z_][A-Za-z_0-9]*|\S/g;
        for(;;) {
            const match = reToken.exec(text);
            if (match === null) {
                break;
            }
            this.tokenList.push(new Token(match[0], match.index));
        }
    }

    // ...
}