Tokenizer ranges

ranges.js

class Tokenizer {
  constructor(source) {
    this.source = source;
    this.result = null; // result is a token, i.e. a stack of tokens
    this.pathRangesByToken = new WeakMap();
    this.prevTokensByToken = new WeakMap();
  }

  ownTokensFor(path) {
    const { prevTokensByToken, pathRangesByToken } = this;
    const [startNodeToken, endNodeToken] = path.outerRange;
    const cstTokens = [];

    for (
      let token = prevTokensByToken.get(endNodeToken);
      token !== startNodeToken;
      token = prevTokensByToken.get(token)
    ) {
      if (token.type === 'EndNode') {
        const range = pathRangesByToken.get(token);
        token = range[0];
      } else {
        cstTokens.push(createToken(token.type, token.value));
      }
    }

    cstTokens.reverse();

    return cstTokens;
  }
}