conartist6

  A range is represented as a [Token, Token] tuple.

  A range can only be iterated backwards, which creates the need to store tokens in an array!

  Each node must have its own tokens array to enable efficient editing.

  A context object often abbreviated `ctx` holds WeakMaps, including `paths` and `previousTokens`.

  Tokens reference their preceding token using the `previousTokens` WeakMap.

conartist6

import { prod, tok, sym, Grammar } from '@/helpers';

new Grammar({
  productions: {
  
    // This version of ImportSpecifier uses helpers to be concise:
    *ImportSpecifier() {
      yield eat(prod`Identifier:imported`);
      yield eatMatch(tok`Keyword:as`, prod`Identifier:local`);
    },

conartist6

const _ = Symbol('private');

class ReadOnlyMap {
  constructor(map) {
    this[_] = map;
  }

  static from(entries) {
    return new ReadOnlyMap(new Map(entries));
  }

conartist6

class Tokenizer {
  constructor(source) {
    this.source = source;
    this.result = null; // result is a token, i.e. a stack of tokens
    this.pathRangesByToken = new WeakMap();
    this.prevTokensByToken = new WeakMap();
  }

  ownTokensFor(path) {
    const { prevTokensByToken, pathRangesByToken } = this;

conartist6

import { Grammar } from '@cst-tokens/grammar';
import { objectEntries } from '@cst-tokens/helpers/iterable';
import { eat, match } from '@cst-tokens/helpers/commands';
import { capture, ref, PN } from '@cst-tokens/helpers/shorthand';

// 2-+2+-2
/*
$capture = '2';
$node = { type: 'Digit', value: $caputure };
$capture = '-';

conartist6

console.time('new Set');
for (let i = 0; i < 1_000_000; i++) {
  new Set();
}
console.timeEnd('new Set');

console.time('new Map');
for (let i = 0; i < 1_000_000; i++) {
  new Map();
}

conartist6

function /*1*/ foo /*2*/ (/*3*/) /*4*/ {/*5*/} /*6*/

conartist6

export class Coroutine {
  constructor(generator) {
    this.generator = generator[Symbol.iterator]();
    this.current = this.generator?.next();
  }

  get value() {
    return this.current.value;
  }

conartist6

Input:

if (foo) {
  // comment
}

Traversal log

conartist6

/*
Lamps often have on-off switches that twist clockwise, clicking as they do.
Some such switches have four states, even though the lamp has only two (on, and off).
On these lamps clicking the switch will only cause an on/off transition every other click.
Just looking at the lamp you'll know if it's on, but not if the next click will turn it off.
To reliably interact with such a lamp you will use an algorithm, even in real life.
I use the term strategy to refer to an algorithm expressed with generators like this --
the strategy pattern is useful to abstract an algorithm from the representation of the data.
*/