ex1.ts

type TokenType = 'ASSIGN' | 'NAME' | 'NUM' | 'OP';
type TokenLiteral = string;
type Token = [TokenType, TokenLiteral];
type Tokens = ReadonlyArray<Token>;

function assert<T extends Tokens>(actual: T, expected: T): void | never {
  const stringActual = JSON.stringify(actual);
  const stringExpected = JSON.stringify(expected);

  if (stringActual !== stringExpected) {
    throw new Error(`Expected ${stringActual} to equal ${stringExpected}`);
  }
}

function isIdentifier(char: string): boolean {
  return char >= 'a' && char <= 'z';
}

function isNumber(char: string): boolean {
  return char >= '0' && char <= '9';
}

function tokenize(text: string): Tokens {
  const tokens: Array<Token> = [];
  
  let start = 0;
  let pos = 0;
  while (pos++ < text.length) {
    const c = text[start];
    switch (text[start]) {
      case '+':
        tokens.push(['OP', c])
        break;
      case '=':
        tokens.push(['ASSIGN', c]);
        break;
      case '*':
        tokens.push(['OP', c]);
        break;
      default:
        if (isIdentifier(c)) {
          while (isIdentifier(text[pos++])) {}
          tokens.push(['NAME', text.substring(start, pos - 1)]);
        } else if (isNumber(c)) {
          while (isNumber(text[pos++])) {}
          tokens.push(['NUM', text.substring(start, pos - 1)]);
        }
    }

    start = pos
  }

  return tokens;
}

assert(tokenize("spam = x + 34 * 567"), [['NAME', 'spam'], ['ASSIGN', '='], ['NAME', 'x'], 
['OP', '+'], ['NUM', '34'], ['OP', '*'], ['NUM', '567']])