tokenizer-2.js

const example = `22 + 23`;

const isWhitespace = character => /\s/.test(character);
const isNumber = character => /[0-9]/.test(character);
const isOperator = character => /[\+\-\*\/]/.test(character);

const tokenize = (input) => {
  let cursor = 0;
  const tokens = [];
  
  while (cursor < input.length) {
    const character = input[cursor];
    
    if (isNumber(character)) {
      let value = character;
      
      while (isNumber(input[++cursor])) {
        value += input[cursor];
      }
      
      tokens.push({
        type: 'Number',
        value,
      });
      
      continue;
    }
    
    if (isOperator(character)) {
      tokens.push({
        type: 'Operator',
        value: character,
      });
      
      cursor++;
      continue;
    }
    
    if (isWhitespace(character)) {      
      cursor++;
      continue;
    }
    
    throw new TypeError(`Unknown character type: "${character}".`);
  }
  
  return tokens;
};

console.log(tokenize(example));

tokenizer.js

const example = `2 + 2`;

const isWhitespace = character => /\s/.test(character);
const isNumber = character => /[0-9]/.test(character);
const isOperator = character => /[\+\-\*\/]/.test(character);

const tokenize = (input) => {
  let cursor = 0;
  const tokens = [];
  
  while (cursor < input.length) {
    const character = input[cursor];
    
    if (isNumber(character)) {
      tokens.push({
        type: 'Number',
        value: +character,
      });
      
      cursor++;
      continue;
    }
    
    if (isOperator(character)) {
      tokens.push({
        type: 'Operator',
        value: character,
      });
      
      cursor++;
      continue;
    }
    
    if (isWhitespace(character)) {      
      cursor++;
      continue;
    }
    
    throw new TypeError(`Unknown character type: "${character}".`);
  }
  
  return tokens;
};

console.log(tokenize(example));