scanner.h

#ifndef _SCHEMESE_SCANNER_H_
#define _SCHEMESE_SCANNER_H_

#include <iostream>
#include <vector>
#include <string>

namespace Schemese {

/*
 * Start compound tokens at end of unicode range
 * for future considirations.
 */
#define TOKENS_START 0x110000

enum SchemeseTokenType {
  LPAREN = '(',
  RPAREN = ')',
  LBRACKET = '{',
  RBRACKET = '}',
  QUOTE = ''',
  BACKQUOTE = '`',
  IDENTIFIER = TOKENS_START,
  BOOLEAN,
  VECTOR,
  BYTE_VECTOR,
  INTEGER,
  REAL,
  CHARACTER,
  STRING
};

struct Token {
  SchemeseTokenType type;
  std::string text;
};

class Scanner {
  public:
    Scanner(std::istream input);
    Token scan();
    Token peek();
    void put_back(Token& token);
  private:
    std::vector<Token> token_buffer;
};

};

#endif