tokenizer函数演示

gistfile1.c

typedef enum {
  LPARENTHESIS,
  RPARENTHESIS,
  SYMBOL_TOKEN,
  EOF_TOKEN,
};

void consume(lexer_t lexer)
{
  if (lexer->c != '\0') {
    lexer->p++;
    lexer->c = lexer->string[lexer->p];
  }
}

token_t tokenizer(lexer_t lexer)
{
  while (lexer->c != '\0') {
    switch (lexer->c) {
      case '(': consume(lexer); return make_token(LPARENTHESIS, "(");
      case ')': consume(lexer); return make_token(RPARENTHESIS, ")");
      case ' ': consume(lexer); continue;
      default: {
        int start = lexer->p;
        while (lexer->c != ' ' && lexer->c != '\0' &&
               lexer->c != '(' && lexer->c != ')')
          consume(lexer);
        return make_token(SYMBOL_TOKEN, strndup(lexer->string + start, lexer->p - start));
      }
    }
  }
  return make_token(EOF_TOKEN, "#<EOF>");
}

tokenizer.c

token_t make_token(TokenType type, char *text)
{
  token_t token = malloc(sizeof(struct token_t));
  token->type = type;
  token->text = text;
  return token;
}

token_t tokenizer(char *string)
{
  switch (*string) {
    case '(': return make_token(LPARENTHESIS, "(");
    case ')': return make_token(RPARENTHESIS, ")");
    case ' ': break;
    default: {
      int i = 0;
      while (string[i] != ' ' && string[i] != '\0' &&
             string[i] != '(' && string[i] != ')')
        i++;
      return make_token(SYMBOL_TOKEN, strndup(string, i));
    }
  }
}