gistfile1.rs

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    StringLiteral(String),

    ClosingBrace,
    ClosingParenthesis,
    OpeningBrace,
    OpeningParenthesis,
    Period,
    Semicolon,

    Proc,
    Use,

    Identifier(String),
}

pub fn tokens(text: &str) -> Option<Vec<Token>> {
    let mut result = vec![];
    let mut chars: Vec<char> = text.chars().collect();
    while !chars.is_empty() {
        match chars.remove(0) {
            c if is_space(c) => continue,

            c if c == '"' => {
                let mut value = "".to_string();
                while !chars.is_empty() && chars[0] != '"' {
                    value.push(chars.remove(0));
                }
                if chars.is_empty() {
                    return None;
                }
                chars.remove(0);
                result.push(Token::StringLiteral(value));
            },

            '}' => result.push(Token::ClosingBrace),
            ')' => result.push(Token::ClosingParenthesis),
            '{' => result.push(Token::OpeningBrace),
            '(' => result.push(Token::OpeningParenthesis),
            '.' => result.push(Token::Period),
            ';' => result.push(Token::Semicolon),

            c if is_identifier_head(c) => {
                let mut name = c.to_string();
                while !chars.is_empty() && is_identifier_tail(chars[0]) {
                    name.push(chars.remove(0));
                }
                result.push(match &name[..] {
                    "proc" => Token::Proc,
                    "use" => Token::Use,
                    _ => Token::Identifier(name),
                });
            },

            _ => return None,
        }
    }
    Some(result)
}

fn is_space(c: char) -> bool {
    c == ' ' || c == '\n'
}

fn is_identifier_head(c: char) -> bool {
    'a' <= c && c <= 'z'
}

fn is_identifier_tail(c: char) -> bool {
    is_identifier_head(c)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn empty() {
        assert_eq!(tokens(""), Some(vec![]));
    }

    #[test]
    fn identifier_at_begin_and_end() {
        assert_eq!(tokens("foo bar"), Some(vec![
            Token::Identifier("foo".to_string()),
            Token::Identifier("bar".to_string()),
        ]));
    }

    #[test]
    fn string_literal() {
        assert_eq!(tokens("\"foo\""), Some(vec![Token::StringLiteral("foo".to_string())]));

        assert_eq!(tokens("\"foo\"\"bar\""), Some(vec![
            Token::StringLiteral("foo".to_string()),
            Token::StringLiteral("bar".to_string()),
        ]));
    }

    #[test]
    fn non_terminating_string_literal() {
        assert_eq!(tokens("\"foo"), None);
    }

    #[test]
    fn hello_world() {
        let program = r#"
            use std.io;

            proc main() {
                io.writeln("Hello, world!");
            }
        "#;
        assert_eq!(tokens(program), Some(vec![
            Token::Use,
            Token::Identifier("std".to_string()),
            Token::Period,
            Token::Identifier("io".to_string()),
            Token::Semicolon,
            Token::Proc,
            Token::Identifier("main".to_string()),
            Token::OpeningParenthesis,
            Token::ClosingParenthesis,
            Token::OpeningBrace,
            Token::Identifier("io".to_string()),
            Token::Period,
            Token::Identifier("writeln".to_string()),
            Token::OpeningParenthesis,
            Token::StringLiteral("Hello, world!".to_string()),
            Token::ClosingParenthesis,
            Token::Semicolon,
            Token::ClosingBrace,
        ]));
    }
}