tokenizer.rs

use std::os;

fn tokenize(raw: &str) -> ~[&str] {
    let replaced: ~str = raw.replace("(", " ( ").replace(")", " ) ");
    replaced.split_iter(' ').collect()
}

fn main() {
    printfln!("%?", tokenize(os::args()[1]));
}

working_tokenizer.rs

use std::os;
 
fn tokenize(raw: &str) -> ~[~str] {
    let replaced: ~str = raw.replace("(", " ( ").replace(")", " ) ");
    replaced.split_iter(' ').transform(|token| token.to_owned()).collect()
}
 
fn main() {
    printfln!("%?", tokenize(os::args()[1]));
}