gistfile1.txt

import io/Reader
import math
import structs/ArrayList
import text/[Buffer,StringReader]

TokenType: enum {
    None
    Number
    Operator
}

Token: class {
    val: String
    type: TokenType
    init: func {
        type = TokenType None
    }
    clone: func -> This {
        toRet := Token new()
        toRet val = val
        toRet type = type
        toRet
    }

    toString: func -> String {
        "TokenType = %d, value = %s" format(type, val)
    }
}

findNumber: func(reader: Reader) -> Int {
    reader rewind(1)
    beginning := reader mark()
    chr: Char
    while(reader hasNext?()) {
        chr = reader read()
        if(!chr digit?())
            break
    }
    end := reader mark()
    reader reset(beginning)
    (end - beginning) as SizeT
}


getToken: func(reader: Reader) -> Token {
    token := Token new()
    if (!reader hasNext?()) return token
    reader skipWhile(' ')
    //skipMany(reader, |c| c whitespace?()) 
    c := reader read()
    match c {
        case '+' || '-' || '*' || '/' => {
            token val = c toString()
            token type = TokenType Operator
        }
        case  => {
            if (c digit?()) {
                length := findNumber(reader)
                length toString() println()
                s := String new(length)
                reader read(s, 0, length)
                token val = s
                token type = TokenType Number
            }

        }


    }
    token
}
                 
skipMany: func(reader: Reader, f: Func(Char) -> Bool) {
    chr := reader read()
    while(f(chr)) {         
        if(!reader hasNext?())
            return
        chr = reader read()
    }
    reader rewind(1)
}    

tokenize: func(str: String) -> ArrayList<Token> {
    reader := StringReader new(str)
    list := ArrayList<Token> new()
    token := getToken(reader)
    while (token type != TokenType None) {
        list add(token clone())
        token = getToken(reader)
    }
    list
}

main: func {
    str := "1+2"
    tokenize(str) each(|t| t toString() println())    
}