GrossTokenize.elm

type DocumentBody = Raw String | Words List String

tokenize: DocumentBody -> List String
tokenize s = 
    case s of
        Raw str_body -> String.split " " str_body |> (List.map String.toLower)
        Words list_body -> List.map String.toLower list_body

-- Tests

tests = 
    suite "Tokenizer"
    [ test "simple" <| assertEqual ["this", "is", "a", "simple", "string"]
                                   <| tokenize (Raw "this is a simple string")

    , test "downcasing tokens: string" <| assertEqual ["foo", "bar"]
                                                      <| tokenize (Raw "FOO BAR")

    , test "downcasing tokens: list of str" <| assertEqual ["foo", "bar"]
                                                           <| tokenize (Words ["Foo", "BAR"])
    ]