test for OmegaT tokenizers

result.txt

org.omegat.tokenizer.LuceneEnglishTokenizer
--------------------
0. "fooled"
1. "by"
2. "a"
3. "smile"

org.omegat.tokenizer.LuceneJapaneseTokenizer
--------------------
0. "fooled"
1. " "
2. "by"
3. " "
4. "a"
5. " "
6. "smile"

org.omegat.tokenizer.TinySegmenterJapaneseTokenizer
--------------------
0. "fooled"
1. " "
2. "by"
3. " "
4. "a"
5. " "
6. "smile"
スクリプトの戻り値
[Ljava.lang.String;@7cecd10e

testTokenizer.groovy

def text = "fooled by a smile"

def dump = { tokenizer ->
    console.println "\n${tokenizer.class.name}\n${'-'*20}"
    def tokenList = tokenizer.tokenizeWordsForSpelling(text)
    def words = org.omegat.util.Token.getTextsFromString(tokenList, text)
    words.eachWithIndex {word, index ->
        console.println "$index. \"$word\""
    }
}

dump new org.omegat.tokenizer.LuceneEnglishTokenizer()
dump new org.omegat.tokenizer.LuceneJapaneseTokenizer()
dump new org.omegat.tokenizer.TinySegmenterJapaneseTokenizer()