n-gram tokenizer lucene 6

twofivegram.java

ArrayList<String> grams = new ArrayList<>();
		
FileReader fr = new FileReader(args[0]);

System.out.println("loading content file: " + args[0]);

BufferedReader br = new BufferedReader(fr);



NGramTokenizer nGramTokenizer = new NGramTokenizer(2,5);
nGramTokenizer.setReader(br);
CharTermAttribute charTermAttribute = nGramTokenizer.addAttribute(CharTermAttribute.class);
nGramTokenizer.reset();

while(nGramTokenizer.incrementToken()) {
	grams.add(charTermAttribute.toString());
}

nGramTokenizer.end();
nGramTokenizer.close();