StandardAnalyzerTest.java

package lucene1;

import java.io.StringReader;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;

/**
 * @see    Lucene 2.4.1
 * @since  September 25, 2009
 * @author Pcdinh
 */
public class StandardAnalyzerTest {

    public static void main(String[] args) {
        try {
            Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
            TokenStream ts = analyzer.tokenStream("myfield", new StringReader("some text goes here"));
            Token t = ts.next(new Token());
            while (t != null) {
                System.out.println("Token: " + String.valueOf(t.termBuffer()));
                t = ts.next(t);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}