Viewing Sitecore search analyzer output

sitecore-analyzers.cs

using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Fr;
using Lucene.Net.Analysis.Tokenattributes;
using Sitecore.ContentSearch.LuceneProvider.Analyzers;
using System;
using System.IO;

namespace SitecoreAnalyzers
{
    class Program
    {
        static void Main(string[] args)
        {
            var text = "Did the Quick Brown Fox jump over the Lazy Dog?";
            var text_fr = "Le Fox brune rapide a-t-il sauté sur le chien paresseux?";

            Console.WriteLine("** CaseSensitiveStandardAnalyzer **");
            displayTokens(new CaseSensitiveStandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), text);

            Console.WriteLine("** LowerCaseKeywordAnalyzer **");
            displayTokens(new LowerCaseKeywordAnalyzer(), text);

            Console.WriteLine("** NGramAnalyzer **");
            displayTokens(new NGramAnalyzer(), text);

            Console.WriteLine("** StandardAnalyzerWithStemming **");
            displayTokens(new StandardAnalyzerWithStemming(Lucene.Net.Util.Version.LUCENE_30), text);

            Console.WriteLine("** SynonymAnalyzer - see http://firebreaksice.com/sitecore-synonym-search-with-lucene/ **");
            displayTokens(new SynonymAnalyzer(new XmlSynonymEngine("synonyms.xml")), text);

            Console.WriteLine("** FrenchAnalyzer **");
            displayTokens(new FrenchAnalyzer(Lucene.Net.Util.Version.LUCENE_30), text_fr);

            Console.ReadKey();
        }

        private static void displayTokens(Analyzer analyzer, string text)
        {
            var stream = analyzer.TokenStream("content", new StringReader(text));
            var term = stream.AddAttribute<ITermAttribute>();
            while (stream.IncrementToken())
            {
                Console.Write("'" + term.Term + "', ");
            }
            Console.Write("\n\n");
        }
    }
}