en.csharp.rl

namespace Gherkin.Lexer
{
	using System.Text;
	using System.Text.RegularExpressions;
	using System.IO;
	using System.Collections.Generic;
	
	public class En : ILexer {
		%%{
			machine lexer;
			alphtype char;

			action begin_content {
				contentStart = p;
				currentLine = lineNumber;
			}

			action start_pystring {
				currentLine = lineNumber;
				startCol = p - lastNewline;
			}
		    
			action begin_pystring_content {
				contentStart = p;
			}

			action store_pystring_content {
				string con = Unindent(startCol, new Regex("(\\r?\\n)?( )*\\Z").Replace(Substring(data, contentStart, nextKeywordStart-1), "", 1));
				listener.PythonString(con, currentLine);
			}

			action store_feature_content {
				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart).Trim());
				listener.Feature(keyword, con, currentLine);
				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
				nextKeywordStart = -1;
			}

			action store_background_content {
				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
				listener.Background(keyword, con, currentLine);
				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
				nextKeywordStart = -1;
			}

			action store_scenario_content {
				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
				listener.Scenario(keyword, con, currentLine);
				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
				nextKeywordStart = -1;
			}

			action store_scenario_outline_content {
				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
				listener.ScenarioOutline(keyword, con, currentLine);
				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
				nextKeywordStart = -1;
			}

			action store_examples_content {
				string con = MultilineStrip(KeywordContent(data, p, eof, nextKeywordStart, contentStart));
				listener.Examples(keyword, con, currentLine);
				if(nextKeywordStart != -1) p = nextKeywordStart - 1;
				nextKeywordStart = -1;
			}

			action store_step_content {
				listener.Step(keyword, Substring(data, contentStart, p).Trim(), currentLine);
			}

			action store_comment_content {
				listener.Comment(Substring(data, contentStart, p).Trim(), lineNumber);
				keywordStart = -1;
			}

			action store_tag_content {
				listener.Tag(Substring(data, contentStart, p).Trim(), currentLine);
				keywordStart = -1;
			}

			action inc_line_number {
				lineNumber++;
			}

			action last_newline {
				lastNewline = p + 1;
			}

			action start_keyword {
				if(keywordStart == -1) keywordStart = p;
			}

			action end_keyword {
				keyword = new Regex(":$").Replace(Substring(data, keywordStart, p), "", 1);
				keywordStart = -1;
			}

			action next_keyword_start {
				nextKeywordStart = p;
			}

			action start_table {
				p = p - 1;
				rows = new List<IList<string>>();
				currentLine = lineNumber;
			}

			action start_row {
				currentRow = new List<string>();
			}

			action begin_cell_content {
				contentStart = p;
			}

			action store_cell_content {
				currentRow.Add(Substring(data, contentStart, p).Trim());
			}

			action store_row {
				rows.Add(currentRow);
			}

			action store_table {
				if(rows.Count > 0) {
					listener.Table(rows, currentLine);
				}
			}

			action end_feature {
				if(cs < lexer_first_final) {
					string content = CurrentLineContent(data, lastNewline);
					throw new LexingException("Lexing error on line " + lineNumber);
				}
			}

			include lexer_common "lexer_common.en.rl"; 
		}%%

		private readonly IListener listener;

		public En(IListener listener) {
			this.listener = listener;
		}

		%% write data noerror;

		public void Scan(TextReader inputSequence) {
			string input = inputSequence.ReadToEnd() + "\n%_FEATURE_END_%";
			char[] data = input.ToCharArray();
			int cs, p = 0, pe = data.Length;
			int eof = pe;

			int lineNumber = 1;
			int lastNewline = 0;

			int contentStart = -1;
			int currentLine = -1;
			int startCol = -1;
			int nextKeywordStart = -1;
			int keywordStart = -1;
			string keyword = null;
			IList<IList<string>> rows = null;
			IList<string> currentRow = null;

			%% write init;
			%% write exec;
		}

		private string KeywordContent(char[] data, int p, int eof, int nextKeywordStart, int contentStart) {
			int endPoint = (nextKeywordStart == -1 || (p == eof)) ? p : nextKeywordStart;
			return Substring(data, contentStart, endPoint);
		}

		private string MultilineStrip(string text) {
			var result = new StringBuilder();
			foreach (var s in text.Split(new [] {'\n'})) {
				result.AppendLine(s.Trim());
			}
			return result.ToString().Trim();
		}

		private string Unindent(int startCol, string text) {
			return new Regex("^ {0," + startCol + "}", RegexOptions.Multiline).Replace(text, "");
		}

		private string CurrentLineContent(char[] data, int lastNewline) {
			return Substring(data, lastNewline, data.Length).Trim();
		}

		private string Substring(char[] data, int start, int end) {
			return new string(data, start, end-start);
		}
	}
}

lexer_common.en.rl

%%{
  machine lexer_common;

  # Language specific
  I18N_Feature = (("Feature") ':') >start_keyword %end_keyword; 
  I18N_Background = (("Background") ':') >start_keyword %end_keyword;
  I18N_ScenarioOutline = (("Scenario Outline") ':') >start_keyword %end_keyword;
  I18N_Scenario = (("Scenario") ':') >start_keyword %end_keyword;
  I18N_Step = ("* " | "Given " | "When " | "Then " | "And " | "But ") >start_keyword %end_keyword;
  I18N_Examples = (("Examples" | "Scenarios") ':') >start_keyword %end_keyword;

  EOF = '%_FEATURE_END_%'; # Explicit EOF added before scanning begins 
  EOL = ('\r'? '\n') @inc_line_number @last_newline;

  FeatureHeadingEnd = EOL+ space* (I18N_Background | I18N_Scenario | I18N_ScenarioOutline | '@' | '#' | EOF) >next_keyword_start;
  ScenarioHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_ScenarioOutline | I18N_Step | '@' | '#' | EOF ) >next_keyword_start;
  BackgroundHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_ScenarioOutline | I18N_Step | '@' | '#'| EOF ) >next_keyword_start;
  ScenarioOutlineHeadingEnd = EOL+ space* ( I18N_Scenario | I18N_Step | '@' | '#' | EOF ) >next_keyword_start;
  ExamplesHeadingEnd = EOL+ space* '|' >next_keyword_start;
 
  FeatureHeading = space* I18N_Feature %begin_content ^FeatureHeadingEnd* :>> FeatureHeadingEnd @store_feature_content;
  BackgroundHeading = space* I18N_Background %begin_content ^BackgroundHeadingEnd* :>> BackgroundHeadingEnd @store_background_content;
  ScenarioHeading = space* I18N_Scenario %begin_content ^ScenarioHeadingEnd* :>> ScenarioHeadingEnd @store_scenario_content;
  ScenarioOutlineHeading = space* I18N_ScenarioOutline %begin_content ^ScenarioOutlineHeadingEnd* :>> ScenarioOutlineHeadingEnd @store_scenario_outline_content;
  ExamplesHeading = space* I18N_Examples %begin_content ^ExamplesHeadingEnd* :>> ExamplesHeadingEnd @store_examples_content;

  Step = space* I18N_Step %begin_content ^EOL+ %store_step_content EOL+;
  Comment = space* '#' >begin_content ^EOL* %store_comment_content EOL+;
  
  Tag = ( '@' [^@\r\n\t ]+ >begin_content ) %store_tag_content;
  Tags = space* (Tag space*)+ EOL+;
  
  StartTable = space* '|' >start_table;
  EndTable = EOL space* ^('|') >next_keyword_start;    
  Cell = '|' (any - '|')* >begin_cell_content %store_cell_content;
  Row = space* Cell* >start_row '|' :>> (space* EOL+ space*) %store_row;
  Table = StartTable :>> Row+ %store_table <: EndTable?;
  
  StartPyString = '"""' >start_pystring space* :>> EOL; 
  EndPyString = (space* '"""') >next_keyword_start; 
  PyString = space* StartPyString %begin_pystring_content (^EOL | EOL)* :>> EndPyString %store_pystring_content space* EOL+;
  
  Tokens = (space | EOL)* (Tags | Comment | FeatureHeading | BackgroundHeading | ScenarioHeading | ScenarioOutlineHeading | ExamplesHeading | Step | Table | PyString)* (space | EOL)* EOF;

  main := Tokens %end_feature @!end_feature;
}%%