A way to pipe in rails logs and split along the double newline

driver.sh

cat sample_rails.log | ruby rails_multiline_log_splitter.rb > output

# ["Started GET \"/\" for 98.113.86.74 at Tue Sep 06 09:44:35 -0400 2011","Processing by LinksController#index as HTML","Completed 401 Unauthorized in 0ms"]
# ["Started GET \"/\" for 98.113.86.74 at Tue Sep 06 09:44:36 -0400 2011","Processing by LinksController#index as HTML","Rendered links/index.html.erb within layouts/main (227.1ms)","Completed 200 OK in 339ms (Views: 228.3ms | ActiveRecord: 21.0ms)"]
# ["Started GET \"/links/11\" for 98.113.86.74 at Tue Sep 06 09:44:52 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"11\"}","Rendered links/show.html.erb within layouts/main (190.2ms)","Completed 200 OK in 194ms (Views: 191.1ms | ActiveRecord: 1.2ms)"]
# ["Started GET \"/links/search?utf8=%E2%9C%93&q=ruby+on+rails&commit=Search\" for 98.113.86.74 at Tue Sep 06 09:45:12 -0400 2011","Processing by LinksController#search as HTML","Parameters: {\"commit\"=>\"Search\", \"utf8\"=>\"✓\", \"q\"=>\"ruby on rails\"}","Completed 401 Unauthorized in 0ms"]
# ["Started GET \"/links/search?utf8=%E2%9C%93&q=ruby+on+rails&commit=Search\" for 98.113.86.74 at Tue Sep 06 09:45:12 -0400 2011","Processing by LinksController#search as HTML","Parameters: {\"commit\"=>\"Search\", \"utf8\"=>\"✓\", \"q\"=>\"ruby on rails\"}","Rendered links/index.html.erb within layouts/main (78.3ms)","Completed 200 OK in 218ms (Views: 79.1ms | ActiveRecord: 10.9ms)"]
# ["Started GET \"/links/137\" for 98.113.86.74 at Tue Sep 06 09:45:28 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"137\"}","Completed 401 Unauthorized in 0ms"]
# ["Started GET \"/links/137\" for 98.113.86.74 at Tue Sep 06 09:45:28 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"137\"}","Rendered links/show.html.erb within layouts/main (35.4ms)","Completed 200 OK in 39ms (Views: 36.4ms | ActiveRecord: 1.1ms)"]
# ["Started GET \"/links/138\" for 98.113.86.74 at Tue Sep 06 09:45:33 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"138\"}","Completed 401 Unauthorized in 0ms"]
# ["Started GET \"/links/138\" for 98.113.86.74 at Tue Sep 06 09:45:33 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"138\"}","Rendered links/show.html.erb within layouts/main (132.9ms)","Completed 200 OK in 136ms (Views: 133.8ms | ActiveRecord: 0.9ms)"]
# ["Started GET \"/links/139\" for 98.113.86.74 at Tue Sep 06 09:45:37 -0400 2011","Processing by LinksController#show as HTML","Parameters: {\"id\"=>\"139\"}","Rendered links/show.html.erb within layouts/main (10.3ms)","Completed 200 OK in 13ms (Views: 11.0ms | ActiveRecord: 0.7ms)","\n","This is a test, only one newline","I just wanted to show some different thing","this s only one"]
# ["whoa i'm a new entry","bring it"]

rails_multiline_log_splitter.rb

require 'rubygems'
require 'yajl/json_gem'

# Simple flush to json, can add timestamp by parsing it.
def flush(buffer)
  puts buffer.to_json
end

# Initialize 
buffer = []
previous_line_matched = false
pattern_matched_count = 0 
current_pattern = 0
split_buffer = []
buffer_last_seen_at = 0


# List of patterns to match
# here we have two consecutive lines with only the newline
patterns = [
  /^\n$/,
  /^\n$/
]

STDIN.each_line do |line|

  # Perform match and increment.
  if matched = (line =~ patterns[current_pattern]) 
    pattern_matched_count += 1 
    current_pattern += 1
  end

  # Flush out if you reached patterns.length or the EOF
  if pattern_matched_count == patterns.length || STDIN.eof
    buffer << line.strip if STDIN.eof # add the last line if it's EOF.
    flush(buffer)
    # Reset 
    previous_line_matched = false
    pattern_matched_count = 0 
    current_pattern = 0
    split_buffer = []
    buffer_last_seen_at = 0
    buffer = []
    next
  elsif matched
    # we need to keep track of where we were when we found the first
    # match.  This is b/c we can get through a bunch of matches but
    # then fail on the last pattern.  We need to be able to keep track
    # so we can inject the patterns matched (wich are part of the log entry)
    # back in
    buffer_last_seen_at = buffer.length if previous_line_matched == false
    split_buffer << line
    previous_line_matched = true
  else
    # We matched a few already, but then didn't get to the finish line
    # we need to add the matched lines back in b/c they weren't really 
    # part of a split
    if pattern_matched_count > 0
      buffer.insert(buffer_last_seen_at, *split_buffer)
    end
    
    # We didn't match earlier, and we're not flushing
    # so we should reset b/c there's no consecutive matches
    # Reset 
    previous_line_matched = false
    pattern_matched_count = 0 
    current_pattern = 0
    split_buffer = []
    buffer_last_seen_at = 0
    
    # buffer up the log entry clean up newlines
    buffer << line.strip
  end
  
end

sample_rails.log

Started GET "/" for 98.113.86.74 at Tue Sep 06 09:44:35 -0400 2011
  Processing by LinksController#index as HTML
Completed 401 Unauthorized in 0ms


Started GET "/" for 98.113.86.74 at Tue Sep 06 09:44:36 -0400 2011
  Processing by LinksController#index as HTML
Rendered links/index.html.erb within layouts/main (227.1ms)
Completed 200 OK in 339ms (Views: 228.3ms | ActiveRecord: 21.0ms)


Started GET "/links/11" for 98.113.86.74 at Tue Sep 06 09:44:52 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"11"}
Rendered links/show.html.erb within layouts/main (190.2ms)
Completed 200 OK in 194ms (Views: 191.1ms | ActiveRecord: 1.2ms)


Started GET "/links/search?utf8=%E2%9C%93&q=ruby+on+rails&commit=Search" for 98.113.86.74 at Tue Sep 06 09:45:12 -0400 2011
  Processing by LinksController#search as HTML
  Parameters: {"commit"=>"Search", "utf8"=>"✓", "q"=>"ruby on rails"}
Completed 401 Unauthorized in 0ms


Started GET "/links/search?utf8=%E2%9C%93&q=ruby+on+rails&commit=Search" for 98.113.86.74 at Tue Sep 06 09:45:12 -0400 2011
  Processing by LinksController#search as HTML
  Parameters: {"commit"=>"Search", "utf8"=>"✓", "q"=>"ruby on rails"}
Rendered links/index.html.erb within layouts/main (78.3ms)
Completed 200 OK in 218ms (Views: 79.1ms | ActiveRecord: 10.9ms)


Started GET "/links/137" for 98.113.86.74 at Tue Sep 06 09:45:28 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"137"}
Completed 401 Unauthorized in 0ms


Started GET "/links/137" for 98.113.86.74 at Tue Sep 06 09:45:28 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"137"}
Rendered links/show.html.erb within layouts/main (35.4ms)
Completed 200 OK in 39ms (Views: 36.4ms | ActiveRecord: 1.1ms)


Started GET "/links/138" for 98.113.86.74 at Tue Sep 06 09:45:33 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"138"}
Completed 401 Unauthorized in 0ms


Started GET "/links/138" for 98.113.86.74 at Tue Sep 06 09:45:33 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"138"}
Rendered links/show.html.erb within layouts/main (132.9ms)
Completed 200 OK in 136ms (Views: 133.8ms | ActiveRecord: 0.9ms)


Started GET "/links/139" for 98.113.86.74 at Tue Sep 06 09:45:37 -0400 2011
  Processing by LinksController#show as HTML
  Parameters: {"id"=>"139"}
Rendered links/show.html.erb within layouts/main (10.3ms)
Completed 200 OK in 13ms (Views: 11.0ms | ActiveRecord: 0.7ms)

This is a test, only one newline
I just wanted to show some different thing
this s only one 


whoa i'm a new entry
bring it