log_splitter.rb

                                                                                                                                                                                                                                                     
# Split a single logfile into multiple multiple logfiles based on PID information                                                                                                                                                                     
# provided in the log lines.                                                                                                                                                                                                                          
#                                                                                                                                                                                                                                                     
# Usage:                                                                                                                                                                                                                                              
#   ruby log_splitter.rb <LOGFILE>                                                                                                                                                                                                                    
#                                                                                                                                                                                                                                                     
# Example:                                                                                                                                                                                                                                            
# $> ruby log_splitter.rb development.log                                                                                                                                                                                                             
# Complete. File list:                                                                                                                                                                                                                                
# split_log_development.catchall.log   # <- log lines without a pid                                                                                                                                                                                   
# split_log_development.14634.log                                                                                                                                                                                                                     
# split_log_development.31277.log                                                                                                                                                                                                                     
# split_log_development.31279.log                                                                                                                                                                                                                     
#                                                                                                                                                                                                                                                     
class LogSplitter
  attr_reader :basename, :files_by_pid, :catchall, :filename

  # All generated files should contain this prefix to allow for easy glob file matches                                                                                                                                                                
  PREFIX      = 'split_log_'
  # The logfile pattern that captures the pid, this will be log format specific. Using Logging::Logger                                                                                                                                                
  # we define: Logging::Layouts::Pattern(:pattern => "%d pid:%p [%c:%l] %m [%F:%L]\n")                                                                                                                                                                
  PID_PATTERN = /pid:(\d+)/

  # Currently expects the filename to exist in the local directory for ease of implementation                                                                                                                                                         
  def initialize(filename_to_split)
    @filename = filename_to_split
    filename =~ /(.*)\.log/
    @basename = $1
    @files_by_pid =
      Hash.new {|hash, pid_key| hash[pid_key] = File.open(PREFIX + basename + ".#{pid_key}.log", 'w') }
    @catchall = File.open(PREFIX + basename + '.catchall.log', 'w')
  end

 def split
    last_pid = nil
    File.open(directory + '/' + filename, 'r') do |f|
      while(line = f.gets)
        outfile =
          if match_data = line.match(PID_PATTERN)
            pid = match_data.captures[0]
            last_pid = pid
            files_by_pid[pid]
          elsif last_pid
            # handle cases where a single log call results in multiple lines of output.                                                                                                                                                               
            files_by_pid[last_pid]
          else
            catchall
          end
        outfile.puts line
      end
    end
  end


  def close_files
    @files_by_pid.values.map(&:close)
    @catchall.close
  end
end

if __FILE__ == $0
  logfile = ARGV[0]
  if logfile.nil?
    puts "Please provide a logfile to split"
    exit(0)
  elsif !File.exist?(logfile)
    puts "the logfile name you provided can not be found"
    exit(0)
  end
  log_splitter = LogSplitter.new(logfile)
  log_splitter.split
  log_splitter.close_files
  puts "Complete. File list:"
  puts log_splitter.catchall.path
  log_splitter.files_by_pid.values.each {|f| puts f.path }
end

I use this script as a pre-process step before feeding log files into the request_log_analyzer (http://github.com/wvanbergen/request-log-analyzer). It helps avoid the problem of multiple processes logging to the same logfile.

Updated script. Now assumes that all lines without a pid are from the same invocation of log output as the last line with a pid. This change is needed to deal with multi-line log output where only the first line gets log metadata (In particular, rails logs logger.info("\n\nProcessing Request..."))