select_rails_log.rbでGroongaで出力したのを検索する https://gist.github.com/arika/61a79110d4677683d05989e9e2ed2d66

groonga_rails_log.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

require 'rroonga'
require_relative 'select_rails_log' if $0 == __FILE__

class GroongaIndex
  REFERENCE_COLUMNS = {
    http_status: 'HttpStatuses',
    http_method: 'HttpMethods',
    controller: 'Controllers',
    action: 'Actions',
  }.freeze

  VALUE_COLUMNS = %i[
    request_id
    pid
    begin_time
    end_time
    duration
    path
    parameters
    client
  ].freeze

  def initialize(path, create: false)
    @db_path = "#{path}/db"
    open_database(create: create)
  end

  def store(data)
    id = data[:id]
    return false if requests.key?(id)

    record = store_request(id, data)
    store_logs(record, data)
    true
  end

  def[](name)
    Groonga[name]
  end

  def select(selector)
    found = false

    selector.run(self) do |record|
      yield(to_data(record))
      found = true
    end

    found
  end

  def requests
    Groonga['Requests']
  end

  def logs
    Groonga['Logs']
  end

  def severities
    Groonga['Severities']
  end

  private

  def to_data(record)
    data = {}

    VALUE_COLUMNS.each do |key|
      data[key] = record[key]
    end

    REFERENCE_COLUMNS.each_key do |key|
      data[key] = record[key].key
    end

    data[:logs] = record[:logs].map do |log|
      {
        time: log[:time],
        interval: log[:interval],
        severity: log[:severity][:_key],
        message: log[:message],
      }
    end

    data
  end

  def store_request(id, data)
    attrs = {}
    REFERENCE_COLUMNS.each do |key, table|
      value = data[key]
      Groonga[table].add(value)
      attrs[key] = value
    end

    VALUE_COLUMNS.each do |key|
      attrs[key] = data[key]
    end

    requests.add(id, attrs)
  end

  def store_logs(record, data)
    record[:logs] = data[:logs].map do |log|
      severities.add(log[:severity])
      logs.add(log)
    end
  end

  def open_database(create:)
    begin
      Groonga::Database.open(@db_path)
      return
    rescue Groonga::NoSuchFileOrDirectory
      raise unless create
    end

    create_database
  end

  def create_database
    Groonga::Database.create(path: @db_path)

    create_reference_tables
    create_logs_table
    create_requests_table
    create_terms_table
    create_search_index
  end

  def create_reference_tables
    %w[HttpMethods Controllers Actions Severities].each do |name|
      Groonga::Schema.create_table(name, type: :hash)
    end
    Groonga::Schema.create_table('HttpStatuses', type: :patricia_trie)
  end

  def create_requests_table
    Groonga::Schema.create_table('Requests', type: :hash) do |t|
      t.reference('http_status', 'HttpStatuses')
      t.reference('http_method')
      t.reference('controller')
      t.reference('action')
      t.reference('logs', 'Logs', type: :vector)
      t.short_text('request_id')
      t.time('begin_time')
      t.time('end_time')
      t.unsigned_integer32('pid')
      t.short_text('parameters')
      t.short_text('path')
      t.short_text('client')
      t.unsigned_integer32('duration')
    end
  end

  def create_logs_table
    Groonga::Schema.create_table('Logs', type: :array) do |t|
      t.reference('severity', 'Severities')
      t.time('time')
      t.float('interval')
      t.short_text('message')
    end
  end

  def create_terms_table
    Groonga::Schema.create_table(
      'Terms',
      type: :patricia_trie,
      normalizer: :NormalizerAuto,
      default_tokenizer: :TokenBigram
    )
  end

  def create_search_index
    Groonga::Schema.change_table('Terms') do |t|
      t.index('Requests.parameters')
      t.index('Logs.message')
    end
    Groonga::Schema.change_table('Logs') do |t|
      t.index('Requests.logs')
    end
  end
end

module Filter
  class Base
    private

    def or_join(cond)
      cond.inject { |a, e| e ? (a | e) : a }
    end

    def and_join(cond)
      cond.inject { |a, e| e ? (a & e) : a }
    end

    def and_not_join(cond)
      cond.inject { |a, e| e ? (a - e) : a }
    end
  end
end

module GroongaFilter
  class RequestId < Filter::RequestId
    def apply(builder)
      or_join(
        @request_ids
        .map { |request_id| builder[:request_id] == request_id }
      )
    end
  end
  class ControllerAction < Filter::ControllerAction
    def apply(builder)
      or_join(
        @controller_actions
        .map { |controller, action| build_cond(builder, controller, action) }
      )
    end

    private

    def build_cond(builder, controller, action)
      and_join(
        [
          builder[:controller] == controller,
          action && builder[:action] == action,
        ]
      )
    end
  end

  class HttpMethod < Filter::HttpMethod
    def apply(builder)
      builder[:http_method] == @http_method
    end
  end

  class HttpStatus < Filter::HttpStatus
    def apply(builder)
      column = builder[:http_status]
      cond = if @includes.empty?
               column != nil
             else
               prefix_matches(column, @includes)
             end
      cond -= prefix_matches(column, @excludes) unless @excludes.empty?
      cond
    end

    private

    def prefix_matches(column, words)
      or_join(words.map { |word| column.prefix_search(word) })
    end
  end

  module RangePattern
    private

    def cover?(column)
      and_join(
        [
          @range_begin && column >= @range_begin,
          if @range_end && @exclude_end
            column < @range_end
          elsif @range_end
            column <= @range_end
          end
        ]
      )
    end
  end

  class TimeRange < ::Filter::TimeRange
    include RangePattern

    def apply(builder)
      cover?(builder[:begin_time]) | cover?(builder[:end_time])
    end
  end

  class DurationRange < Filter::DurationRange
    include RangePattern

    def apply(builder)
      cover?(builder[:duration])
    end
  end

  module GroongaQuery
    def initialize(query)
      @query = query
    end

    def apply(builder)
      builder.match(@query, default_column: self.class::TARGET_COLUMN)
    end
  end

  class ParamsQuery < Filter::Base
    include GroongaQuery

    TARGET_COLUMN = 'parameters'
  end

  class LogsQuery < Filter::Base
    include GroongaQuery

    TARGET_COLUMN = 'logs.message'
  end
end

class GroongaSelector < Selector
  class QuerySyntaxError < RuntimeError; end

  class << self
    def filter_types
      @filter_types ||= {
        request_ids: [GroongaFilter::RequestId, :filter],
        controller_actions: [GroongaFilter::ControllerAction, :filter],
        http_method: [GroongaFilter::HttpMethod, :filter],
        http_status: [GroongaFilter::HttpStatus, :filter],
        time_range: [GroongaFilter::TimeRange, :filter],
        duration_range: [GroongaFilter::DurationRange, :filter],
        params_query: [GroongaFilter::ParamsQuery, :filter],
        logs_query: [GroongaFilter::LogsQuery, :filter],
      }
    end
  end

  def initialize
    super
    @pre_filters = @filters
  end

  def run(index, &block)
    requests = index.requests
    records = apply_filters(requests)
    records.each(&block)
  rescue Groonga::SyntaxError => e
    raise QuerySyntaxError, "syntax error found in query: #{raw_syntax_error_message(e)}"
  end

  private

  def apply_filters(requests)
    return requests.select if @filters.empty?

    requests.select do |builder|
      bs = @filters.map { |filter| filter.apply(builder) }
      bs.inject { |a, e| a & e }
    end
  end

  def raw_syntax_error_message(exception)
    exception
      .message
      .sub(/\Asyntax error: Syntax error: /, '')
      .sub(/: \[#<Groonga::Expression\n.*?\z/m, '')
  end
end

class GroongaCommandlineOption < CommandlineOption
  private

  def define_banner(parser)
    parser.banner = "usage: #{$0} [options] dir"
  end

  def define_filter_options(parser)
    super
    define_params_query_option(parser)
    define_logs_query_option(parser)
  end

  def define_groonga_printer_option(*); end

  def define_params_regexp_option(*); end

  def define_logs_regexp_option(*); end

  def define_params_query_option(parser)
    parser.on(
      '-P', '--params-query=QUERY', String,
      'Filter by parameters'
    ) do |query|
      @selector.add_filter(:params_query, query)
    end
  end

  def define_logs_query_option(parser)
    parser.on(
      '-L', '--logs-query=QUERY', String,
      'Filter by log messages'
    ) do |query|
      @selector.add_filter(:logs_query, query)
    end
  end
end

if $0 == __FILE__
  count = 0

  begin
    selector = GroongaSelector.new
    printer = Printer.new
    options = GroongaCommandlineOption.parse(selector: selector, printer: printer)

    index = GroongaIndex.new(ARGV.shift)
    index.select(selector) do |data|
      count += 1
      printer.print(data)
    end
  rescue Errno::EPIPE, Interrupt
    # noop
  rescue StandardError => e
    raise if options&.debug

    abort e.message
  end

  exit(count.zero? ? 1 : 0)
end