fluent-plugin-datadog-batchsize-bench.rb

require "fluent/plugin/out_datadog"
require "ougai"
require "json"

DD_API_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXX"
DD_DEFAULT_HTTP_ENDPOINT = "http-intake.logs.datadoghq.com"
COMPRESSION_LEVEL = 6

class DDBenchClient
  attr_reader :logger, :dd, :client

  def initialize
    @logger = Ougai::Logger.new("bench.log.#{Time.now.to_s}")
    @dd = Fluent::DatadogOutput.new
    @client =
      @dd.new_client(
        logger,
        DD_API_KEY,
        true,
        true,
        false,
        DD_DEFAULT_HTTP_ENDPOINT,
        443,
        nil,
        nil,
        true,
        false
      )
  end
  def event
    {
      ddsource: "nginx",
      ddtags: "env:staging,version:5.1",
      hostname: "i-012345678",
      message:
        Time.now.strftime(
          "%Y-%M-%DT%H:%m:%s,%S" + "INFO [process.name][20081] Hello"
        ),
      service: "payment"
    }
  end

  def send_with_batch_size(size)
    # イベントを指定個数生成する
    events = size.times.map { event.to_json }

    # バッチサイズに分ける
    batches =
      dd.batch_http_events(
        events,
        Fluent::DatadogOutput::DD_MAX_BATCH_LENGTH,
        Fluent::DatadogOutput::DD_MAX_BATCH_SIZE
      )

    logger.info(
      { size: size, bytes: events.map(&:bytesize).sum, batches: batches.size }
    )
    # 送信する
    batches.each do |batch|
      client.send_retries(
        dd.gzip_compress(dd.format_http_event_batch(batch), COMPRESSION_LEVEL),
        -1,
        30
      )
    end
  end
end

require "benchmark"

d = DDBenchClient.new

# "500個でバッチ1つ"

puts "## イベント数を変えるパターン"
sizes = (1..10).map { |n| n * 500 }
Benchmark.bm 25 do |r|
  sizes.each do |size|
    r.report("#{size} with #{size / 500} batches") do
      d.send_with_batch_size(size)
    end
  end
end

puts "## イベント数は変えずにバッチ内の個数と送信回数を変えるパターン"
Benchmark.bm 25 do |r|
  [[50, 100], [500, 10], [5000, 1]].each do |num, repeat|
    r.report("events: #{num}, #{repeat} times") do
      repeat.times { d.send_with_batch_size(num) }
    end
  end
end

puts "## 1バッチだけで個数を変えるパターン"
sizes = [100, 200, 300, 400, 500]
Benchmark.bm 25 do |r|
  sizes.each { |size| r.report(size.to_s) { d.send_with_batch_size(size) } }
end