count_envs_in_access_logs.rb

#!/usr/bin/env ruby

request_count = 0
catalog_request_count = 0
environment_cat_requests = {}

access_logs = Dir["*access*"]
access_logs.each do |f|
   File.open(f).each do |line|
      match = line.match(/^([^\s]+) - - - ([^\s])+ ([^s]+) ("[^\"]+") (\d+) (\d+) ([^\s]+) ([^\s]+) (\d+) (\d+)$/)
      unless match
         raise "Line does not match regex!: '#{line}'"
      end
      host, timestamp, timezone, request, status, size, ip, port, duration = match.captures
      #puts "REQUEST: #{request}"
      request_count += 1
      catalog_match = request.match(/^"POST \/(\w+)\/catalog\/([\w\.\-]+) HTTP\/1\.1"$/)
      if catalog_match
         catalog_request_count += 1
         environment, certname = catalog_match.captures
         environment_cat_requests[environment] ||= 0
         environment_cat_requests[environment] += 1
      end
   end
end

puts "Total requests: #{request_count}"
puts "Catalog requests: #{catalog_request_count}"
puts "Environment catalog  requests: #{environment_cat_requests}"

Here's an updated version that handles changes to the request logging formant in PE 2016.1:

#!/usr/bin/env ruby

require 'zlib'

request_count = 0
catalog_request_count = 0
environment_cat_requests = {}

PE_38 = /[^\s]+ [^\s]+ [^\s]+ [^\s]+ .+ "(?<url>.*)" [^\s]+ [^\s]+ [^\s]+ [^\s]+ [^\s]+ [^\s]+/
PE_20161 = /[^\s]+ [^\s]+ [^\s]+ \[.+\] "(?<url>.*)" [^\s]+ [^\s]+ "[^\s]+" "[^\s]+" [^\s]+/

access_logs = Dir["*puppetserver-access*"]
access_logs.each do |f|
  file = if File.extname(f) == '.gz'
    Zlib::GzipReader.new(File.open(f, 'rb'))
  else
    File.open(f, 'rb')
  end

  file.each_line do |line|
    match = PE_20161.match(line)
    match ||= PE_38.match(line)
    unless match
      raise "No match for:\n#{line}"
    end

    request = match['url']
    request_count += 1

    # PE 2015.2.x and PE 2015.3.x didn't record the environment used by catalog requests.
    # So, inspect the report instead.
    catalog_match = request.match(/^PUT \/puppet\/v3\/report\/(?<node>[\w\.\-]+)\?environment=(?<env>\w+)/)
    catalog_match ||= request.match(/^POST \/(?<env>\w+)\/catalog\/(?<node>[\w\.\-]+) HTTP\/1\.1$/)

    if catalog_match
      catalog_request_count += 1

      certname = catalog_match['node']
      environment = catalog_match['env']

      environment_cat_requests[environment] ||= 0
      environment_cat_requests[environment] += 1
    end
  end

  file.close
end

puts "Total requests: #{request_count}"
puts "Catalog requests: #{catalog_request_count}"
puts "Environment catalog  requests: #{environment_cat_requests}"