Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Example code for PuppetConf 2014 presentation
+ # Logstash-style report processer
+ class { 'logstash_reporter':
+ logstash_port => '2253',
+ logstash_host => hiera('logstash_reporter::logstash_host'),
+ config_file => '/etc/puppetlabs/puppet/logstash.yaml',
+ }
---
:host: heka-server-hostname
:port: 22253
-- Port of puppet-logstash-reporter/blob/master/lib/puppet/reports/logstash.rb for Heka (and Lua)
require "cjson"
local l = require "lpeg"
l.locale(l)
local dt = require "date_time"
local tablelib = require "table"
function process_message()
local raw_message = read_message("Payload")
local ok, json = pcall(cjson.decode, raw_message)
if not ok then
-- When plain text is found, ship it raw.
inject_message({
Type = "JsonParseFailure",
Logger = "PuppetLogstashReporter",
Payload = raw_message
})
return 0
end
local timestamp_grammar = dt.rfc3339 / dt.time_to_ns
local msg_timestamp = l.match(timestamp_grammar, json["@timestamp"])
-- build fields incrementally since failed runs have no metrics
local my_fields = {}
my_fields.report_format = json["report_format"]
my_fields.run_type = json["tags"]
my_fields.puppet_environment = json["environment"]
my_fields.puppet_version = json["puppet_version"]
my_fields.report_log = tablelib.concat(json["logs"], "\n")
my_fields.start_time = {value = json["start_time"], representation = 'date-time'}
my_fields.end_time = {value = json["end_time"], representation = 'date-time'}
my_fields.status = json["status"]
if json["metrics"] ~= nil then
if json["metrics"]["changes"] ~= nil then
my_fields.changes = json["metrics"]["changes"]["Total"]
end
if json["metrics"]["events"] ~= nil then
my_fields.events = { value =
{
json["metrics"]["events"]["Failure"],
json["metrics"]["events"]["Success"],
json["metrics"]["events"]["Total"],
},
representation = 'event-totals'
}
end
end
local message = {
Type = "RunReport",
Logger = "PuppetLogstashReporter",
Timestamp = msg_timestamp,
Hostname = json["host"],
Payload = json["message"],
Fields = my_fields,
}
inject_message(message)
return 0
end
common: &default_settings
license_key: '000-your-license-key-here-000'
app_name: 'Puppet DB'
log_level: info
log_file_path: /var/log/pe-puppetdb/
ssl: false
attributes:
enabled: true
transaction_tracer:
enabled: true
transaction_threshold: apdex_f
record_sql: obfuscated
stack_trace_threshold: 0.2
explain_enabled: true
explain_threshold: 0.2
error_collector:
enabled: true
ignore_errors: akka.actor.ActorKilledException
ignore_status_codes: 404
cross_application_tracer:
enabled: true
browser_monitoring:
auto_instrument: false
class_transformer:
# This instrumentation reports the name of the user principal returned from
# HttpServletRequest.getUserPrincipal() when servlets and filters are invoked.
com.newrelic.instrumentation.servlet-user:
enabled: false
#
production:
<<: *default_settings
###########################################
# Init settings for pe-puppetdb
###########################################
# Location of your Java binary (version 6 or higher)
JAVA_BIN="/opt/puppet/bin/java"
# Modify this if you'd like to change the memory allocation, enable JMX, etc
JAVA_ARGS="-Xmx19339m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pe-puppetdb/puppetdb-oom.hprof -Xms19339m"
# These normally shouldn't need to be edited if using OS packages
USER="pe-puppetdb"
INSTALL_DIR="/opt/puppet/share/puppetdb"
CONFIG="/etc/puppetlabs/puppetdb/conf.d"
# Enable the New Relic Java Agent by adding this to the bottom of the file
JAVA_ARGS="-javaagent:/opt/puppet/newrelic/newrelic.jar ${JAVA_ARGS}"
# -*- mode: ruby -*-
# Setup the New Relic Ruby Agent
require 'rubygems'
require 'newrelic_rpm'
# END New Relic Ruby Agent setup
# A "config.ru", for use with every Rack-compatible webserver.
# SSL needs to be handled outside this, though.
$0 = "master"
# If you want debugging, uncomment the following line:
# ARGV << "--debug"
ARGV << "--rack"
ARGV << "--confdir" << "/etc/puppetlabs/puppet"
ARGV << "--vardir" << "/var/opt/lib/pe-puppet"
require "puppet/application/master"
class Puppet::Application::Master
unless defined?(setup_original) then
alias :setup_original :setup
end
def setup
setup_original
end
end
#
# We don't want every transaction to just say "call", so define a
# Transaction "namer" for the Ruby Agent based on the URL structure
#
class CoolNamer
def initialize(app, opts = {})
@app = app
end
def call(env)
req = Rack::Request.new(env)
_, environment, kind_of_thing, resource = req.path.split('/',4)
NewRelic::Agent.set_transaction_name(kind_of_thing, :category => :rack)
NewRelic::Agent.add_custom_parameters(:puppet_env => environment)
NewRelic::Agent.add_custom_parameters(:puppet_resource => resource)
@app.call(env)
end
end
use CoolNamer
# END Transaction Namer
#
# To get better visibility into our transactions, we can hijack Puppet's
# built-in Profiler.
# NOTE: This hack is needed for Puppet 3.3.1, newer versions of Puppet
# allow you to add new profilers without clobbering the existing one.
#
require "puppet/util/profiler"
module Puppet::Util::Profiler
include ::NewRelic::Agent::MethodTracer
def self.profile(message, &block)
if /^Processed request / === message
# Top level trace, skip for now
yield
else
metric_name = self.message_to_metric(message)
# STDERR.puts(metric_name)
trace_execution_scoped([metric_name]) do
yield
end
end
end
def self.message_to_metric(message)
message.chomp!
# Compiled catalog for chi-staging-collector-2a.nr-ops.net in environment staging
message.gsub!(/^(Compiled catalog) for .*/, '\1')
# Filtered result for catalog chi-staging-collector-2a.nr-ops.net
message.gsub!(/^(Filtered result for catalog) .*/, '\1')
# Iterated (6) on generators
message.gsub!(/\(\d+\) /, '')
# Evaluated resource Nr_users::User[coolio] -> Evaluated resource/Nr_users::User
message.gsub!(/^(Evaluated resource) /, '\1/')
message.gsub!(/\[.*\]/,'')
# Compile: Finished catalog -> Compile/Finished catalog
message.gsub!(/\:\s/,'/')
# Called create_resources -> Called/create_resources
message.gsub!(/^(Called) /, '\1/')
"Custom/Puppet/#{message}"
end
end
Puppet::Util::Profiler.profile("Test") { sleep(1) }
# END Profiler hack
require 'puppet/util/command_line'
# we're usually running inside a Rack::Builder.new {} block,
# therefore we need to call run *here*.
run Puppet::Util::CommandLine.new.execute
common: &default_settings
license_key: '000-your-license-key-here-000'
app_name: "Puppet Master"
log_level: info
log_file_path: '/var/log/pe-puppet/'
ssl: false
capture_params: true
transaction_tracer:
enabled: true
error_collector:
enabled: true
ignore_errors: "ActionController::RoutingError"
#
production:
<<: *default_settings
monitor_mode: true

have you designed an informative dashboard for kibana?

TJM commented Aug 29, 2016

@benders: Have you updated the config.ru recently? The version above fails violently, even on the rather old version of puppet that we are running (3.8.7). Thanks in advance!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment