dwbutler

# Tested in Ruby 1.8.7

class ParentClass
  def call_private_good
    # Private methods can only be called without an explicit receiver
    private_test
  end
  
  def call_private_bad
    # Calling a private method with an explicit receiver (self) will fail

dwbutler

CLEANUP_THRESHOLD = 250 * 1_024_000 # 250 megabytes

# See https://robots.thoughtbot.com/how-to-reduce-a-large-heroku-compiled-slug-size
desc "Clean up the Heroku repo to reduce compiled slug size"
task cleanup: :set_app do
  force = ENV.fetch('FORCE_CLEANUP', false)
  if force || slug_size > CLEANUP_THRESHOLD
    puts "Heroku compiled slug size exceeds 250 MB. Cleaning up the Heroku repo."
    install_heroku_plugin('heroku-repo')
    _execute heroku_command("repo:gc")

dwbutler

• Downtime window: 4:41 PM EST - 4:55 PM EST
• Total downtime: 14 minutes

"Bob" (backend engineer) was the developer on call during this incident. All names have been changed to protect the innocent. :)

Summary

Bad code which prevented server startup was deployed to production. It had not been manually tested in a dev environment, but had passed all unit and

dwbutler

Written by David. Much thanks to Lukasz for technical and moral support, and helping me grep the Papertrail logs.

All times are in EST.

• Downtime window: 8:00 AM - 8:25 AM
• Total downtime: 25 minutes

TL;DR

Heroku PGBackups was running during a database migration, and was unexpectedly holding a lock. Postgres stopped serving any queries until the backup was canceled.

dwbutler

namespace :sync do
  desc "Run production sync"
  task run: [
      :set_app,
      :maint_on,
      :scale_down,
      :cancel_backup,
      :delete_database,
      :create_production_fork,
      :migrate,

dwbutler

$ logstash -f samples/tcp.conf

Settings: Default pipeline workers: 8
Pipeline main started
{"message":"\"2017-04-28T22:44:30.424-07:00\"","@timestamp":"2017-04-29T05:44:30.424Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.528-07:00\"","@timestamp":"2017-04-29T05:44:30.528Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.633-07:00\"","@timestamp":"2017-04-29T05:44:30.633Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.737-07:00\"","@timestamp":"2017-04-29T05:44:30.737Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.842-07:00\"","@timestamp":"2017-04-29T05:44:30.842Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.947-07:00\"","@timestamp":"2017-04-29T05:44:30.9

dwbutler

class Config
  CONSTANT = 1

  def CONSTANT=(val)
    fail
  end
end

config = Config.new
config.CONSTANT = 2

dwbutler

class Config
  attr_accessor :thing

  def thing=(val)
    raise ArgumentError if defined?(@thing)
    @thing = val
  end
end

config = Config.new

dwbutler

namespace :db do
  desc 'Open a MongoDB console with connection parameters for the current Rails.env'
  task :console, [:session] => :environment do |t, args|
    session_name = args[:session] || :default

    config = Mongoid.sessions[session_name]
    username = config[:username]
    password = config[:password]
    database = config[:database]

dwbutler

require 'benchmark'

 Benchmark.bm {|x| x.report { 1_000.times { defined?(::UNF::Normalizer) } } }
       user     system      total        real
   0.300000   0.010000   0.310000 (  0.340000)