dwbutler

CLEANUP_THRESHOLD = 250 * 1_024_000 # 250 megabytes

# See https://robots.thoughtbot.com/how-to-reduce-a-large-heroku-compiled-slug-size
desc "Clean up the Heroku repo to reduce compiled slug size"
task cleanup: :set_app do
  force = ENV.fetch('FORCE_CLEANUP', false)
  if force || slug_size > CLEANUP_THRESHOLD
    puts "Heroku compiled slug size exceeds 250 MB. Cleaning up the Heroku repo."
    install_heroku_plugin('heroku-repo')
    _execute heroku_command("repo:gc")

dwbutler

• Downtime window: 4:41 PM EST - 4:55 PM EST
• Total downtime: 14 minutes

"Bob" (backend engineer) was the developer on call during this incident. All names have been changed to protect the innocent. :)

Summary

Bad code which prevented server startup was deployed to production. It had not been manually tested in a dev environment, but had passed all unit and

dwbutler

Written by David. Much thanks to Lukasz for technical and moral support, and helping me grep the Papertrail logs.

All times are in EST.

• Downtime window: 8:00 AM - 8:25 AM
• Total downtime: 25 minutes

TL;DR

Heroku PGBackups was running during a database migration, and was unexpectedly holding a lock. Postgres stopped serving any queries until the backup was canceled.

dwbutler

namespace :sync do
  desc "Run production sync"
  task run: [
      :set_app,
      :maint_on,
      :scale_down,
      :cancel_backup,
      :delete_database,
      :create_production_fork,
      :migrate,

dwbutler

$ logstash -f samples/tcp.conf

Settings: Default pipeline workers: 8
Pipeline main started
{"message":"\"2017-04-28T22:44:30.424-07:00\"","@timestamp":"2017-04-29T05:44:30.424Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.528-07:00\"","@timestamp":"2017-04-29T05:44:30.528Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.633-07:00\"","@timestamp":"2017-04-29T05:44:30.633Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.737-07:00\"","@timestamp":"2017-04-29T05:44:30.737Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.842-07:00\"","@timestamp":"2017-04-29T05:44:30.842Z","@version":"1","severity":"INFO","host":"Davids-MacBook-Pro-6.local","port":62671}
{"message":"\"2017-04-28T22:44:30.947-07:00\"","@timestamp":"2017-04-29T05:44:30.9

dwbutler

class Config
  CONSTANT = 1

  def CONSTANT=(val)
    fail
  end
end

config = Config.new
config.CONSTANT = 2

dwbutler

class Config
  attr_accessor :thing

  def thing=(val)
    raise ArgumentError if defined?(@thing)
    @thing = val
  end
end

config = Config.new

dwbutler

require 'appcanary'

Appcanary.configure do |config|
  config.api_key = ENV['APPCANARY_API_KEY']
end

# Explicitly specify language and file
Appcanary.check(language: :ruby, file: 'Gemfile.lock')

# Autodetect Ruby as the language and Gemfile.lock as the file

dwbutler

# Tested with ActiveRecord 4.0.5
# Run using:
# BUNDLE_GEMFILE=gemfiles/rails_4.0.gemfile bundle console

require 'sqlite3'
require 'active_record'
require 'groupify/adapter/active_record'

ActiveSupport::VERSION::STRING
# => "4.0.5" 

dwbutler

require 'time'
require 'benchmark'

time = Time.now.iso8601(3)
#  => "2014-05-26T23:26:08.628-07:00"

Benchmark.bm do |x|
  x.report('parse') { 100_000.times { Time.parse(time).gmtime } } 
  x.report('iso8601') { 100_000.times { Time.iso8601(time).gmtime } }
end