method.rb

require 'open3'

class PowerDNSRectifyWorker
  include Resque::Plugins::UniqueJob

  @queue = :powerdns_rectify

  ALLOWED_RETRIES = 3

  def self.redis_key payload
    (payload[:args] || payload['args']).first # fqdn
  end

  def self.perform fqdn
    attempts ||= 0
    # Since pdnssec always exits with 0 and only logs to STDERR we have to capture it
    # and strip the response message from there. If it does not match the expected message
    # we should think that it has failed.
    output = Open3.capture2e("sudo pdnssec rectify-zone '#{fqdn}'")[0].strip
    case output
    when 'Adding NSEC ordering information'
      # Not an error, doing nothing
    when "No SOA known for '#{fqdn}', is such a zone in the database?"
      raise PowerDNSNoSOAFoundError
    else
      raise PowerDNSRectifyError
    end
  rescue PowerDNSNoSOAFoundError => e
    attempts += 1
    if attempts > ALLOWED_RETRIES
      RemoteLogger.notify message: "PowerDNS rectify script failed for #{fqdn}.", fqdn: fqdn, output: output
      Airbrake.notify_or_ignore e, parameters: { fqdn: fqdn, output: output }
    else
      sleep 1
      retry
    end
  rescue => e
    RemoteLogger.notify message: "PowerDNS rectify script failed for #{fqdn}.", fqdn: fqdn, output: output
    Airbrake.notify_or_ignore e, parameters: { fqdn: fqdn, output: output }
  end
end

class PowerDNSRectifyError < StandardError; end
class PowerDNSNoSOAFoundError < StandardError; end

spec.rb

require 'rails_helper'

describe PowerDNSRectifyWorker do
  before(:each) do
    allow(PowerDNSRectifyWorker).to receive(:perform).and_call_original
  end

  describe '#perform' do
    context 'when it succeeds before max allowed retries are reached' do
      it 'does not raise an error' do
        @times_called = 0
        allow(Open3).to receive(:capture2e) do
          @times_called += 1
          if @times_called > (PowerDNSRectifyWorker::ALLOWED_RETRIES - 1)
            ["Adding NSEC ordering information"]
          else
            ["No SOA known for 'example.com', is such a zone in the database?"]
          end
        end
        expect(RemoteLogger).to_not receive(:notify)
        PowerDNSRectifyWorker.perform('example.com')
      end
    end

    context 'when every attempt fails' do
      before :each do
        allow(Open3).to receive_messages(capture2e: ["No SOA known for 'example.com', is such a zone in the database?"])
      end

      it 'raises an error' do
        expect(RemoteLogger).to receive(:notify).exactly(1).times
        PowerDNSRectifyWorker.perform('example.com')
      end
    end
  end
end