Ruby script to test how to fetch IMAP mails (IDLE "push" mode) without pulling (in "real-time")

idle.rb

# Encoding: utf-8

#
# idle.rb
# 
# goal:
#   Ruby script to test how to fetch IMAP mails with IDLE mode.
#   IMAP IDLE allow a sort of "push" / "real-time" delivery.
#
#   I used the script to test LATENCY (end-to-end delivery times)
#   of some "transactional email" service providers:
#   script track elapsed time (hours, minutes, seconds) between 
#   - the instant a mail is submitted to an email provider (using API calls)  
#   - the instant the mail is received by IMAP IDLE client -> IMAP SEARCH -> IMAP FETCH
#
# search_condition:
#   all "UNSEEN" mails from a sender; 
#   IMAP query statement: ['UNSEEN', 'FROM', <sender_email>]
#
#  Thanks to mzolin for the essential trick,see: 
#    http://stackoverflow.com/questions/4611716/how-imap-idle-works
#
#  Obscure Ruby documentation (without smart examples): 
#    http://ruby-doc.org/stdlib-2.1.5/libdoc/net/imap/rdoc/Net/IMAP.html
#
# usage:
#   1. set env vars:
#
#     $ export USERNAME=<recipient_email> 
#     $ export PW=<your_password>
#     $ export SENDER_MAIL=<sender_email>
#
#   2. run the script:
#
#     $ ruby idle.rb
#
#   3. send e-mails from <sender_email> to <recipient_email>
#      optionally with subject in format:
#
#      ID: <a sequence number> TIME: <ISO8601 timestamp>  
#       
#      specifying subject in that way, 
#      script will pretty print some latency (ene-to-end time delivery) statistics    
#
#
# TODO:
#   imap idle is critical because depending on IMAP servers, connection is closed
#   I experincied the IMAP IDLE sometime hang-up :-(
#   A possible workaround is to run a Thread that every N minutes force a idle_done...
#
#
# E-mail: giorgio.robino@gmail.com 
# Github: wwww.github.com/solyaris 
#

require 'time'
require 'mail'
require 'net/imap'
require 'colorize'

# flag to print Ruby library debug info (very detailed)
@net_imap_debug = false

# script application level debug flag
@debug = false

# set true to delete mail after processing 
@expunge = true

# Initialize an array to store all end-to-end elapsed_time
@latency_history = []

# return timestamp in ISO8601 with precision in milliseconds 
def time_now
  Time.now.utc.iso8601(3)
end

#
# return the Time from subject of e-mail, for statistic caluclations
# supposing subject contain a timestamp (ISO8601) in format defined by regex (raugh):
#
#   /ID: (\d*) TIME: (\S*)/
# 
# example of subject:
#   ID: 200 TIME: 2014-12-15T11:18:44.030Z
# 
#   get_time_from_subject("ID: 200 TIME: 2014-12-15T11:18:44.030Z") 
#     # => 2014-12-15 11:18:44 UTC
#     # => nil if time if subject do not match regex (do not contains timestamp)
#
def get_time_from_subject(subject)
  # I got a timestamp in the subject ?
  m = /ID: (\d*) TIME: (\S*)/.match subject
  # if yes, convert it to internal Time format
  Time.iso8601(m[2]) if m
end


def get_id_from_subject(subject)
  # I got a timestamp in the subject ?
  m = /ID: (\d*)/.match subject
  
  # if yes, return ID string
  m[1] if m
end

# coloured pretty print a bit of statistic on time delays
def statistics(before)
  
  timestamp_format = "%M:%S" # "%H:%M:%S"
  curr_value = Time.now.utc - before
  @latency_history << curr_value

  # current vale
  now = Time.at(curr_value).utc.strftime timestamp_format

  # minimum value 
  min = Time.at(@latency_history.min).utc.strftime timestamp_format

  # average value
  average = @latency_history.reduce(:+).to_f / @latency_history.size
  avg = Time.at(average).utc.strftime timestamp_format

  # maximum value
  max = Time.at(@latency_history.max).utc.strftime timestamp_format
    
  print "now:"; print " #{now} ".black.on_white 
  print " min:"; print " #{min} ".on_green 
  print " avg:"; print " #{avg} ".black.on_yellow 
  print " max:"; print " #{max} ".on_red
  print "\n"
end


#
# imap_connection  
#
# connect to a specified serve and login
#
def imap_connection(server, username, password)
  # connect to IMAP server
  imap = Net::IMAP.new server, ssl: true, certs: nil, verify: false

  Net::IMAP.debug = @net_imap_debug

  # http://ruby-doc.org/stdlib-2.1.5/libdoc/net/imap/rdoc/Net/IMAP.html#method-i-capability
  capabilities = imap.capability 

  puts "imap capabilities: #{capabilities.join(',')}" if @debug

  unless capabilities.include? "IDLE"
    puts "'IDLE' IMAP capability not available in server: #{server}".red
    imap.disconnect
    exit
  end

  # login
  imap.login username, password

  # return IMAP connection handler
  imap
end


#
# retrieve_emails
#
# retrieve any mail from a folder, followin specified serach condition
# for any mail retrieved call a specified block
#
def retrieve_emails(imap, search_condition, folder, &process_email_block)

  # select folder
  imap.select folder

  # search messages that satisfy condition
  message_ids = imap.search(search_condition) 

  if @debug
    if message_ids.empty?
      puts "\nno messages found.\n"
      return
    else
      puts "\n#{message_ids.count} messages processed.\n".blue
    end    
  end

  message_ids.each do |message_id|

    # fetch all the email contents
    msg = imap.fetch(message_id,'RFC822')[0].attr['RFC822']
    
    # instantiate a Mail object to avoid further IMAP parameters nightmares 
    mail = Mail.read_from_string msg

    # call the block with mail object as param 
    process_email_block.call mail
    
    # mark as read ore deleted
    if @expunge
      imap.store(message_id, "+FLAGS", [:Deleted])
    else
      imap.store(message_id, "+FLAGS", [:Seen])
    end
  end
end  


#
# process_mail
#
# do something with the e-mail content (param is a Mail gem instance)
#
def process_email(mail)
  #
  # just puts to stdout 
  #
  subject = mail.subject
  body = mail.body.decoded
  #puts (mail.text_part.body.to_s).green

  print "\nnew mail ID: "
  print " #{get_id_from_subject subject} ".black.on_white
  puts "\n         at: #{time_now}"

  puts " message id: #{mail.message_id}" 
  #puts "intern.date: #{mail.date.to_s}"       

  puts "    subject: #{subject}"

  puts "  text body: #{body[0..80]}#{(body.size > 80) ? '...': ''}"
  
  #
  # calculate end-to-end e-mail delivery time 
  #
  mail_sent_at = get_time_from_subject subject
  if mail_sent_at
    print "    LATENCY: "
    statistics mail_sent_at
  end  
end


def shutdown(imap)
    imap.idle_done
    imap.logout unless imap.disconnected?
    imap.disconnect
  
    puts "#{$0} has ended (crowd applauds)".green
    exit 0
end

#
# idle_loop
#
# check for any further mail with "real-time" responsiveness.
# retrieve any mail from a folder, following specified search condition
# for any mail retrieved call a specified block
#
def idle_loop(imap, search_condition, folder, server, username, password)

  puts "\nwaiting new mails (IDLE loop)..."

  # http://stackoverflow.com/questions/4611716/how-imap-idle-works
  loop do
    begin
      imap.select folder
      imap.idle do |resp|
 
        # You'll get all the things from the server. For new emails (EXISTS)
        if resp.kind_of?(Net::IMAP::UntaggedResponse) and resp.name == "EXISTS"

          puts resp.inspect if @debug
          # Got something. Send DONE. This breaks you out of the blocking call
          imap.idle_done
        end
      end

      # We're out, which means there are some emails ready for us.
      # Go do a search for UNSEEN and fetch them.
      retrieve_emails(imap, search_condition, folder) { |mail| process_email mail }

      # delete processed mails (or just flah them as "seen" ) 
      imap.expunge if @expunge

    rescue SignalException => e
      # http://stackoverflow.com/questions/2089421/capturing-ctrl-c-in-ruby
      puts "Signal received at #{time_now}: #{e.class}. #{e.message}".light_red
      shutdown imap
    
    rescue Net::IMAP::Error => e
      puts "Net::IMAP::Error at #{time_now}: #{e.class}. #{e.message}".light_red

      # timeout ? reopen connection
      imap = imap_connection(server, username, password) #if e.message == 'connection closed'
      puts "reconnected to server: #{server}"

    rescue Exception => e
      puts "Something went wrong at #{time_now}: #{e.class}. #{e.message}".red

      imap = imap_connection(server, username, password)
      puts "reconnected to server: #{server}"
    end
  end
end


#
# main
#
# get parameters from environment variables
#
server = ENV['SERVER'] ||= 'imap.gmail.com'
username = ENV['USERNAME']
password = ENV['PW']
folder = ENV['FOLDER'] ||= 'INBOX'
from = ENV['SENDER_MAIL']
search_condition = ['UNSEEN', 'FROM', from ]

if !password or !username
  puts "specify USERNAME and PW env vars".red
  exit
end

puts "\n      imap server: #{server}"
puts "         username: #{username}"
puts "           folder: #{folder}"
puts " search condition: #{search_condition.join(',')}"

imap = imap_connection(server, username, password)

# at start-up check for any mail (already received) and process them 
retrieve_emails(imap, search_condition, folder) { |mail| process_email mail }

# check for any further mail with "real-time" responsiveness
idle_loop(imap, search_condition, folder, server, username, password)

imap.logout
imap.disconnect