email_processing_in_ruby

# Simple way to process incoming emails in ruby
# 
# Emails can be pretty complex with their attachments.  For example:
# 
#  email
#    alternatives
#      text
#      HTML
#    attachment
# 
# the following uses a "recursive search" for text/plain portions.
# 
# Adapted from somewhere by Charles Brian Quinn cbq x highgroove.com
# Updated by James Edward Gray II  james x highgroove.com

  class << self
    
    # for tag stripping and simple formatting
    include ActionView::Helpers::TextHelper
    include ActionView::Helpers::TagHelper
    
    # use the default AR logger for normal log messages
    def logger
      ActiveRecord::Base.logger
    end
    
    # Handles retrieving emails from the specified mailbox and performing
    # preliminary checks on whether the email is a Post or not.
    #
    # As of right now, also handles Messages, though this probably should
    # be abstracted out into another class/model.
    # 
    # TODO:
    # * connection details defaults/configuration
    # * connect to the POP mailbox
    # * test for post matches
    # ** +to+ matches discussion group
    # ** +from+ matches someone who can post to the discussion group
    # * process matches
    # * OR handle non-matches
    # 
    # Post.receive_mail(connection details) do
    #   Net::POP3.start(connection details) do |email|
    #     if email.to matches list_of_discussion_groups
    #     and email.from matches someone_who_can_post_to_discussion_group
    #       Post.process_incoming_mail(email)
    #     else
    #       if email.to matches Users
    #         # notify user they do not belong to discussion group
    #       else
    #         # decide later to delete, forward, or otherwise non-matches
    # 
    def receive_mail(options = {})
      total_posts = Post.count
      total_messages = Message.count
      
      self.incoming_mail_logger.info "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] Checking for incoming mail"
      
      config = AppConstants::IncomingMail::DEFAULT_CONNECTION.merge(options)
      imap = Net::IMAP.new(config[:host], config[:port], config[:use_ssl])
      # imap.authenticate('LOGIN', config[:username], config[:password])  # login works better (authenticate doesn't work on all imap implementations)
      
      imap.login(config[:username], config[:password])
      self.incoming_mail_logger.debug "*** Logged in as #{config[:username]} at #{config[:host]}:#{config[:port]}"
      
      # imap.examine('INBOX')  # examine is for read-only, need to use select
      imap.select('INBOX')
      self.incoming_mail_logger.debug "*** Inbox selected"
      
      imap.search(['UNSEEN']).each do |message_id| # also could be "RECENT" but "NEW" is recent + not seen
        self.incoming_mail_logger.debug "*** Processing MessageID: #{message_id}"
        mail = TMail::Mail.parse(imap.fetch(message_id, 'RFC822')[0].attr['RFC822'])
        # imap.fetch(6, 'BODY').first.attr['BODY'].is_a?(Net::IMAP::BodyTypeMultipart)
        self.incoming_mail_logger.debug "FROM: #{mail.from} TO: #{mail.to}"
        self.incoming_mail_logger.debug mail.to_s
        
        if (dg = find_discussion_group_addressed_to(mail.to)) and
          (dg.subscribers.include?(User.find_by_email(mail.from)))
          self.incoming_mail_logger.debug "*** Discussion Group Found: #{dg.title} (and user is a subscriber)"
          
          if post = Post.process_incoming_mail(mail)
            self.incoming_mail_logger.debug "=> Successfully posted incoming mail as #{post.id}!"
            
            imap.store(message_id, "+FLAGS", [:Deleted]) # mark email to be deleted
          else
            self.incoming_mail_logger.debug "=> Message #{message_id} marked as seen and flagged. May need special attention."
            imap.store(message_id, "+FLAGS", [:Seen, :Flagged]) # mark email seen # may already be performed implicitly
          end
        elsif (mail.to.to_s =~ /messages@sidebaronline.org/) && (User.find_by_email(mail.from))
          self.incoming_mail_logger.debug "*** Incoming Message: (and user is a valid user)"
          
          if message = Message.process_incoming_mail(mail)
            self.incoming_mail_logger.debug "=> Successfully replied to incoming mail as #{message.id}!"
            
            imap.store(message_id, "+FLAGS", [:Deleted]) # mark email to be deleted
          else
            self.incoming_mail_logger.debug "=> Message #{message_id} marked as seen and flagged. May need special attention."
            imap.store(message_id, "+FLAGS", [:Seen, :Flagged]) # mark email seen # may already be performed implicitly
          end 
        else
          # leave junk in there
          self.incoming_mail_logger.debug "=> Marking #{message_id} as seen and ignoring (possible junk)"
          imap.store(message_id, "+FLAGS", [:Seen]) # mark email seen # may already be performed implicitly
        end
      end
      # imap.expunge # deletes all messages marked to be deleted
      
      new_posts = Post.count - total_posts # the number of new posts after running
      new_messages = Message.count - total_messages # the number of new messages after running
      
      self.incoming_mail_logger.info "#{new_posts} new posts/replies. #{new_messages} new messages."
      self.incoming_mail_logger.info "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] Finished.\n\n\n"
      return new_posts
      
    rescue Net::IMAP::NoResponseError => e
      self.incoming_mail_logger.error "The IMAP server did not respond. Try again later."
      self.incoming_mail_logger.error e.message
    rescue Net::IMAP::ByeResponseError => e
      self.incoming_mail_logger.error "The IMAP server has closed communication. Try again later."
      self.incoming_mail_logger.error e.message
    rescue Exception => e
      self.incoming_mail_logger.error e.message
      self.incoming_mail_logger.error "\n\t" + e.backtrace.join("\n\t")
      false
    end
    
    # Processes a positive post email into either a new post or a reply.
    # 
    # TODO:
    # * determine new post or reply
    # * create appropriate entity
    # 
    # Post.process_incoming_mail(email) do
    #   if email.body matches format_of_a_reply
    #     Post.create_as_reply
    #   else
    #     Post.create_as_new_post
    # 
    # Also, eventually we'll handle attachments.
    # 
    def process_incoming_mail(email)
      should_retry = true
      
      post = Post.new(
        :title => email.subject,
        :user => User.find_by_email(email.from),
        :discussion_group => self.find_discussion_group_addressed_to(email.to)
      )
      
      body = ""
      extract_text = lambda do |message_or_part|
        if message_or_part.multipart?
          message_or_part.each_part do |part|
            extract_text[part]
          end
        elsif message_or_part.content_type == "text/plain"
          body += "\n#{message_or_part.body}"
        end
      end
      extract_text[mail]
      
      if (reply = body.match(REPLY))
        reply_body, post_id, original_body = reply.captures
        # reply
        
        post.body = simple_format(reply_body)
        # remove the discussion group name from the post title
        post.title.gsub!("[#{post.discussion_group.title}] ", '')
        
        if Post.find(post_id).children << post
          return post # respond with the Post just created
        else
          self.incoming_mail_logger.error "Problem saving!"
          self.incoming_mail_logger.debug "Post is a reply to #{post_id}."
          self.incoming_mail_logger.debug "Valid? #{post.valid?.to_s} (#{post.errors.full_messages.join(', ')})"
          false
        end
      else
        # new post
        post.body = simple_format(body)
        if post.save
          return post # respond with the Post just created
        else
          self.incoming_mail_logger.error "Problem saving!"
          self.incoming_mail_logger.debug "Valid? #{post.valid?.to_s} (#{post.errors.full_messages.join(', ')})"
          false
        end
      end
    rescue Exception => e
      # document the actual problem
      case e
      when Ferret::FileNotFoundError
        # http://www.ruby-forum.com/topic/124914
        self.incoming_mail_logger.error "Problem occurred accessing the Ferret index, preventing the post to be created:"
        self.incoming_mail_logger.error post.inspect
      end
      
      self.incoming_mail_logger.error e.message
      self.incoming_mail_logger.error "\n\t" << e.backtrace.join("\n\t")
      
      if should_retry
        self.incoming_mail_logger.error "Retrying..."
        should_retry = false
        sleep 0.7
        retry
      end
      
      return false
    end
    
    # Provides a specific logging point for all of the incoming mailing
    # activity.
    # 
    def incoming_mail_logger
      @_incoming_mail_logger ||= begin
        logger = Logger.new("#{RAILS_ROOT}/log/incoming_mail.#{RAILS_ENV}.log")
        logger.level = ActiveRecord::Base.logger.level
        logger
      end
    end

  end