OSTicket IMAP monitor with IDLE

osticket-imap-monitor

Modified the imap-monitor script from https://gist.github.com/shimofuri/4348943 for Python3, 
not using eventlet, just IMAPClient and requests (installed with pip)

This version is specifically for OSTicket, it grabs the email as-is and post to OSTicket, 
so that tickets can be generated from emails ASAP.

osticket-imap-monitor.ini

[setting]
idle_timeout = 900

[imap]
host = imap.example.com
username = super.man@example.com
password = Kryptonized
ssl = True
folder = INBOX

[osticket]
post_url = http://support.my-site.com/api/tickets.email
api_key = CA0AFA0AFA0AFA0AFA0AFA0AFA0AFA06

osticket-imap-monitor.py

#!/usr/bin/python3
# modified from https://gist.github.com/shimofuri/4348943

from imapclient import IMAPClient
import requests

import sys
import traceback
import logging
from logging.handlers import RotatingFileHandler
import configparser
from time import sleep

# Setup the log handlers to stdout and file.
log = logging.getLogger('imap_monitor')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(message)s')

handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(logging.DEBUG)
handler_stdout.setFormatter(formatter)
log.addHandler(handler_stdout)

handler_file = RotatingFileHandler(
    '/var/log/imap_monitor.log',
    mode='a',
    maxBytes=1048576,
    backupCount=9,
    encoding='UTF-8',
    delay=True)
handler_file.setLevel(logging.DEBUG)
handler_file.setFormatter(formatter)
log.addHandler(handler_file)

# TODO: Support SMTP log handling for CRITICAL errors.

post_url = '';  # to be read from config
api_key  = '';  # to be read from config

def main():
    global post_url;
    global api_key;

    log.info('... script started')
    while True:
        # <--- Start of configuration section

        # Read config file - halt script on failure
        try:
            config_file = open('/etc/imap_monitor.ini','r+')
        except IOError:
            log.critical('configuration file is missing')
            break
        config = configparser.ConfigParser()
        config.read_file(config_file)

        # retrieve idle_timeout
        try:
            idle_timeout = int(config.get('setting', 'idle_timeout'))
        except configparser.NoSectionError:
            log.critical('no "setting" section in configuration file')
            break
        except configparser.NoOptionError:
            log.critical('no idle_timeout specified in configuration file')
            break

        # retrieve OSTicket post_url
        try:
            post_url = config.get('osticket', 'post_url')
        except configparser.NoSectionError:
            log.critical('no "osticket" section in configuration file')
            break
        except configparser.NoOptionError:
            log.critical('no OSTicket post_url specified in configuration file')
            break

        # retrieve OSTicket API key
        try:
            api_key = config.get('osticket', 'api_key')
        except configparser.NoSectionError:
            log.critical('no "osticket" section in configuration file')
            break
        except configparser.NoOptionError:
            log.critical('no OSTicket api_key specified in configuration file')
            break


        # Retrieve IMAP host - halt script if section 'imap' or value missing
        try:
            host = config.get('imap', 'host')
        except configparser.NoSectionError:
            log.critical('no "imap" section in configuration file')
            break
        except configparser.NoOptionError:
            log.critical('no IMAP host specified in configuration file')
            break

        # Retrieve IMAP username - halt script if missing
        try:
            username = config.get('imap', 'username')
        except configparser.NoOptionError:
            log.critical('no IMAP username specified in configuration file')
            break

        # Retrieve IMAP password - halt script if missing
        try:
            password = config.get('imap', 'password')
        except configparser.NoOptionError:
            log.critical('no IMAP password specified in configuration file')
            break

        # Retrieve IMAP SSL setting - warn if missing, halt if not boolean
        try:
            ssl = config.getboolean('imap', 'ssl')
        except configparser.NoOptionError:
            # Default SSL setting to False if missing
            log.warning('no IMAP SSL setting specified in configuration file')
            ssl = False
        except ValueError:
            log.critical('IMAP SSL setting invalid - not boolean')
            break

        # Retrieve IMAP folder to monitor - warn if missing
        try:
            folder = config.get('imap', 'folder')
        except configparser.NoOptionError:
            # Default folder to monitor to 'INBOX' if missing
            log.warning('no IMAP folder specified in configuration file')
            folder = 'INBOX'

        while True:
            # <--- Start of IMAP server connection loop

            # Attempt connection to IMAP server
            log.info('connecting to IMAP server - {0}'.format(host))
            try:
                server = IMAPClient(host, use_uid=True, ssl=ssl)
            except Exception:
                # If connection attempt to IMAP server fails, retry
                etype, evalue = sys.exc_info()[:2]
                estr = traceback.format_exception_only(etype, evalue)
                logstr = 'failed to connect to IMAP server - '
                for each in estr:
                    logstr += '{0}; '.format(each.strip('\n'))
                log.error(logstr)
                sleep(10)
                continue
            log.info('server connection established')

            # Attempt login to IMAP server
            log.info('logging in to IMAP server - {0}'.format(username))
            try:
                result = server.login(username, password)
                log.info('login successful - {0}'.format(result))
            except Exception:
                # Halt script when login fails
                etype, evalue = sys.exc_info()[:2]
                estr = traceback.format_exception_only(etype, evalue)
                logstr = 'failed to login to IMAP server - '
                for each in estr:
                    logstr += '{0}; '.format(each.strip('\n'))
                log.critical(logstr)
                break

            # Select IMAP folder to monitor
            log.info('selecting IMAP folder - {0}'.format(folder))
            try:
                result = server.select_folder(folder)
                log.info('folder selected')
            except Exception:
                # Halt script when folder selection fails
                etype, evalue = sys.exc_info()[:2]
                estr = traceback.format_exception_only(etype, evalue)
                logstr = 'failed to select IMAP folder - '
                for each in estr:
                    logstr += '{0}; '.format(each.strip('\n'))
                log.critical(logstr)
                break

            # retrieve unseen mails
            get_unseen_mails(server, log)

            try:
                while True:
                    # <--- Start of mail monitoring loop

                    # After all unread emails are cleared on initial login, start
                    # monitoring the folder for new email arrivals and process
                    # accordingly. Use the IDLE check combined with occassional NOOP
                    # to refresh. Should errors occur in this loop (due to loss of
                    # connection), return control to IMAP server connection loop to
                    # attempt restablishing connection instead of halting script.
                    log.info('Entering IDLE mode')
                    server.idle()

                    response = server.idle_check(idle_timeout)

                    if response:
                        server.idle_done()
                        get_unseen_mails(server, log)
                    else:
                        server.idle_done()
                        server.noop()
                        log.info('IDLE timed out, no new messages seen, sending NOOP')

                    # End of mail monitoring loop --->
                    continue
            except:
                # log error, most likely SSLSysCallError if
                # server decided we've camped too long and dropped
                etype, evalue = sys.exc_info()[:2]
                estr = traceback.format_exception_only(etype, evalue)
                logstr = 'Exception while staying in IDLE: '
                for each in estr:
                    logstr += '{0}; '.format(each.strip('\n'))
                log.critical(logstr)

            # End of IMAP server connection loop --->
            break

        # End of configuration section --->
        break
    log.info('script stopped ...')



def get_unseen_mails(server, log):
    # Retrieve and process all unread messages. Should errors occur due
    # to loss of connection, attempt restablishing connection
    try:
        messages = server.search([b'UNSEEN'])
    except Exception:
        pass

    log.info('%d unseen messages' % len(messages))

    # fetch all those messages and process them
    try:
        response = server.fetch(messages, ['FLAGS', 'RFC822'])
    except Exception:
        log.error('failed to fetch emails: ' + ', '.join(messages))
        pass

    for msgid, data in response.items():
        try:
            post_mail(msgid, data, log)
        except Exception:
            log.error('failed to process message %d' % msgid)
            continue


def post_mail(msgid, data, log_):
    """Email processing, mainly posting it to OSTicket.
        msgid is the UID of the message
        data is the dict fetched from server (FLAGS, RFC822)
        log_ is the logger object.
    """

    # params to post to OSTicket
    headers = {'Expect': '', 'X-API-Key': api_key}

    r = requests.post(post_url, data = data[b'RFC822'], headers=headers)
    if r.status_code == 201:
        log_.info('Posted msgId %d to OSTicket ... ' % msgid)
    else:
        log_.error('Failed to post msgId %d to OSTicket (status_code %d)' % (msgid, r.status_code))


if __name__ == '__main__':
    main()