Download messages matching a search using Python and imapclient

fetch_gmail_messages.py

# -*- coding: utf-8 -*-

"""Code to fetch Gmail messages matching a particular pattern."""

"""
Run:
python -m imapclient.interact -H imap.gmail.com --ssl -P 993

Then in the interactive session, run:
%run <this file>
dump_all_messages_matching_search(c, search, shelf_fn)
"""

import itertools
import os
import sys
sys.path.append(os.path.expanduser('~/code/python'))
import ib.shelve


def split_into_batches(iter, batchsize):
    """Split an iterator into batches of a given size. Each batch is returned
    as a list. The last batch may have fewer elements."""
    iter = iter.__iter__()
    while True:
        res = list(itertools.islice(iter, batchsize))
        if len(res):
            yield res
        else:
            return


def store_messages(c, messages, shelf, parts=['FLAGS', 'INTERNALDATE', 'RFC822'], batch_size=10):
    print 'Started'
    missing_indices = shelf.get('missing_indices', list())

    n_done = 0
    for message_batch in split_into_batches(messages, batch_size):
        res = c.fetch(message_batch, parts)
        this_missing_indices = sorted(set(message_batch) - set(res.keys()))
        if this_missing_indices:
            print 'Missing indices:', this_missing_indices

        if this_missing_indices:
            missing_indices += this_missing_indices
            shelf['missing_indices'] = missing_indices

        for k, v in res.iteritems():
            shelf[str(k)] = v

        n_done += len(message_batch)
        print 'Finished {0} of {1} messages'.format(n_done, len(messages))


def store_search_results(c, search, shelf):
    messages = c.search(search)
    store_messages(c, messages, shelf)


def dump_all_messages_matching_search(c, search, shelf_fn, folder='[Gmail]/All Mail'):
    # Note that a search for 'ALL' should return all messages.
    c.select_folder(folder)
    with ib.shelve.open(shelf_fn) as s:
        store_search_results(c, search, s)