gistfile1.txt

from disco.core import Job
from disco.worker.classic.func import chain_reader

import re

download_pattern = re.compile("{ _id: ObjectId\('([^']*)'\), d: ([^,]*), doi: \"([^\"]*)\", i: \"([^\"]*)\", s: ([^,]*), ip: \"([^\"]*)\" }")

class ParseDownloads(Job):
    map_reader = staticmethod(chain_reader)

    @staticmethod
    def map(line, params):
        match = poc.download_pattern.match(line)
        if match:
            (id, date, doi, _, _, ip) = match.groups()
            yield id, json.dumps({'id':id, 'doi':doi, 'date':date, 'ip':ip})
        else:
            Task.put('nomatch', line)