Retrieves domains from the database and tries to get records for each entry and updates the database when neccessary

extract_records.py

#!/usr/bin/env python

from dns import resolver
from dns.name import EmptyLabel
from dns.resolver import NoAnswer
from dns.resolver import NXDOMAIN
from pymongo import MongoClient
from pymongo.errors import DuplicateKeyError


def get_connection(database_url):
    client = MongoClient(database_url)
    db = client['certs']
    return db


def update_data(db, item, post):
    try:
        return db.domains.update_one({'_id': item['_id']}, {'$set': post}, upsert=False)
    except DuplicateKeyError as e:
        return


def get_data(db, skip, limit):
    data = db.domains.find({})[skip:limit]
    return data


def get_address(domain, record):
    address_list = []

    try:
        data = resolver.query(domain, record)

        for item in data:
            if record != 'MX' and record != 'CNAME':
                address_list.append(item.address)

            elif record == 'CNAME':
                post = {'target': item.target.to_unicode()}
                address_list.append(post)

            else:
                post = {'preference': item.preference, 'exchange': item.exchange.to_unicode()}
                address_list.append(post)

        return address_list
    except (EmptyLabel, NoAnswer, NXDOMAIN) as e:
        return


def main():
    db = get_connection('localhost:27017')

    skip = 0
    limit = 1000

    known_docs = get_data(db, skip, limit)

    for item in known_docs:
        a_record = get_address(item['domain'], 'A')
        mx_record = get_address(item['domain'], 'MX')
        aaaa_record = get_address(item['domain'], 'AAAA')
        cname_record = get_address(item['domain'], 'CNAME')

        if a_record:
            data = update_data(db, item, {'a_record': a_record})
            print 'INFO: updated {}, with {} documents'.format(item['domain'], data.modified_count)

        if aaaa_record:
            data = update_data(db, item, {'aaaa_record': aaaa_record})
            print 'INFO: updated {}, with {} documents'.format(item['domain'], data.modified_count)

        if mx_record:
            data = update_data(db, item, {'mx_record': mx_record})
            print 'INFO: updated {}, with {} documents'.format(item['domain'], data.modified_count)

        if cname_record:
            data = update_data(db, item, {'cname_record': cname_record})
            print 'INFO: updated {}, with {} documents'.format(item['domain'], data.modified_count)


if __name__ == '__main__':
    main()