dump_mongo.py

from mongo import *
from api.maps import *
from api.poster import *
import json as simple_json
from bson import json_util

from datetime import datetime

def json_serial(obj):
    if isinstance(obj, datetime):
        res = str(obj)
        return res
    else:
        return json_util.default(obj)

with open('startups.json','w') as out:
    for doc in startups.find({'hidden':False}):
        del doc['_id']
        del doc['status']
        if 'angellist_id' in doc:
            doc['angellist_id'] = int(doc['angellist_id'])
        out.write(simple_json.dumps(doc, default=json_serial) + '\n')

with open('jobs.json', 'w') as out:
    for page in jobs.find():
        for job in page['jobs']:
            out.write(simple_json.dumps(job, default=json_serial) + '\n')

with open('users.json', 'w') as out:
    for doc in users.find():
        del doc['_id']
        out.write(simple_json.dumps(doc, default=json_serial) + '\n')

with open('followers.json','w') as out:
    for company in followers.find({'users': {'$exists': True}}):
        for user in company['users']:
            out.write(simple_json.dumps({'company_id': company['id'], 'person_id': user['id']}) + '\n')

with open('roles.json', 'w') as out:
    for company in startup_roles.find({'startup_roles': { '$exists': True }}):
        try:
            for role in company['startup_roles']:
                role_data = {
                    'company_id': company['id'],
                    'tag_id': role['tagged']['id'],
                    'tag_type': role['tagged']['type'],
                    'created_at': role['created_at'],
                    'ended_at': role['ended_at'],
                    'confirmed': role['confirmed'],
                    'title': role['title'],
                    'role': role['role']
                }
                out.write(simple_json.dumps(role_data)+'\n')
        except KeyError:
            print >> sys.stderr, repr(company)