Latest update (currently on server)

bagnowka_dump_to_mongo.py

#!/usr/bin/env python
from pymongo import MongoClient
import os
import requests

client = MongoClient(os.environ.get("MONGO_HOST", "localhost"), int(os.environ.get("MONGO_PORT", "27017")))
db = client[os.environ["MONGO_DB"]]
photoUnits = db['photoUnits']

data = requests.get("https://raw.githubusercontent.com/Beit-Hatfutsot/dbs-bagnowka-scrape/master/bagnowka_all.json").json()

count = 0
photoUnits.delete_many({"bagnowka": "True"})
for slug, doc in data.items():
    try:
        for k in doc:
            if "He" in doc[k]:
                del doc[k]["He"]

        photoUnits.insert_one(data[slug])
        count += 1
        print("1 item was added to photoUnits")
    except Exception:
        print("unexpected exception processing slug {} count {}".format(slug, count))
        raise

print("{} items were inserted to photoUnits.".format(count))