nesco_sorcerer_backfill.py

__author__ = 'SpacedOut Engineering'
__copyright__ = 'Copyright 2019, Spaced Out, Inc'

import os
import requests
from boltons.jsonutils import JSONLIterator
from boltons.iterutils import chunked_iter
from pathlib import Path

# Directory with the jobdiva cached files
CACHED_FILES_DIR = os.getenv('CACHED_FILES_DIR')

# Sorcerer URL
SORCERER_URL = os.getenv("SORCERER_URL")

# Agency ID
AGENCY_ID = os.getenv("AGENCY_ID")

BATCH_SIZE = os.getenv("BATCH_SIZE", 500)

for file in Path(CACHED_FILES_DIR).glob("*.json.txt"):
    entity_type = file.name.split(".")[0].upper()
    for batch in chunked_iter(JSONLIterator(file.open('r')), BATCH_SIZE):
        batch_payloads = [{'entity_type': entity_type,
                           'entity_source': 'jobdiva',
                           'entity_id': b['ID'],
                           'payload': b} for b in batch]
        params = {
            'post_to_eu': False,
            'entity_data': batch_payloads,
        }
        requests.post(f"{SORCERER_URL}/v1/entity", json=params)