NER_dataviz.py

import spacy
from collections import Counter, defaultdict
import srsly
from datetime import datetime
import pandas as pd

counts = defaultdict(Counter)
nlp = spacy.load(SPACY_MODEL)
data = srsly.read_jsonl(DATA_FILE)

data_tuples = ((eg["text"], eg) for eg in data)
for doc, eg in nlp.pipe(data_tuples, as_tuples=True, n_process=N_PROCESSES):
    timestamp = int(eg["meta"]["utc"])
    year_month = datetime.utcfromtimestamp(timestamp).strftime("%Y-%m")
    for ent in doc.ents:
        if ent.label_ == ENTITY_LABEL:
            counts[ent.lower_][year_month] += 1

df = pd.DataFrame(data=counts).transpose()
df.to_csv(OUTPUT_FILE)