solr_export_ijson.ipynb

test_stream_1.py

import urllib.parse
import urllib.request
import ijson

def stream_records(q:str):
    fields = [
        "id",
        "source",
        "label",
        "hasMaterialCategory",
        "hasSpecimenCategory",
        "hasContextCategory",
        "keywords",
        "producedBy_label",
        "producedBy_description",
        "producedBy_responsibility",
        "producedBy_resultTime",
        "producedBy_samplingSite_label",
        "producedBy_samplingSite_location_latitude",
        "producedBy_samplingSite_location_longitude",
        "curation_responsibility",
        "relatedResource_isb_core_id",
    ]
    
    params = {
        "q":q,
        "sort":"id asc",
        "fl": ",".join(fields),
        "wt":"json"
    }
    url = "http://localhost:8983/solr/isb_core_records/export"
    url = f"{url}?{urllib.parse.urlencode(params)}"
    print(url)
    src = urllib.request.urlopen(url)
    docs = ijson.items(src, "response.docs.item")
    counter = 0
    for doc in docs:
        if counter % 10000 == 0:
            print(f"{counter:07}")
        counter += 1
    print(f"Total = {counter}")


q = "source:GEOME"
stream_records(q)