Created
September 3, 2019 20:18
Star
You must be signed in to star a gist
Function to load JSON objects as a stream from a local filepath. Works with JSON list and JSON object per newline.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
from pathlib import Path | |
from typing import Any, Dict, Iterable | |
def load_json(json_file_path: Path) -> Iterable[Dict[str, Any]]: | |
"""Loads a sequence of JSON objects: either a JSON list or a newline-separated list of objects. | |
:raises TypeError If the | |
""" | |
with open(str(json_file_path), "rt") as r: | |
first_char = peek_read(1, r) | |
if first_char == "[": | |
# file is a single JSON list | |
try: | |
json_list: Sequence[Dict[str, Any]] = json.load(rt) | |
except JSONDecodeError as e: | |
raise JSONDecodeError(f"File not valid JSON: {json_file_path}") from e | |
for j in json_list: | |
yield j | |
else: | |
# object per newline | |
json_list = [] | |
for i, line in enumerate(r): | |
try: | |
j = json.loads(line) | |
except JSONDecodeError as e: | |
raise JSONDecodeError(f"The {i + 1}th line of {json_file_path} " | |
"is not a valid JSON object") from e | |
yield j |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment