Skip to content

Instantly share code, notes, and snippets.

@malcolmgreaves
Created September 3, 2019 20:18
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save malcolmgreaves/23676397ca1c067fd03812347f4ed5d5 to your computer and use it in GitHub Desktop.
Function to load JSON objects as a stream from a local filepath. Works with JSON list and JSON object per newline.
import json
from pathlib import Path
from typing import Any, Dict, Iterable
def load_json(json_file_path: Path) -> Iterable[Dict[str, Any]]:
"""Loads a sequence of JSON objects: either a JSON list or a newline-separated list of objects.
:raises TypeError If the
"""
with open(str(json_file_path), "rt") as r:
first_char = peek_read(1, r)
if first_char == "[":
# file is a single JSON list
try:
json_list: Sequence[Dict[str, Any]] = json.load(rt)
except JSONDecodeError as e:
raise JSONDecodeError(f"File not valid JSON: {json_file_path}") from e
for j in json_list:
yield j
else:
# object per newline
json_list = []
for i, line in enumerate(r):
try:
j = json.loads(line)
except JSONDecodeError as e:
raise JSONDecodeError(f"The {i + 1}th line of {json_file_path} "
"is not a valid JSON object") from e
yield j
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment