Load Gzipped JSON Lines generated by Spark into Pandas

pandas.py

import pandas as pd
import numpy as np
import glob

pd.set_option('display.max_columns', 500)

all_files = glob.glob('../data/patent_applications/2019-04-07.jsonl.gz/part-*.json.gz')
li = []

for filename in all_files:
    df = pd.read_json(
        filename,
        lines=True,
        compression='gzip'
    )
    
    li.append(df)

patents = pd.concat(li, axis=0, ignore_index=True)

patents['patent_index'] = patents.index

print('Patent records: {:,}'.format(len(patents)))

patents = patents[['patent_index', 'application_id', 'app_date', 'title', 'abstract', 'description', 'granted']]
patents.head(5)