Class providing methods and context manager to efficiently read and write list-based json documents.

jsonlistfile.py

import json

class JsonListFile(object):
    """Class providing methods and context manager to read and write list-based
    json documents.
    This class will write out to the file on each append (maybe, os-dependent), 
    eliminating the need to hold the entire json object in memory before writing 
    (as is necessary with the plain stdlib json module).
    Primitive, but useful."""
                
    def __init__(self, filename):
        self.filename = filename        
        self.json_file = None
        self.empty = True

    def _open(self, for_write=False):
        self.json_file = open(self.filename, 'w' if for_write else 'r')
        if for_write:
            self.json_file.write('[\n')

    def close(self):
        if not self.json_file.closed:
            if 'w' in self.json_file.mode:
                self.json_file.write(']\n')
            self.json_file.close()

    def __del__(self):
        self.close()

    def __iter__(self):
        """Lazily iterate through encoded docs in file"""
        if not self.json_file:
            self._open()
        for line in self.json_file:
            if len(line) > 2:
                yield json.loads(line)

    def append(self, thing):
        if not self.json_file:
            self._open(for_write=True)

        json_thing = json.dumps(thing)
        if not self.empty:
            self.json_file.write(',\n')
        self.json_file.writelines([json_thing, '\n'])
        self.empty = False

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.close()