file_chunks.py

#!/usr/bin/env python

import sys
import json

CHUNK_SIZE = 500

def chunks(l, n):
    """ Yield successive n-sized chunks from array l
    """
    for i in xrange(0, len(l), n):
        yield l[i:i+n]

def file_chunks(file_lines, n):
    """ Yield successive n-sized chunks from file-like object
    """
    arr = []
    for line in file_lines:
        arr.append(line)
        if len(arr)>=n:
            yield arr
            arr = []
    if len(arr):
        yield arr

with open('duns-2.json', 'rb') as f:
    duns_list = json.load(f)
print len(duns_list)

duns_arr = [e[0] + '\n' for e in duns_list]
with open('duns-2.txt', 'wb+') as f:
    for line in duns_arr:
        f.writelines(line)


s = 0
with open('duns-2.txt', 'rb') as f:
    for chunk in file_chunks(f, CHUNK_SIZE):
        s += len(chunk)
print s

s = 0
with open('duns-2.txt', 'rb') as f:
    for idx, chunk in enumerate(file_chunks(f, CHUNK_SIZE)):
        s += len(chunk)
print s

s = 0
for chunk in chunks(duns_list, CHUNK_SIZE):
    s += len(chunk)
print s