example code for http://stackoverflow.com/questions/24952918/

stackoverflow24952918.py

import sys
import json
import urllib2
import multiprocessing as mp

def worker(url):
    """ reading lots of data """
    print "# Process %s running on file %s"%(mp.current_process(), url)
    raw = json.load(urllib2.urlopen(url))
    data = {}
    for r in raw:
        data[r] = raw[r]
    return data

# example urls to read
url = "http://eu.battle.net/auction-data/258993a3c6b974ef3e6f22ea6f822720/auctions.json"
urls_to_parse = (url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url, url)

# multiprocessing options
nproc = mp.cpu_count()
pool = mp.Pool(processes=nproc)
parsed = pool.map(worker, urls_to_parse)
pool.close()