Incorporating Multiprocessing

get_social_stats.py

import psycopg2
from socialanalytics import pinterest
from socialanalytics import facebook
from socialanalytics import twitter
from socialanalytics import google_plus
from time import strftime, sleep

def getSocialStats(url):
    p = pinterest.getPins(url)
    f = facebook.getObject(url)
    t = twitter.getShares(url)
    g = google_plus.getPlusOnes(url)
    return [p, f, t, g]
    
def saveToDB(stas):
    # Save results
    try:
        now = strftime("%Y-%m-%d %H:%M:%S")
        cur.execute("INSERT INTO ...")
        conn.commit()
    except:
        conn.rollback()
        

conn = psycopg2.connect("dbname='***' user='***' host='***' password='***'")
cur = conn.cursor()

# Select all URLs
cur.execute("SELECT * FROM urls;")
urls = cur.fetchall() # 160,000+

# 1) Span 10 or more processes
# 2) Each process takes a url from urls
# 3) Calls getSocialStats(url)
# 4) Calls saveToDB(data)
# 5) Moves on to the next url