Skip to content

Instantly share code, notes, and snippets.

/error.txt Secret

Created November 10, 2014 04:05
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/1db5b8d29ba78aea174a to your computer and use it in GitHub Desktop.
Save anonymous/1db5b8d29ba78aea174a to your computer and use it in GitHub Desktop.
Error after 6th iteration, ran out of RAM
OSError: [Errno 12] Cannot allocate memory
import math
import psycopg2
from socialanalytics import pinterest
from socialanalytics import facebook
from socialanalytics import twitter
from socialanalytics import google_plus
from time import strftime, sleep
import multiprocessing as mp
NUM_PROCESSES = 10
NUM_URLS = 1000
print "Number of Processes:", NUM_PROCESSES
conn = psycopg2.connect("dbname='***' user='***' host='***' password='***'")
cur = conn.cursor()
def getSocialStats(url):
now = strftime("%Y-%m-%d %H:%M:%S")
p = pinterest.getPins(url[1])
f = facebook.getObject(url[1])
t = twitter.getShares(url[1])
g = google_plus.getPlusOnes(url[1])
return [url[0], url[1], now, p['pin_count'], f['like_count'], f['share_count'], f['comment_count'], t['share_count'], g['plus_count']]
# Get Count of urls
cur.execute("SELECT COUNT(*) FROM urls;")
COUNT_URLS = float(cur.fetchone()[0])
SELECT_CYCLES = math.ceil(COUNT_URLS / NUM_URLS)
for i in range(int(SELECT_CYCLES)):
offset = i * NUM_URLS
# Select urls
cur.execute("SELECT id, url FROM urls LIMIT " + str(NUM_URLS) + " OFFSET " + str(offset) + ";")
urls = cur.fetchall()
url_chunks = []
temp = []
# Parse URLs into chunks
while len(urls) > 0:
temp.append(urls.pop())
if len(temp) == NUM_PROCESSES:
url_chunks.append(temp)
temp = []
# Add any remaining urls
if len(temp) > 0:
url_chunks.append(temp)
p = mp.Pool(NUM_PROCESSES)
res = []
print "Starting Fetching", strftime("%Y-%m-%d %H:%M:%S")
for url_chunk in url_chunks:
res += p.map(getSocialStats, url_chunk)
for r in res:
try:
cur.execute("INSERT INTO social_stats_staging (url_id, fetched_at, pinterest_pins, facebook_likes, facebook_shares, facebook_comments, twitter_shares, google_plus_ones) VALUES(%s, %s, %s, %s, %s, %s, %s, %s);", (r[0], r[2], r[3], r[4], r[5], r[6], r[7], r[8]))
conn.commit()
except:
conn.rollback()
print "Error saving to DB"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment