Fetch Reddit Pushshift

fetch_reddit_pushshift.py

from dateutil.relativedelta import relativedelta
from datetime import date, datetime
import json
import requests
import calendar
import time
import sys

end = date.today()
start = end - relativedelta(years=3)

# convert to utc seconds
end = calendar.timegm(end.timetuple())
start = calendar.timegm(start.timetuple())

n = 500
delta = (end-start)//n

subreddit = 'amitheasshole'
for i in range(n):
  while True: # handle 502
    url = 'https://api.pushshift.io/reddit/submission/search/?after={}&before={}&sort_type=score&sort=desc&subreddit={}'.format(
      (i*delta) + start,
      ((i+1)*delta) + start,
      subreddit,
    )
    r = requests.get(url)
    j = r.json()
    with open('{}-{}.json'.format(subreddit, i), 'w') as out:
      json.dump(j, out)
    print(i, n, url)
    time.sleep(2)
    if 'data' in j:
      print(len(j['data']))
      break