PushShift.py

import pandas as pd
import requests
import json

def getPushshiftData(after, sub):
    url = 'https://api.pushshift.io/reddit/search/submission?&size=1000&after='+str(after)+'&subreddit='+str(sub)
    r = requests.get(url)
    data = json.loads(r.text)
    return data['data']

#list of post ID's
post_ids = []
#Subreddit to query
sub='btc'
# Unix timestamp of date to crawl from.
# 2018/04/01
after = "1522618956"

data = getPushshiftData(after, sub)

# Will run until all posts have been gathered 
# from the 'after' date up until todays date
while len(data) > 0:
    for submission in data:
        post_ids.append(submission["id"])
    # Calls getPushshiftData() with the created date of the last submission
    data = getPushshiftData(sub=sub, after=data[-1]['created_utc'])


obj = {}
obj['sub'] = sub
obj['id'] = post_ids
# Save to json for later use
with open("submissions.json", "w") as jsonFile:
    json.dump(obj, jsonFile)

Hey,
I was just going through your code, can you please let me know what is the size parameter in the above code in the line(6) url.

Hey,
I was just going through your code, can you please let me know what is the size parameter in the above code in the line(6) url.

Size is "limit of returned entries"