Skip to content

Instantly share code, notes, and snippets.

Last active September 9, 2016 04:06
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
What would you like to do?
import urllib.request
import json
from pymongo import MongoClient
import time
def fetchAndStash(url, client):
response = urllib.request.urlopen(url + '&count=100')
f =
payload = f.decode('utf-8')
j = json.loads(payload)
if j['meta']['code'] == 200 : # Request worked
for i in j['data']: # Loop through Instagram posts and save them
# Hardcoded Database/Collection names.. change this
result = client.DeepDress.InstagramV4.insert({'_id' : i['link'], 'payload' : i, 'image' : i['images']['low_resolution']['url']})
print("bad batch url: " + url + '; tried to insert ' + i)
with open('error', 'a') as of:
time.sleep(1) # Be nice
fetchAndStash(j['pagination']['next_url'], client) # Follow the white rabbit
else :
print("Failed url ", url)
def main():
client = MongoClient()
url = ''
fetchAndStash(url, client) # could hit stack overflow.. but unlikely
if __name__ == "__main__":
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment