Skip to content

Instantly share code, notes, and snippets.

@ajinabraham
Last active May 15, 2020 00:02
Show Gist options
  • Save ajinabraham/da501def7cad4ff79ca65ebd9d1ed689 to your computer and use it in GitHub Desktop.
Save ajinabraham/da501def7cad4ff79ca65ebd9d1ed689 to your computer and use it in GitHub Desktop.
import time
import requests
user = ''
password = ''
session = requests.Session()
session.auth = (user, password)
url= 'https://api.github.com/search/repositories?language:javascript&sort=stars&q=topic:nodejs+topic:express'
i = 1
repos = set()
def dump_repo(url):
global i, session
wait = 60
res = session.get(url)
limit = res.headers['X-Ratelimit-Remaining']
print(f'Request Number: {i}')
print(f'Remaining Quota: {limit}')
if 'first 1000 search results' in res.json().get('message', ''):
return
if limit == '0' or 'rate limit exceeded' in res.json().get('message', ''):
print(f'Github Ratelimit hit!, waiting for {wait} sec')
time.sleep(wait)
dump_repo(url)
for item in res.json().get('items', []):
repos.add(item['html_url'])
i += 1
dump_repo(f'{url}&page={i}')
dump_repo(url)
for i in repos:
print(i)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment