Skip to content

Instantly share code, notes, and snippets.

@dreizehnutters
Created January 8, 2020 11:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dreizehnutters/6ac65948b506fbcd9f67ad0dd11ce50a to your computer and use it in GitHub Desktop.
Save dreizehnutters/6ac65948b506fbcd9f67ad0dd11ce50a to your computer and use it in GitHub Desktop.
scrape github gists
"""scrape github gists"""
#!/usr/bin/env python
from sys import argv
from json import load
from urllib import request
from subprocess import call
GITHUB = "https://api.github.com/users/"
USER = argv[1]
call(['mkdir', '-p', USER])
COUNT = 0
PERPAGE = 100
GISTCOUNT = load(request.urlopen(f"{GITHUB}{USER}"))['public_gists']
PAGES = (GISTCOUNT//PERPAGE)+2
print(f"Found {GISTCOUNT} gists in {PAGES-1} pages")
for page in range(1, PAGES):
for gist in load(request.urlopen(f"{GITHUB}{USER}/gists?page={page}&per_page={PERPAGE}")):
COUNT += 1
gistd = list(gist['files'].keys())[0]
gistUrl = f"git://gist.github.com/{gist['id']}.git"
path = f"{USER}/{gist['id']}"
print(f"[*] cloning {gistUrl} from page {page} [{COUNT}/{GISTCOUNT}]")
call(['git', 'clone', '--quiet', gistUrl, path])
call(['mv', path, f"{USER}/{gistd}"])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment