codecrawl.py

__author__ = 'leonrenkema'

from urllib import request
from http.client import HTTPException
from json import loads
from os import system
from os.path import isdir

class GitHubApi():
    def get_repositories(self, page):
        query = "https://api.github.com/search/repositories?q=language:PHP&page=%s" % page

        try:
            response = request.urlopen(query)
            print(response.info()['X-RateLimit-Remaining'])
            print(response.info()['Link'])
            return loads(response.read().decode('utf-8'))

        except HTTPException:
            print("error")

api = GitHubApi()

for page in range(1, 10):
    repositories = api.get_repositories(page)
    print("total count %s" % repositories['total_count'])

    for repo in repositories['items']:

        if not isdir(repo['full_name']):
            command = "git clone %s %s" % (repo['clone_url'], repo['full_name'])
            system(command)