scrap_python_from_gh.py

# Video link: https://youtu.be/3P3TcKaegbA?list=PLxvWWIq3F0k0GE1EYvricLniNSdv9gvZS

import os
import time
from datetime import datetime
from github import Github
from curtsies.fmtfuncs import yellow

ACCESS_TOKEN = open('token', 'r').read()
g = Github(ACCESS_TOKEN)

end_time = time.time()
start_time = end_time - 86400

for i in range(5):
    start_time_str = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%d')
    end_time_str = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%d')
    end_time -= 86400
    start_time -= 86400

    query = f'language:python created:{start_time_str}..{end_time_str}'
    result = g.search_repositories(query)

    for repo in result:
        contents = repo.get_contents("")
        try:
            while contents:
                file_content = contents.pop(0)
                if file_content.type == "dir":
                    contents.extend(repo.get_contents(file_content.path))
                elif file_content.path.endswith('.py'):
                    url = file_content.download_url
                    print(f'[+] {url}')
                    os.system(f'wget -q {url} -P python_files/')
        except:
            print(yellow('Going to get some sleep :)'))
            time.sleep(60)