Retrieves log files from given repositories' Issues for offline/batch processing/searching

collect_vita3k_logs.py

"""
Retrieves log files from given ("REPO_NAMES") repositories
for offline/batch processing/searching.

Iterates thought all issues, finds the last posted comment
that contains at least one file and downloads all files to
appropriately named directories, with the filename being
the Issue title (game name/titleID).
Replaces invalid characters in that file name with '-' and
if there are multiple files in an issue, appends the
filenames with "_N" where "N" is a number starting from 0.

Author: VelocityRa
Date: 18/8/2018
"""

from github import *
import re
import os
import os.path
import urllib.request

# Enter your GitHub token here
GITHUB_TOKEN = ""
REPO_NAMES = ["Vita3K/compatibility", "Vita3K/homebrew-compatibility"]
LOGS_BASE_PATH = "logs"


def find_log_files(comment):
    return log_files_re.findall(comment)


def fixup_log_file_paths(log_files, repo_name):
    return [r"https://github.com/" + repo_name + "/" + log_file for log_file in log_files]


def normalize_file_name(file_name):
    return file_name.replace(":", "-").replace(r"/", "-").replace(r"\\", "-").replace("*", "-")

try:
    log_files_re = re.compile(r"/files/\d+/\w+\.\w+")

    logs_path = os.path.normpath(os.path.join(os.getcwd(), LOGS_BASE_PATH))
    
    if not os.path.exists(logs_path):
        os.mkdir(logs_path)

    # enable_console_debug_logging()
    g = Github(login_or_token=GITHUB_TOKEN)

    for repo_full_name in REPO_NAMES:
        print("Getting logs for repo: {}".format(repo_full_name))

        (_, repo_name) = os.path.split(repo_full_name)
        log_base_path = os.path.join(logs_path, repo_name)
        if not os.path.exists(log_base_path):
            os.mkdir(log_base_path)

        repo = g.get_repo(repo_full_name, lazy=False)
        issues = repo.get_issues()

        for issue in issues:
            comments_num = issue.comments

            logs_posted = []
            if comments_num == 0:
                last_comment = issue.body
            else:
                last_comment = issue.get_comments()[comments_num - 1].body

                # if on logs are found on last message, revert to main issue
                logs_posted = find_log_files(last_comment)
                if len(logs_posted) == 0:
                    last_comment = issue.body

            if len(logs_posted) == 0:
                logs_posted = find_log_files(last_comment)

            logs_posted = fixup_log_file_paths(logs_posted, repo_full_name)

            print("Issue #{} for game {}".format(issue.id, issue.title))
            for log_id, log in enumerate(logs_posted):
                normalized_file_name = normalize_file_name(issue.title)
                # name multiple files in a comment appropriately
                if len(logs_posted) > 1:
                    normalized_file_name += "_" + str(log_id)
                    
                normalized_file_name += ".log"

                log_full_path_cur = os.path.join(log_base_path, normalized_file_name)

                print("Retrieving: {} as {}".format(log, normalized_file_name))
                urllib.request.urlretrieve(log, log_full_path_cur)

except GithubException as e:
    print(e)