Batch grading script

batch_grade.py

"""
This script clones students' github repos with HW solutions, checks out a commit specified by the student,
and runs corresponding tests, generating a directory with reports.

Just place the script into the root of cloned "tests" repo and change global variables below appropriately.
Then run as "python batch_grade.py"

"""


import os
import shutil
import subprocess
import sys


# parameters assume this is start from the root of "tests" repo
import time
from os.path import join

HW_NAME = 'hw5'
HW_TESTS = f'./{HW_NAME}'
REPORT_DIR = './report'
JSON_DIR = join(REPORT_DIR, 'details')
MD_DIR = join(REPORT_DIR, 'formatted')

SOLUTIONS_DIR = './solutions'

GITHUB = 'https://github.com/usc-csci104-spring2021/'


STUDENTS = [
    'csowers',
    'oclavijo',
    'gracesha',
    'liangang',
    'sangaraj',
    'adeline',
    'alexanxo',
    'wytong',
    'sxfan',
    'sbogert',
    'bulusu',
    'nykim',
    'kamakian',
    'bgbell',
    'eddiel',
    'cblanton',
    'skawaguc',
    'jasongao',
    'mabdulwa',
    'sbusa',
    'cheungje',
    'ishuagra',
    'amchen',
    'sbadams',
    'farokhha',
    'amkhan',
    'kingzhou',
    'mihirsin',
    'jiwonc',
    'sbrownle',
    'mehakchh',
    'costroff',
]

GIT_COMMITS = [
    'a424ceede5905240b3e5e8b9318a746bccb63bea',
    '276b8a278df9f4d0879646d9ef3dcb3f7892998c',
    '',
    '6383bde553cc0427a3bd7ecd6738b7bb900fb3b9',
    '4fea767680c1610f4b01019bcd9047aa691f19ef',
    'cd0deec81d72509ef52bd52b779e2f07ad1c7296',
    '89a80fbca98c3a3582bfd56a686f9d0aa68a0d91',
    '78d7de6dcc1bec5eb176b798629dbf661dfad49a',
    '',
    '29735b080483b4014a8bc8a14b0da83453662e6a',
    'bfec92f3826f3b73e99b6460aa33c5bcd434c15c',
    '',
    '3c7e371c4ec82d4dadf163234f828ec6e95aed26',
    '',
    'fc66f3efdcdf708465050c4a75f5835cbf924a78',
    '4be2286e67ba4ce7981a3e6c212a0dfa926b565d',
    '9c09a7d4ff43d9c957f470121062f1d2e737712a',
    '13fc0ef90762b803b814b562f4e6480309162724',
    '92da7f5b0a5a7a20d9de6107c20727504c15c185',
    '31ee47aedeae1c7a7cba80a361f1157f6c779f1c',
    'e95e0a0e56cfc75874996b823a0a710ba377dc94',
    '',
    '',
    '2c76fbf69181059404fe687a258099450240c739',
    '4ec78720c9a2cfb7e66358bf423fab92333a223f',
    '885ad52212282e3f953a92f84365786c9bea14d5',
    'e8af39c202f52da42d96efd8c4fea38be1dbdfa6',
    '7c9a996c05761340180df99fb05bcc16618651b1',
    '',
    '5e4e265befc2dea972bb53b598fd402c3f83955b',
    '59583f0660b523353e94eb14878c9973b48406cc',
    '',
]


def ensure_dir_exists(path):
    if not os.path.exists(path):
        os.makedirs(path)
    return path


def main():
    """Script entry point."""
    ensure_dir_exists(SOLUTIONS_DIR)
    ensure_dir_exists(JSON_DIR)
    ensure_dir_exists(MD_DIR)

    assert len(STUDENTS) == len(GIT_COMMITS)
    for i, student in enumerate(STUDENTS):
        # clone the git repos
        repo_name = f'hw-{student}'
        repo_path = join(SOLUTIONS_DIR, repo_name)
        if not os.path.exists(repo_path):
            git_args = [
                'git', 'clone', f'{GITHUB}{repo_name}.git', repo_path
            ]
            print(f'Cloning {repo_path}...')
            out = subprocess.run(git_args)
            time.sleep(0.1)
            returncode = out.returncode
            print(f'Result code: {returncode}')

        git_hash = GIT_COMMITS[i]
        if git_hash:
            git_args = [
                'git', 'checkout', git_hash,
            ]

            print(f'Checking out commit {git_hash}...')
            out = subprocess.run(git_args, cwd=join(repo_path))
            time.sleep(0.1)
            returncode = out.returncode
            print(f'Result code: {returncode}')

    for student in STUDENTS:
        solution = join(SOLUTIONS_DIR, f'hw-{student}', HW_NAME)
        proc_args = [
            sys.executable, '-m',
            'curricula', 'grade', 'run', '--progress', '--summarize', '--format', '-d', REPORT_DIR, '-g', HW_TESTS, solution,
        ]

        print(f'Testing {solution}...')

        out = subprocess.run(proc_args)
        time.sleep(0.1)
        returncode = out.returncode
        print(f'Result code: {returncode}')

        # the report is always named "hw#.report.md". Here we rename and put them into different folders
        shutil.move(join(REPORT_DIR, f'{HW_NAME}.report.json'), join(JSON_DIR, f'{student}.report.json'))

        formatted_report_fname = join(MD_DIR, f'{student}.report.md')
        shutil.move(join(REPORT_DIR, f'{HW_NAME}.report.md'), formatted_report_fname)

        with open(formatted_report_fname) as f:
            lines = f.read().splitlines()
            for l in lines:
                if 'Automated tests' in l:
                    print(f'Result: {l}')

    return 0


if __name__ == '__main__':
    sys.exit(main())