[prologin] Test all problems on camisole

test-all-problems.py

import django
django.setup()

from problems.models import Challenge
from prologin.languages import Language
import requests
from pathlib import Path

def get_in_out(p, tests, name):
    refs = Path(p.file_path()).rglob('ref.*')
    if not refs:
        return
    for r in sorted(refs):
        lang = Language.guess(r.suffix)
        if not lang: continue
        if lang is Language.python2: continue
        code = r.open().read()
        try:
            yield (p, name, code, lang, tests[name + '.in'], tests[name + '.out'])
        except KeyError:
            pass

def all_tests():
    for chal in sorted(Challenge.all(), key=lambda c: c.name):
        for p in sorted(chal.problems, key=lambda p: p.name):
            tests = p.tests
            for name in p.correction_tests:
                yield from get_in_out(p, tests, name)
            for name in p.performance_tests:
                yield from get_in_out(p, tests, name)

def run():
    for p, name, source, lang, _in, _out in all_tests():
        l = lang.name_display().lower()
        print(repr(p), name, l)
        rep = requests.post('http://camisole:8080/run', json={
            'lang': l,
            'execute': {'mem': int(1e7), 'time': 20, 'wall-time': 60, 'fsize': 4000},
            'compile': {'mem': int(1e7), 'time': 20, 'wall-time': 60, 'fsize': 4000},
            'source': source,
            'tests': [{'name': name, 'stdin': _in}]
        })
        if not rep.ok:
            print("", "NOPE", "http", rep, sep="\t")
            continue
        data = rep.json()
        try:
            test = data['tests'][0] 
            out = test['stdout'].strip()
            if out == _out.strip():
                print("", "OK", test['meta'], sep="\t")
            else:
                print("", "NOPE", "bad-stdout", out, sep="\t")
        except (KeyError, IndexError):
            print("", "NOPE", "error", data, sep="\t")

run()