Competitive programming simple input output testing.

test.py

import argparse
import os
import re
import sys
from sys import platform
from subprocess import check_output, STDOUT, CalledProcessError
from itertools import izip_longest
from timeit import timeit
from subprocess import Popen
from threading import Timer

PLANGUAGE_EXTENSIONS = {
    ".c": "c",
    ".cpp": "cpp",
    ".cxx": "cpp",
    ".c++": "cpp",
    ".cc": "cpp",
    ".py": "py",
    ".js": "js",
}

COMPILE_COMMANDS = {
    "c": ("gcc -DDEBUG(...) -std=gnu99 --static %(srcfile)s -lm -o "
          "%(srcfile_base)s"),
    "cpp": ("g++ -DLOCAL -lm -O2 -std=c++11 -Wall -static "
            "%(srcfile)s -o %(srcfile_base)s"),
}

TESTCASE_EXTENSIONS = ["txt", "in"]

SHELL = True

if platform == "win32":
    COMPILE_COMMANDS["c"] = COMPILE_COMMANDS["c"]\
        .replace("--static ", "--static -Wl,--stack,64777216 ")
    COMPILE_COMMANDS["cpp"] = COMPILE_COMMANDS["cpp"]\
        .replace("-static ", "-static -Wl,--stack,64777216 ")
    SHELL = False

RUN_COMMANDS = {
    "py": "python %s",
    "js": "d8 %s",
}


def subprocess_execute(command, fi, fo, timeout=2):
    def kill(p):
        try:
            p.kill()
        except OSError:
            pass

    p = Popen(command, stdin=fi, stdout=fo, shell=SHELL)
    t = Timer(timeout, kill, [p])
    t.start()
    p.wait()
    t.cancel()


def tryint(s):
    try:
        return int(s)
    except:
        return s


def alphanum_key(s):
    return [tryint(c) for c in re.split('([0-9]+)', s)]


def compile(srcfile, language, is_static):
    compile_command = COMPILE_COMMANDS[language] % {
        "srcfile": srcfile,
        "srcfile_base": srcfile.split(".")[0],
    }
    if not is_static:
        compile_command = re.sub("\s-?-static", "", compile_command)
    try:
        check_output(compile_command.split(), stderr=STDOUT)
    except CalledProcessError, e:
        print "Compile error (%s):" % " ".join(e.cmd)
        print " Output: \n%s", e.output
    return 0


def normalize(lines):
    return re.sub('\s+', ' ', "".join(lines)).strip()


def checker_out(infile, outfile, solfile):
    # Checkers that utilize testlib take 3 arguments: input file,
    # participant output and jury output
    try:
        out = check_output(
            [command_frm % "checker", infile, outfile, solfile],
            stderr=STDOUT
        )
        ret = 0
    except CalledProcessError, e:
        ret = e.returncode
        out = e.output
    if ret in (1, 2):
        # PE and WA (FA) are the same on codeforces,
        # http://codeforces.com/blog/entry/18431
        return ("WA", out)
    elif ret > 2:
        # something unexpected happened
        return ("SU", out)
    return ("AC", out)


def check_out(infile, outfile, solfile):
    if not os.path.exists(solfile):
        return ("SKIP", "")
    if os.path.exists("checker.cpp"):
        return checker_out(infile, outfile, solfile)
    with open(solfile, "r") as f:
        sollines = f.readlines()
    with open(outfile, "r") as f:
        outlines = f.readlines()
    out = ""
    if normalize(sollines) == normalize(outlines):
        return ("AC", out)
    for (i, (linesol, lineout)) in enumerate(izip_longest(sollines, outlines,
                                                          fillvalue="EOF")):
        if normalize(linesol) != normalize(lineout):
            out += "line %d: " % (i+1, )
            out += "\"%s\" != \"%s\"" % (normalize(linesol),
                                         normalize(lineout))
            break
    else:
        out = ""
    return ("WA", out)


def test_in_out(cmd, timeout, check_solution, write_solution):
    files = filter(os.path.isfile, os.listdir('.'))
    infiles = filter(lambda f: "in" in f and len(f.split(".")) > 1 and
                     f.split(".")[-1] in TESTCASE_EXTENSIONS, files)
    infiles.sort(key=alphanum_key)
    solfiles = map(lambda f: f.replace("in", "sol"), infiles)
    outfile_name = "sol" if write_solution else "out"
    outfiles = map(lambda f: f.replace("in", outfile_name), infiles)
    times = []
    verdicts = []
    outputs = []
    for infile, solfile, outfile in zip(infiles, solfiles, outfiles):
        setup = "fi = open('%s', 'r');" % infile
        setup += "fo = open('%s', 'w');" % outfile
        setup += "from __main__ import subprocess_execute"
        statement = "subprocess_execute('%s', fi, fo, %d)" % (cmd, timeout)
        times.append(timeit(stmt=statement, setup=setup, number=1))
        if times[-1] > timeout:
            verdicts.append("TLE")
            outputs.append("")
        elif check_solution:
            ver, out = check_out(infile, outfile, solfile)
            verdicts.append(ver)
            outputs.append(out)

    width = max(map(len, infiles)) + 3
    length = len(infiles)
    num = 160/width
    for l, r in zip(range(0, length+num, num), range(num, length+num, num)):
        print "Test files: ",
        print " ".join([f.ljust(width) for f in infiles[l:r]])
        print "Times [s]:  ",
        print " ".join([("%.6f" % t).ljust(width) for t in times[l:r]])
        if check_solution:
            print "Verdict:    ",
            print " ".join([verdict.ljust(width) for verdict in verdicts[l:r]])
        print "-"*21
    if sum(map(len, outputs)) and \
            sum(map(lambda x: 1 if x not in ["AC", "SKIP"] else 0, verdicts)):
        print "Output"
        for f, v, o in zip(infiles, verdicts, outputs):
            if v not in ["AC", "SKIP"] and o:
                print " | ".join([f.ljust(width), v, o.rstrip()])


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Simple script for competitive programming input output '
        'testing.',
        epilog="""

Examples:

    python -u test.py --check-solution src.cpp
    python -u test.py --timeout 3 --check-solution src.cpp
    python -u test.py --write-solution src.cpp

    This command will:
    * Compile src.cpp file using default compile command for C++.
      Default compile commands is defined in COMPILE_COMMANDS.
    * If compilation is successful:
        * For every input file run executable and produce output file.
        * If there is checker.cpp file compile it and use this program to
          analyse input, output and solution files and get and print verdicts.
        * Otherwise check if solution files match related output files and
          print verdicts.
    * If program runs for more than <timeout> seconds verdict is TLE
      Default timeout can be changed using --timeout option.
    * Input files are all files in the directory of the src file that have
      substring 'in' in their name
    * Output and solution files are all files that have same names as input
      files with replacement 'in' -> 'out' or 'sol'.

Currenty supported languages are C, C++, python, and javascript

""")
    parser.add_argument('source', metavar='src', help='Source file')
    parser.add_argument(
        '--static',
        dest='static',
        action='store_true',
        help='Compile with -static option.'
    )
    parser.add_argument(
        '--check-solution',
        dest='check_solution',
        action='store_true',
        help='Check if output equals solution for each input file.'
    )
    parser.add_argument(
        '--write-solution',
        dest='write_solution',
        action='store_true',
        help='Write solution for each input file.'
    )
    parser.add_argument(
        '--timeout',
        dest='timeout',
        type=int,
        default=3,
        help='Timeout for execution of each test case in seconds.'
    )
    try:
        args = parser.parse_args()
    except SystemExit as err:
        if err.code == 2:
            parser.print_help()
            sys.exit(0)
        raise err

    if (args.check_solution and args.write_solution):
        print "Cannot check and write solution at the same time."
        sys.exit(1)
    srcfile = args.source
    if not os.path.isfile(srcfile):
        print "Source file %s does not exist." % srcfile
        sys.exit(1)
    srcname, srcext = os.path.splitext(srcfile)
    if srcext in PLANGUAGE_EXTENSIONS.keys():
        planguage = PLANGUAGE_EXTENSIONS[srcext]
    else:
        print "Extension %s is currently not supported." % srcext
        sys.exit(1)

    if platform == "win32":
        command_frm = "%s.exe"
    elif platform.startswith("linux"):
        command_frm = "./%s"
    if os.path.exists("checker.cpp"):
        if compile("checker.cpp", "cpp", False):
            sys.exit(0)
    if planguage in COMPILE_COMMANDS.keys():
        command = command_frm % srcname
        if compile(srcfile, planguage, args.static):
            sys.exit(0)
    elif planguage in RUN_COMMANDS.keys():
        command = RUN_COMMANDS[planguage] % srcfile

    test_in_out(command, args.timeout, args.check_solution,
                args.write_solution)

test_case_copy.py

import argparse
import os.path
from shutil import copyfile
import sys


def copy(src, dest):
    if (os.path.isfile(src)):
        if fake:
            print "%s -> %s faked copy" % (src, dest)
        else:
            copyfile(src, dest)
            print "%s -> %s copied" % (src, dest)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Copy test case.')
    parser.add_argument('infile', metavar='infile', help='Input file')
    parser.add_argument(
        '--out2sol',
        dest='o2s',
        action='store_true',
        help='Instead of sol file copy out file to new sol file.'
    )
    parser.add_argument(
        '--fake',
        dest='fake',
        action='store_true',
        help='Do not copy just print.'
    )

    args = parser.parse_args()

    infile = args.infile
    o2s = args.o2s
    # global fake
    fake = args.fake
    if not os.path.isfile(infile):
        print "Input file %s does not exist." % infile
        sys.exit(1)
    infile_name, infile_extension = os.path.splitext(infile)
    solfile_name = infile_name.replace("in", "sol")
    outfile_name = infile_name.replace("in", "out")
    outfile_extension = infile_extension
    solfile_extension = infile_extension
    if infile_extension not in [".in", ".txt"]:
        print "Input file %s does not end with in or txt." % infile
        sys.exit(1)
    files = filter(os.path.isfile, os.listdir('.'))
    i = 1
    while(os.path.isfile(infile_name + str(i) + infile_extension)):
        i += 1

    copy(infile, infile_name + str(i) + infile_extension)

    if o2s:
        copy(outfile_name + outfile_extension,
             solfile_name + str(i) + solfile_extension)
    else:
        copy(outfile_name + outfile_extension,
             outfile_name + str(i) + outfile_extension)
        copy(solfile_name + solfile_extension,
             solfile_name + str(i) + solfile_extension)