jakcharvat/progtest.sh

## progtest.sh
#!/bin/bash


VERSION='3.0.0'


BOLD='\033[1m'
RED='\033[0;31m'
GREEN='\033[0;32m'
ORANGE='\033[0;33m'
DARKGRAY='\033[0;90m'
NC='\033[0m'


error () {
    echo -e "${RED}$1${NC}" >&2
}


usage () {
    echo
    echo -e "${BOLD}USAGE:${NC}"
    echo -e " -> ${BOLD}$0 [SOURCE_FILE ...]${NC}"
    echo -e "    - where ${BOLD}SOURCE_FILE${NC} is an absolute or relative path to"
    echo -e "      the .c file you would like to test, not a compiled binary."
    echo -e "      You can test several files at once, just pass them in as"
    echo -e "      separate arguments."
    echo
    echo -e " -> ${BOLD}$0 [--help | -h]${NC}"
    echo -e "    - print more information about the usage of the script."
    echo
    echo -e " -> ${BOLD}$0 [--version | -v]${NC}"
    echo -e "    - print more detailed information about the version of the script and its dependencies."
    if [[ $# -ne 1 && ! "$1" == "--no-link" ]]
    then
        echo
        echo -e "${BOLD} See ${ORANGE}$0 --help${NC}${BOLD} for more info."
    fi
}


print_header () {
    echo -e "${BOLD}PA1 ProgTest validation script${NC}"
}


print_bugs () {
    echo
    echo -e "${BOLD}BUGS:${NC}"
    echo -e "I'm pretty much a noob with shell scripting, so if this thing doesn't work the way it should then please let me know in person,"
    echo -e "on Discord (${ORANGE}jakcharvat#3310${NC}), or if ur really into that sort of stuff through email (${ORANGE}mailto:jakcharvat@gmail.com${NC})"
    echo -e "${GREEN}${BOLD}Thanks :)${NC}"
}


print_plug () {
    echo
    echo -e "${BOLD}SMALL PLUG:${NC}"
    echo -e "Written by ${ORANGE}${BOLD}Jakub Charvat${NC}. Pls check out my GitHub (${BOLD}https://github.com/jakcharvat${NC}). TY ${RED}${BOLD}<3${NC}"
}


print_help () {
    print_header
    echo "Version ${VERSION}"
    echo
    echo -e "${BOLD}OVERVIEW:${NC}"
    echo "A script to simplify testing ProgTest challenge solutions. The script only needs the"
    echo "path to the file that should be tested, and handles all compilation and testing itself."
    echo
    echo -e "${BOLD}REQUIREMENTS:${NC}"
    echo "For simplicity, this script makes the following assumption about your file structure:"
    echo -e "  The test data for the solution you would like to test lives in a directory called ${ORANGE}\"sample\"${NC}"
    echo "  that's located in the same directory as the solution. So if your solution file is called"
    echo -e "  ${ORANGE}solution.c${NC}, then the file tree would look like so:"
    echo
    echo "      projectDir/"
    echo "        |- solution.c"
    echo "        |- sample/"
    echo "             |- CZE/"
    echo "                  |- 0000_in.txt"
    echo "                  |- 0000_out.txt"
    echo "                  |- 0001_in.txt"
    echo "                  |- 0001_out.txt"
    echo "                         ..."
    echo
    echo "  This structure can be located anywhere in your filesystem, and when calling the script"
    echo -e "  you must point at the ${ORANGE}solution.c${NC} file as the only argument."
    usage --no-link
    print_bugs
    print_plug
}


print_version () {
    print_header
    echo -e "Version ${ORANGE}${VERSION}${NC}"
    echo
    echo -e "${BOLD}DEPENDENCIES:${NC}"
    echo -e " -> ${ORANGE}${BOLD}bash${NC}"
    echo -e " -> ${ORANGE}${BOLD}g++${NC}"
    echo -e "     - On Darwin (macOS) the script uses g++-11, available from"
    echo -e "       ${BOLD}https://formulae.brew.sh/formula/gcc#default${NC}."
    echo -e "       This is because by default the g++ command links to Xcode's"
    echo -e "       clang C compiler, whereas the brew version is actual GCC."
    echo -e " -> a bunch of other ${DARKGRAY}${BOLD}(hopefully)${NC} built-in commands:"
    echo -e "     - ${ORANGE}echo${NC}"
    echo -e "     - ${ORANGE}basename${NC}"
    echo -e "     - ${ORANGE}dirname${NC}"
    echo -e "     - ${ORANGE}grep${NC}"
    echo -e "     - ${ORANGE}sed${NC}"
    echo -e "     - ${ORANGE}diff${NC}"
    echo -e "     - ${ORANGE}timeout${NC}"
    echo -e "     - ${ORANGE}trap${NC}"
    print_bugs
    print_plug
}


builddir () {
    DIR='./.builddir'
    if [[ ! "$1" == "--no-create" ]]
    then
        if [[ ! -d "$DIR" ]]
        then
            mkdir -p "$DIR"
        fi
    fi
    echo "$DIR"
}


cleanup_exit () {
    if [[ -d "$( builddir --no-create )" ]]
    then
        rm -r "$( builddir --no-create )"
    fi
    exit $1
}


sigint_exit () {
    error
    error
    error "SIGINT: Cleaning up and exiting..."
    cleanup_exit 130
}


trap sigint_exit INT


if [[ $# -lt 1 ]]
then
    error "ERROR: You must provide at least one argument - either"
    error "       an input file or a script flag."
    usage >&2
    cleanup_exit 2
fi


PRINT_HELP=0
PRINT_VERSION=0
SOURCE_FILES=""

while [[ $# -gt 0 ]]
do
    ARG="$1"

    if [[ "$ARG" == "--help" || "$ARG" == "-h" ]]
    then
        PRINT_HELP=1
    elif [[ "$ARG" == "--version" || "$ARG" == "-v" ]]
    then
        PRINT_VERSION=1
    else
        if [[ ! -z "$SOURCE_FILES" ]]
        then
            SOURCE_FILES="$SOURCE_FILES "
        fi
        SOURCE_FILES="$SOURCE_FILES$ARG"
    fi

    if [[ ! -z "$SOURCE_FILES" && $PRINT_HELP != 0 ]]
    then
        error "ERROR: Combining help flag with sources."
        usage >&2
        cleanup_exit 2
    fi

    if [[ ! -z "$SOURCE_FILES" && $PRINT_VERSION != 0 ]]
    then
        error "ERROR: Combining version flag with sources."
        usage >&2
        cleanup_exit 2
    fi

    if [[ $PRINT_HELP != 0 && $PRINT_VERSION != 0 ]]
    then
        error "ERROR: Combining version flag with help flag."
        usage >&2
        cleanup_exit 2
    fi

    shift
done


if [[ $PRINT_HELP == 1 ]]
then
    print_help
    cleanup_exit 0
fi


if [[ $PRINT_VERSION == 1 ]]
then
    print_version
    cleanup_exit 0
fi


# ========================= BUILD PHASE =========================
compile() {
    GCC="g++"
    if [[ "$( uname -s )" == "Darwin" ]]; then GCC="g++-11"; fi
    "$GCC" -Wall -pedantic "$1" -o "$( builddir )/$2"
}


copy_tests () {
    SAMPLE_DIR="$( dirname "$1" )/sample"

    if [[ -d "$( builddir --no-create )/tests/$2" ]]
    then
        rm -r "$( builddir --no-create )/tests/$2"
    fi

    mkdir -p "$( builddir )/tests/$2"
    ASSERT_MODE=0

    if [[ -d "$SAMPLE_DIR/CZE" ]]
    then
        cp "$SAMPLE_DIR/CZE"/* "$( builddir )/tests/$2/"
    elif [[ -d "$SAMPLE_DIR" ]]
    then
        TEST_DIRS=$( find "$SAMPLE_DIR" -name 'CZE' )
        for DIR in $TEST_DIRS
        do
            TEST_NAME="$( basename "$( dirname "$DIR" )" )"
            mkdir -p "$( builddir )/tests/$2/$TEST_NAME"
            cp "$DIR"/* "$( builddir )/tests/$2/$TEST_NAME"
        done
    else
        ASSERT_MODE=1
    fi
}


get_build_file_base() {
    DIR="$( dirname "${1##*./}" )"
    BASE="$( tr '/' '-' <<<"${DIR#/}" )"
    if [[ -z "$( echo "$BASE" | tr -d '.' | tr -d '/' )" ]]
    then
        BASE="$( basename "${1%.*}" )"
    fi
    echo "$BASE"
}


get_build_file() {
    echo "$1.out"
}


build_source_file() {
    SOURCE_FILE="$1"
    echo -ne "${BOLD}Building file ${ORANGE}${SOURCE_FILE}${NC}${BOLD}..."
    BUILD_FILE_BASE=$( get_build_file_base "$SOURCE_FILE" )
    BUILD_FILE=$( get_build_file "$BUILD_FILE_BASE" )
    copy_tests "$SOURCE_FILE" "$BUILD_FILE_BASE"
    ERROR="$( compile "$SOURCE_FILE" "$BUILD_FILE" 2>&1 )"
    GCC_EXIT="$?"
    if [[ "$GCC_EXIT" -ne 0 ]]
    then
        echo
        echo -e "$ERROR"
        echo -e "${RED}ERROR Compiling ${SOURCE_FILE}. There should be more info above."
        cleanup_exit "$GCC_EXIT"
    fi
    echo -ne "${GREEN}${BOLD} BUILT ${BUILD_FILE}${NC}"
    if [[ "$ASSERT_MODE" == 1 ]]; then echo -ne "${ORANGE} -- Assert mode${NC}"; fi
    echo
    if [[ ! -z "$ERROR" ]]; then echo -e "${DARKGRAY}${ERROR}${NC}"; fi
    echo
}


if [[ -d "$( builddir --no-create )" ]]; then rm -r "$( builddir --no-create )"; fi

for SOURCE_FILE in $SOURCE_FILES
do
     if [[ ! -e "$SOURCE_FILE" ]]
     then
         if [[ -z "$SOURCE_FILE" ]]; then SOURCE_FILE="NO SOURCE"; fi
         echo -e "${RED}ERROR Cannot find a source file in the specified path: ${BOLD}../$1${RED} (${BOLD}${SOURCE_FILE}${RED}).${NC}" >&2
         cleanup_exit 2
     fi
     build_source_file "$SOURCE_FILE"
done


# ========================= TEST PHASE =========================
ERROR_COUNT=0
TIMEOUT_COUNT=0
for BINARY in "$( builddir )"/*.out
do
    echo
    echo
    BINARY_NAME="$( basename "$BINARY" )"
    echo -ne "${BOLD}Testing ${ORANGE}${BINARY_NAME}${NC}"

    # the %.* drops the last component of the filename - the extension
    BASENAME="${BINARY_NAME%.*}"

    if [[ -z "$BASENAME" ]]; then BASENAME="$( basename "$BINARY" )"; fi
    TEST_DIR="$( dirname "$BINARY" )/tests/$BASENAME"

    if [[ ! -d "$TEST_DIR" || ! "$( ls -A "$TEST_DIR" )" ]]
    then
        echo -e "${ORANGE} -- Assert mode${NC}"
        TEST_TIMEOUT=10
        { timeout --foreground "$TEST_TIMEOUT" "$BINARY"; } > "$( builddir )/out.txt" 2>&1

        if [[ $? == 124 ]]
        then
            echo -e "${ORANGE}TIMEOUT:${NC} ${BINARY}"
            TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
        else
            if [[ -e "$( builddir )/out.txt" && ! -s "$( builddir )/out.txt" ]]
            then
                echo -e "${GREEN}OK:${NC}      ${BINARY}"
            else
                echo -e "${RED}FAIL:${NC}    ${BINARY}"
                ERROR_COUNT=$(( ERROR_COUNT + 1 ))
                echo -e "${RED}Unexpected output:${NC}"
                cat "$( builddir )/out.txt"
                echo
            fi
        fi

        continue
    fi

    echo
    MULTIPLE_DIRS=1
    REF_DIRS=$( ls -p "$TEST_DIR" | grep '^.*/$' | sed 's /$  ' )
    if [[ -z "$REF_DIRS" ]]
    then
        REF_DIRS="$TEST_DIR"
        MULTIPLE_DIRS=0
    fi

    TEST_IDX=0
    for REF_DIR in $REF_DIRS
    do
        if [[ $MULTIPLE_DIRS == 1 ]]
        then
            echo
            echo -e "-> Test set ${ORANGE}${REF_DIR}${NC}"
        fi
        if [[ ! -d "$REF_DIR" ]]; then REF_DIR="${TEST_DIR}/${REF_DIR}"; fi
        for INPUT_FILE in "$REF_DIR"/????_in.txt
        do
            REF_FILE=$( sed 's-\(.*\)_in\.txt-\1_out.txt-' <<<"$INPUT_FILE" )
            if [[ ! -e "$REF_FILE" ]]
            then
                echo "ERROR: The reference file ($REF_FILE) doesn't exist."
                cleanup_exit 2
            fi

            INPUT_FILE_FMT="${INPUT_FILE#*/${BASENAME}/}"

            TEST_TIMEOUT=3
            # Give a longer timeout for first iterations of tests to allow caches to load
            # up and shit. Basically first tests timed out on my machine so I tried
            # doing this and it helped, therefore I'm keeping it here ¯\_(ツ)_/¯
            if [[ $TEST_IDX == 0 ]]; then TEST_TIMEOUT=10; fi
            timeout --foreground "$TEST_TIMEOUT" "$BINARY" < "$INPUT_FILE" > "$( builddir )/out.txt"

            if [[ $? == 124 ]]
            then
                echo -e "${ORANGE}TIMEOUT:${NC} ${INPUT_FILE_FMT}"
                TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
            else
                if ! diff "$REF_FILE" "$( builddir )/out.txt" > "$( builddir )/temp.txt"
                then
                    echo -e "${RED}FAIL:${NC}    ${INPUT_FILE_FMT}"
                    ERROR_COUNT=$(( ERROR_COUNT + 1 ))
                    cat "$( builddir )/temp.txt"
                else
                    echo -e "${GREEN}OK:${NC}      ${INPUT_FILE_FMT}"
                fi
            fi
        done
        (( TEST_IDX++ ))
    done
done

echo

if [[ $ERROR_COUNT == 0 && $TIMEOUT_COUNT == 0 ]]
then
    echo -e "${GREEN}${BOLD}SUCCESS: All tests passed :)${NC}"
    cleanup_exit 0
fi


if [[ $ERROR_COUNT -ne 0 ]]
then
    echo -e "${RED}${BOLD}FAIL: There were $ERROR_COUNT errors :(${NC}"
fi

if [[ $TIMEOUT_COUNT -ne 0 ]]
then
    echo -e "${ORANGE}${BOLD}FAIL: There were $TIMEOUT_COUNT timeouts :/${NC}"
fi

cleanup_exit 5
	#!/bin/bash


	VERSION='3.0.0'


	BOLD='\033[1m'
	RED='\033[0;31m'
	GREEN='\033[0;32m'
	ORANGE='\033[0;33m'
	DARKGRAY='\033[0;90m'
	NC='\033[0m'


	error () {
	echo -e "${RED}$1${NC}" >&2
	}


	usage () {
	echo
	echo -e "${BOLD}USAGE:${NC}"
	echo -e " -> ${BOLD}$0 [SOURCE_FILE ...]${NC}"
	echo -e " - where ${BOLD}SOURCE_FILE${NC} is an absolute or relative path to"
	echo -e " the .c file you would like to test, not a compiled binary."
	echo -e " You can test several files at once, just pass them in as"
	echo -e " separate arguments."
	echo
	echo -e " -> ${BOLD}$0 [--help \| -h]${NC}"
	echo -e " - print more information about the usage of the script."
	echo
	echo -e " -> ${BOLD}$0 [--version \| -v]${NC}"
	echo -e " - print more detailed information about the version of the script and its dependencies."
	if [[ $# -ne 1 && ! "$1" == "--no-link" ]]
	then
	echo
	echo -e "${BOLD} See ${ORANGE}$0 --help${NC}${BOLD} for more info."
	fi
	}


	print_header () {
	echo -e "${BOLD}PA1 ProgTest validation script${NC}"
	}


	print_bugs () {
	echo
	echo -e "${BOLD}BUGS:${NC}"
	echo -e "I'm pretty much a noob with shell scripting, so if this thing doesn't work the way it should then please let me know in person,"
	echo -e "on Discord (${ORANGE}jakcharvat#3310${NC}), or if ur really into that sort of stuff through email (${ORANGE}mailto:jakcharvat@gmail.com${NC})"
	echo -e "${GREEN}${BOLD}Thanks :)${NC}"
	}


	print_plug () {
	echo
	echo -e "${BOLD}SMALL PLUG:${NC}"
	echo -e "Written by ${ORANGE}${BOLD}Jakub Charvat${NC}. Pls check out my GitHub (${BOLD}https://github.com/jakcharvat${NC}). TY ${RED}${BOLD}<3${NC}"
	}


	print_help () {
	print_header
	echo "Version ${VERSION}"
	echo
	echo -e "${BOLD}OVERVIEW:${NC}"
	echo "A script to simplify testing ProgTest challenge solutions. The script only needs the"
	echo "path to the file that should be tested, and handles all compilation and testing itself."
	echo
	echo -e "${BOLD}REQUIREMENTS:${NC}"
	echo "For simplicity, this script makes the following assumption about your file structure:"
	echo -e " The test data for the solution you would like to test lives in a directory called ${ORANGE}\"sample\"${NC}"
	echo " that's located in the same directory as the solution. So if your solution file is called"
	echo -e " ${ORANGE}solution.c${NC}, then the file tree would look like so:"
	echo
	echo " projectDir/"
	echo " \|- solution.c"
	echo " \|- sample/"
	echo " \|- CZE/"
	echo " \|- 0000_in.txt"
	echo " \|- 0000_out.txt"
	echo " \|- 0001_in.txt"
	echo " \|- 0001_out.txt"
	echo " ..."
	echo
	echo " This structure can be located anywhere in your filesystem, and when calling the script"
	echo -e " you must point at the ${ORANGE}solution.c${NC} file as the only argument."
	usage --no-link
	print_bugs
	print_plug
	}


	print_version () {
	print_header
	echo -e "Version ${ORANGE}${VERSION}${NC}"
	echo
	echo -e "${BOLD}DEPENDENCIES:${NC}"
	echo -e " -> ${ORANGE}${BOLD}bash${NC}"
	echo -e " -> ${ORANGE}${BOLD}g++${NC}"
	echo -e " - On Darwin (macOS) the script uses g++-11, available from"
	echo -e " ${BOLD}https://formulae.brew.sh/formula/gcc#default${NC}."
	echo -e " This is because by default the g++ command links to Xcode's"
	echo -e " clang C compiler, whereas the brew version is actual GCC."
	echo -e " -> a bunch of other ${DARKGRAY}${BOLD}(hopefully)${NC} built-in commands:"
	echo -e " - ${ORANGE}echo${NC}"
	echo -e " - ${ORANGE}basename${NC}"
	echo -e " - ${ORANGE}dirname${NC}"
	echo -e " - ${ORANGE}grep${NC}"
	echo -e " - ${ORANGE}sed${NC}"
	echo -e " - ${ORANGE}diff${NC}"
	echo -e " - ${ORANGE}timeout${NC}"
	echo -e " - ${ORANGE}trap${NC}"
	print_bugs
	print_plug
	}


	builddir () {
	DIR='./.builddir'
	if [[ ! "$1" == "--no-create" ]]
	then
	if [[ ! -d "$DIR" ]]
	then
	mkdir -p "$DIR"
	fi
	fi
	echo "$DIR"
	}


	cleanup_exit () {
	if [[ -d "$( builddir --no-create )" ]]
	then
	rm -r "$( builddir --no-create )"
	fi
	exit $1
	}


	sigint_exit () {
	error
	error
	error "SIGINT: Cleaning up and exiting..."
	cleanup_exit 130
	}


	trap sigint_exit INT


	if [[ $# -lt 1 ]]
	then
	error "ERROR: You must provide at least one argument - either"
	error " an input file or a script flag."
	usage >&2
	cleanup_exit 2
	fi


	PRINT_HELP=0
	PRINT_VERSION=0
	SOURCE_FILES=""

	while [[ $# -gt 0 ]]
	do
	ARG="$1"

	if [[ "$ARG" == "--help" \|\| "$ARG" == "-h" ]]
	then
	PRINT_HELP=1
	elif [[ "$ARG" == "--version" \|\| "$ARG" == "-v" ]]
	then
	PRINT_VERSION=1
	else
	if [[ ! -z "$SOURCE_FILES" ]]
	then
	SOURCE_FILES="$SOURCE_FILES "
	fi
	SOURCE_FILES="$SOURCE_FILES$ARG"
	fi

	if [[ ! -z "$SOURCE_FILES" && $PRINT_HELP != 0 ]]
	then
	error "ERROR: Combining help flag with sources."
	usage >&2
	cleanup_exit 2
	fi

	if [[ ! -z "$SOURCE_FILES" && $PRINT_VERSION != 0 ]]
	then
	error "ERROR: Combining version flag with sources."
	usage >&2
	cleanup_exit 2
	fi

	if [[ $PRINT_HELP != 0 && $PRINT_VERSION != 0 ]]
	then
	error "ERROR: Combining version flag with help flag."
	usage >&2
	cleanup_exit 2
	fi

	shift
	done


	if [[ $PRINT_HELP == 1 ]]
	then
	print_help
	cleanup_exit 0
	fi


	if [[ $PRINT_VERSION == 1 ]]
	then
	print_version
	cleanup_exit 0
	fi



	# ========================= BUILD PHASE =========================
	compile() {
	GCC="g++"
	if [[ "$( uname -s )" == "Darwin" ]]; then GCC="g++-11"; fi
	"$GCC" -Wall -pedantic "$1" -o "$( builddir )/$2"
	}


	copy_tests () {
	SAMPLE_DIR="$( dirname "$1" )/sample"

	if [[ -d "$( builddir --no-create )/tests/$2" ]]
	then
	rm -r "$( builddir --no-create )/tests/$2"
	fi

	mkdir -p "$( builddir )/tests/$2"
	ASSERT_MODE=0

	if [[ -d "$SAMPLE_DIR/CZE" ]]
	then
	cp "$SAMPLE_DIR/CZE"/* "$( builddir )/tests/$2/"
	elif [[ -d "$SAMPLE_DIR" ]]
	then
	TEST_DIRS=$( find "$SAMPLE_DIR" -name 'CZE' )
	for DIR in $TEST_DIRS
	do
	TEST_NAME="$( basename "$( dirname "$DIR" )" )"
	mkdir -p "$( builddir )/tests/$2/$TEST_NAME"
	cp "$DIR"/* "$( builddir )/tests/$2/$TEST_NAME"
	done
	else
	ASSERT_MODE=1
	fi
	}


	get_build_file_base() {
	DIR="$( dirname "${1##*./}" )"
	BASE="$( tr '/' '-' <<<"${DIR#/}" )"
	if [[ -z "$( echo "$BASE" \| tr -d '.' \| tr -d '/' )" ]]
	then
	BASE="$( basename "${1%.*}" )"
	fi
	echo "$BASE"
	}


	get_build_file() {
	echo "$1.out"
	}


	build_source_file() {
	SOURCE_FILE="$1"
	echo -ne "${BOLD}Building file ${ORANGE}${SOURCE_FILE}${NC}${BOLD}..."
	BUILD_FILE_BASE=$( get_build_file_base "$SOURCE_FILE" )
	BUILD_FILE=$( get_build_file "$BUILD_FILE_BASE" )
	copy_tests "$SOURCE_FILE" "$BUILD_FILE_BASE"
	ERROR="$( compile "$SOURCE_FILE" "$BUILD_FILE" 2>&1 )"
	GCC_EXIT="$?"
	if [[ "$GCC_EXIT" -ne 0 ]]
	then
	echo
	echo -e "$ERROR"
	echo -e "${RED}ERROR Compiling ${SOURCE_FILE}. There should be more info above."
	cleanup_exit "$GCC_EXIT"
	fi
	echo -ne "${GREEN}${BOLD} BUILT ${BUILD_FILE}${NC}"
	if [[ "$ASSERT_MODE" == 1 ]]; then echo -ne "${ORANGE} -- Assert mode${NC}"; fi
	echo
	if [[ ! -z "$ERROR" ]]; then echo -e "${DARKGRAY}${ERROR}${NC}"; fi
	echo
	}


	if [[ -d "$( builddir --no-create )" ]]; then rm -r "$( builddir --no-create )"; fi

	for SOURCE_FILE in $SOURCE_FILES
	do
	if [[ ! -e "$SOURCE_FILE" ]]
	then
	if [[ -z "$SOURCE_FILE" ]]; then SOURCE_FILE="NO SOURCE"; fi
	echo -e "${RED}ERROR Cannot find a source file in the specified path: ${BOLD}../$1${RED} (${BOLD}${SOURCE_FILE}${RED}).${NC}" >&2
	cleanup_exit 2
	fi
	build_source_file "$SOURCE_FILE"
	done



	# ========================= TEST PHASE =========================
	ERROR_COUNT=0
	TIMEOUT_COUNT=0
	for BINARY in "$( builddir )"/*.out
	do
	echo
	echo
	BINARY_NAME="$( basename "$BINARY" )"
	echo -ne "${BOLD}Testing ${ORANGE}${BINARY_NAME}${NC}"

	# the %.* drops the last component of the filename - the extension
	BASENAME="${BINARY_NAME%.*}"

	if [[ -z "$BASENAME" ]]; then BASENAME="$( basename "$BINARY" )"; fi
	TEST_DIR="$( dirname "$BINARY" )/tests/$BASENAME"

	if [[ ! -d "$TEST_DIR" \|\| ! "$( ls -A "$TEST_DIR" )" ]]
	then
	echo -e "${ORANGE} -- Assert mode${NC}"
	TEST_TIMEOUT=10
	{ timeout --foreground "$TEST_TIMEOUT" "$BINARY"; } > "$( builddir )/out.txt" 2>&1

	if [[ $? == 124 ]]
	then
	echo -e "${ORANGE}TIMEOUT:${NC} ${BINARY}"
	TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
	else
	if [[ -e "$( builddir )/out.txt" && ! -s "$( builddir )/out.txt" ]]
	then
	echo -e "${GREEN}OK:${NC} ${BINARY}"
	else
	echo -e "${RED}FAIL:${NC} ${BINARY}"
	ERROR_COUNT=$(( ERROR_COUNT + 1 ))
	echo -e "${RED}Unexpected output:${NC}"
	cat "$( builddir )/out.txt"
	echo
	fi
	fi

	continue
	fi

	echo
	MULTIPLE_DIRS=1
	REF_DIRS=$( ls -p "$TEST_DIR" \| grep '^.*/$' \| sed 's /$ ' )
	if [[ -z "$REF_DIRS" ]]
	then
	REF_DIRS="$TEST_DIR"
	MULTIPLE_DIRS=0
	fi

	TEST_IDX=0
	for REF_DIR in $REF_DIRS
	do
	if [[ $MULTIPLE_DIRS == 1 ]]
	then
	echo
	echo -e "-> Test set ${ORANGE}${REF_DIR}${NC}"
	fi
	if [[ ! -d "$REF_DIR" ]]; then REF_DIR="${TEST_DIR}/${REF_DIR}"; fi
	for INPUT_FILE in "$REF_DIR"/????_in.txt
	do
	REF_FILE=$( sed 's-\(.*\)_in\.txt-\1_out.txt-' <<<"$INPUT_FILE" )
	if [[ ! -e "$REF_FILE" ]]
	then
	echo "ERROR: The reference file ($REF_FILE) doesn't exist."
	cleanup_exit 2
	fi

	INPUT_FILE_FMT="${INPUT_FILE#*/${BASENAME}/}"

	TEST_TIMEOUT=3
	# Give a longer timeout for first iterations of tests to allow caches to load
	# up and shit. Basically first tests timed out on my machine so I tried
	# doing this and it helped, therefore I'm keeping it here ¯\_(ツ)_/¯
	if [[ $TEST_IDX == 0 ]]; then TEST_TIMEOUT=10; fi
	timeout --foreground "$TEST_TIMEOUT" "$BINARY" < "$INPUT_FILE" > "$( builddir )/out.txt"

	if [[ $? == 124 ]]
	then
	echo -e "${ORANGE}TIMEOUT:${NC} ${INPUT_FILE_FMT}"
	TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
	else
	if ! diff "$REF_FILE" "$( builddir )/out.txt" > "$( builddir )/temp.txt"
	then
	echo -e "${RED}FAIL:${NC} ${INPUT_FILE_FMT}"
	ERROR_COUNT=$(( ERROR_COUNT + 1 ))
	cat "$( builddir )/temp.txt"
	else
	echo -e "${GREEN}OK:${NC} ${INPUT_FILE_FMT}"
	fi
	fi
	done
	(( TEST_IDX++ ))
	done
	done

	echo

	if [[ $ERROR_COUNT == 0 && $TIMEOUT_COUNT == 0 ]]
	then
	echo -e "${GREEN}${BOLD}SUCCESS: All tests passed :)${NC}"
	cleanup_exit 0
	fi


	if [[ $ERROR_COUNT -ne 0 ]]
	then
	echo -e "${RED}${BOLD}FAIL: There were $ERROR_COUNT errors :(${NC}"
	fi

	if [[ $TIMEOUT_COUNT -ne 0 ]]
	then
	echo -e "${ORANGE}${BOLD}FAIL: There were $TIMEOUT_COUNT timeouts :/${NC}"
	fi

	cleanup_exit 5