Grading for CSCI 561

grade.sh

#!/bin/bash

echo "HW3 Report" > $vocareumReportFile
echo "Programming language..." >> $vocareumReportFile

filename=$(ls|grep NeuralNetwork)
if [[ $filename =~ (^|[[:space:]])"NeuralNetwork.py"($|[[:space:]]) ]]; then
	cmd="python NeuralNetwork.py train_image.csv train_label.csv test_image.csv"
    mnist_cmd="python NeuralNetwork.py grading_train_image.csv grading_train_label.csv grading_test_image.csv"
    ta_cmd="python NeuralNetwork.py grading_train_image.csv grading_train_label.csv additional_test_image.csv"
    trick_cmd="python NeuralNetwork.py train_trick_image.csv train_trick_label.csv test_image.csv"
	echo "Python2" >> $vocareumReportFile
elif [[ $filename =~ (^|[[:space:]])"NeuralNetwork3.py"($|[[:space:]]) ]]; then
    cmd="python3 NeuralNetwork3.py train_image.csv train_label.csv test_image.csv"
    mnist_cmd="python3 NeuralNetwork3.py grading_train_image.csv grading_train_label.csv grading_test_image.csv"
    ta_cmd="python3 NeuralNetwork3.py grading_train_image.csv grading_train_label.csv additional_test_image.csv"
    trick_cmd="python3 NeuralNetwork3.py train_trick_image.csv train_trick_label.csv test_image.csv"
	echo "Python3" >> $vocareumReportFile
elif [[ $filename =~ (^|[[:space:]])"NeuralNetwork.cpp"($|[[:space:]]) ]]; then
	g++ *.cpp -o NeuralNetwork
	cmd="./NeuralNetwork train_image.csv train_label.csv test_image.csv"
    trick_cmd="./NeuralNetwork train_trick_image.csv train_trick_label.csv test_image.csv"
	echo "C++" >> $vocareumReportFile
elif [[ $filename =~ (^|[[:space:]])"NeuralNetwork11.cpp"($|[[:space:]]) ]]; then
	g++ -std=c++0x *.cpp -o NeuralNetwork
	cmd="./NeuralNetwork train_image.csv train_label.csv test_image.csv"
    trick_cmd="./NeuralNetwork train_trick_image.csv train_trick_label.csv test_image.csv"
	echo "C++11" >> $vocareumReportFile
elif [[ $filename =~ (^|[[:space:]])"NeuralNetwork.java"($|[[:space:]]) ]]; then
	javac NeuralNetwork.java
	cmd="java NeuralNetwork train_image.csv train_label.csv test_image.csv"
    trick_cmd="java NeuralNetwork train_trick_image.csv train_trick_label.csv test_image.csv"
	echo "JAVA" >> $vocareumReportFile
else
    echo "ERROR: Invalid Filename" >> $vocareumReportFile
	exit 1
fi

if [ -f "test_image.csv" ]; then
    rm test_image.csv
fi
if [ -f "test_label.csv" ]; then
    rm test_label.csv
fi
if [ -f "train_image.csv.csv" ]; then
    rm train_image.csv.csv
fi
if [ -f "train_label.csv" ]; then
    rm train_label.csv
fi

cp $ASNLIB/grade/mnist.pkl.gz mnist.pkl.gz
python3 $ASNLIB/mnist_csv.py
cp $ASNLIB/grade/additional_test_image.csv .
cp $ASNLIB/grade/additional_test_label.csv .

echo Start processing.. >> $vocareumReportFile
echo $(date) >> $vocareumReportFile

cheat_bar=50.00

# Detect cheating
penalty=0
echo "Detecting cheating of dataset by using 10 random samples for training.." >> $vocareumReportFile
eval "$trick_cmd"
acc=$(python3 $ASNLIB/check.py test_label.csv test_predictions.csv)
if [ -z "$acc" ]; then
    echo "Cheating test no output" >> $vocareumReportFile
    echo "Cheating-check acc: $acc" >> $vocareumReportFile 
    echo "It is possbile that:" >> $vocareumReportFile
    echo "1. your program does not read the filenames as the parameters from the command." >> $vocareumReportFile
    echo "2. your program read test_label.csv during the execution, which is not expected." >> $vocareumReportFile
else
    echo $acc >> $vocareumReportFile
    if (( $(echo "$acc > $cheat_bar" | bc -l) )); then
        echo "It is possbile that:" >> $vocareumReportFile
        echo "1. your program does not only use the specified dataset for training." >> $vocareumReportFile
        echo "2. your program does not read the filenames as the parameters from the command." >> $vocareumReportFile
        echo "3. your program read test_label.csv during the execution, which is not expected." >> $vocareumReportFile
        penalty=1
    fi
    rm test_predictions.csv
fi

echo Start grading.. >> $vocareumReportFile
echo $(date) >> $vocareumReportFile


score=0

eval "$cmd"
acc=$(python3 $ASNLIB/check.py test_label.csv test_predictions.csv)
rm test_predictions.csv

if [ -z "$acc" ]; then
    echo "MNIST-Test no output" >> $vocareumReportFile
echo $acc >> $vocareumReportFile
else
    echo "(non-grading stage) MNIST-ACC: $acc" >> $vocareumReportFile 
fi


eval "$mnist_cmd"
mnist_acc=$(python3 $ASNLIB/check.py grading_test_label.csv test_predictions.csv)
rm test_predictions.csv

if [ -z "$mnist_acc" ]; then
    echo "MNIST-Test no output" >> $vocareumReportFile
echo $mnist_acc >> $vocareumReportFile
else
    # echo "MNIST-ACC: $mnist_acc" >> $vocareumReportFile 
    hidden_mnist_acc=$(python3 $ASNLIB/hide_acc.py $mnist_acc)  
    echo "MNIST-ACC: hidden for now ($hidden_mnist_acc)" >> $vocareumReportFile 
fi


eval "$ta_cmd"
ta_acc=$(python3 $ASNLIB/check.py additional_test_label.csv test_predictions.csv)
rm test_predictions.csv

if [ -z "$ta_acc" ]; then
    echo "TA-Test no output" >> $vocareumReportFile
echo $ta_acc >> $vocareumReportFile
else
    # echo "TA-ACC: $ta_acc" >> $vocareumReportFile 
    hidden_ta_acc=$(python3 $ASNLIB/hide_acc.py $ta_acc)  
    echo "TA-ACC: hidden for now ($hidden_ta_acc)" >> $vocareumReportFile 
fi


score=$(python3 $ASNLIB/acc_grading.py $mnist_acc $ta_acc)

if (( penalty==1 )); then
    score=0
fi

echo Mission Completed. >> $vocareumReportFile
echo $(date) >> $vocareumReportFile


# score=-1 # for hiding the score from students now, as we haven't finished this part. They can see the accuracy, though.

echo "score", $score >> $vocareumReportFile
echo "score", $score >> $vocareumGradeFile




# mnist_acc=-1

# echo "MNIST-acc", $mnist_acc >> $vocareumReportFile
# echo "MNIST-acc", $mnist_acc >> $vocareumGradeFile


# ta_acc=-1
# echo "TA-acc", $ta_acc >> $vocareumReportFile
# echo "TA-acc", $ta_acc >> $vocareumGradeFile