Batch grading using autograder.py from Berkeley AI course

batch_grading.sh

#!/usr/bin/env bash
# batch_grading.sh for COMP3270
# by Yijie
# example: 
# (1) ZIP_DIR=folder_of_decompressed_submissions STUDENT_ROOT=output_folder PROJECT=a1 timelimit=60s ./batch_grading.sh
# (2) ./batch_grading.sh summary time_60s.txt > error_ids.txt

CMD="$0"
ZIP_DIR=${ZIP_DIR:-../a1_zip} # download all submitted files and decompress into this directory
STUDENT_ROOT=${STUDENT_ROOT:-student_code}
COURSE=COMP3270
PROJECT=${PROJECT:-a1}

# check
if ! [ -f ./autograder.py ] ; then
  echo "Please run in the folder containing autograder.py!";
  exit 1
fi

CODE_FILE="$(grep CODE projectParams.py)";
CODE_FILE=${CODE_FILE#*"= '"};
CODE_FILE=${CODE_FILE%"'"};
CODE_FILE=$(echo $CODE_FILE | sed 's/,/ /g');

# set up directory structure 
setup(){
  echo "code files for ${PROJECT}: ${CODE_FILE}"
  mkdir -p ${STUDENT_ROOT}/${PROJECT}_backup
  # will change to ${PROJECT}_${STUDENT_ID} dynamically
  ln -s ${PROJECT}_backup ${STUDENT_ROOT}/${PROJECT}_active;

  for CF in ${CODE_FILE}; do 
    mv ./$CF ${STUDENT_ROOT}/${PROJECT}_backup/    
    ln -s ${STUDENT_ROOT}/${PROJECT}_active/$CF ./$CF;
  done
  # storing outputs
  ln -s ${STUDENT_ROOT}/${PROJECT}_active/gradescope_response.json ./gradescope_response.json

  # student ids to grade
  ls ${ZIP_DIR}/*/${PROJECT}_*.zip | rev | cut -d'/' -f1 |rev |cut -d'_' -f2|cut -d'.' -f1 | sort > ${STUDENT_ROOT}/student_ids.txt
}

# for each STUDENT_ID update softlink:
grade_one(){
  local STUDENT_ID=${1:-backup} # default
  local OUTPUT_DIR=${STUDENT_ROOT}/${PROJECT}_${STUDENT_ID}
  mkdir -p ${OUTPUT_DIR}/
  # check if score exists
  if [ -f ${OUTPUT_DIR}/grading_output.log ] ; then
    TMP="$( < ${OUTPUT_DIR}/gradescope_response.json)"
    if [ "${#TMP}" -gt 10 ] ; then
      echo "student $STUDENT_ID already graded!";
      return;
    fi
  fi

  unzip -o -q -j ${ZIP_DIR}/*/${PROJECT}_${STUDENT_ID}.zip -d ${OUTPUT_DIR}/ || \
    ( echo "student $STUDENT_ID not found!" 1>&2 && return )
  ln -sfn ${PROJECT}_${STUDENT_ID} ${STUDENT_ROOT}/${PROJECT}_active;
  
  echo "{}" > ${OUTPUT_DIR}/gradescope_response.json;
  local START=$(date +%s)
  python autograder.py --gradescope-output --mute > ${OUTPUT_DIR}/grading_output.log;
  local END=$(date +%s)
  local DIFF=$(( $END - $START ))
  echo "student ${STUDENT_ID} took ${DIFF} seconds"
  echo "${STUDENT_ID} time: ${DIFF} seconds" > ${OUTPUT_DIR}/time.log

}

# gather scores
summary(){
  for ii in ${STUDENT_ROOT}/*/gradescope_response.json; do
    local STUDENT_ID=$(echo $ii | cut -d'/' -f2 | cut -d'_' -f2);
    local STUDENT_SCORE=$(cat $ii | cut -d',' -f1 | cut -d":" -f2);
    if [ "$STUDENT_ID" == "active" ] ; then continue; fi;
    if [[ "$STUDENT_SCORE" == "{}" ]] ; then continue; fi;
    echo "$STUDENT_ID: $STUDENT_SCORE";
  done
  # run your python script to collect scores from json
}

# detect plagiarism
# TODO: add solutions on the Web (e.g., GitHub)
detect(){
  local BASE_FLAG=$(for CF in $CODE_FILE; do echo -n "-b ${STUDENT_ROOT}/${PROJECT}_backup/$CF "; done)
  local ALL_FILES=$(for CF in $CODE_FILE; do ls ${STUDENT_ROOT}/${PROJECT}_30*/$CF; done)
  MOSS_CMD="${MOSS_CMD:-../moss}"
  ${MOSS_CMD} -l python -m 10 -n 100 ${BASE_FLAG} -c "${COURSE}-${PROJECT}" -d ${ALL_FILES}

  # link to the results is the last line of outputs
}

download(){
  local INDEX="$1"
  local OUTPUT_DIR="$2"
  wget -r -l 2 -e robots=off -nd -P ${OUTPUT_DIR} -q -k ${INDEX}
}

clean(){
  rm ${CODE_FILE};
  rm ./gradescope_response.json;
  for CF in ${CODE_FILE}; do
    mv ${STUDENT_ROOT}/${PROJECT}_backup/${CF} ./
  done
  rm -rf ${STUDENT_ROOT}  
}


# options
if [ "$1" == "clean" ] ; then
  echo "cleaning ...";
  clean;
  exit 0
elif [ "$1" == "summary" ]; then
  echo "summary ...";
  OUTFILE="${2:-/dev/stdin}"
  summary > ${OUTFILE};
  if [ "${OUTFILE}" != "/dev/stdin" ] ; then
    echo "students with no outputs:"
    cat ${OUTFILE} | cut -d':' -f1| diff ${STUDENT_ROOT}/student_ids.txt - | grep '<'|cut -d' ' -f2
  fi
  exit 0;
elif [ "$1" == "setup" ]; then
  echo "setup ...";
  setup;
  exit 0;
elif [ "$1" == "detect" ]; then
  echo "checking plagiarism ..."
  INDEX=$(detect | tail -1);
  echo "downloading ${INDEX} ...";
  download ${INDEX}/index.html moss_${COURSE}_${PROJECT};
  exit 0;
elif [ "$1" == "grading" ]; then
  grade_one $2;
  exit 0;
fi

# main

if [ -e ./gradescope_response.json ] ; then
  echo "already setup!"
else
  setup;
fi

TIME_CMD=timeout
which ${TIME_CMD} || TIME_CMD=gtimeout # MacOS: brew install coreutils
timelimit=${timelimit:-60s}

TOTAL=$(cat ${STUDENT_ROOT}/student_ids.txt | wc -l|sed -e 's/ //g')
kk=0
for ii in $(<${STUDENT_ROOT}/student_ids.txt); do
  ((kk=kk+1))
  echo "$kk/$TOTAL: start grading $ii at $(date)"
  ${TIME_CMD} ${timelimit} ${CMD} grading $ii;
done