Many children performance test

shallow_performance_test.sh

#!/bin/bash
#set -x

#### CONFIG VARIABLES ####

# Your base Fedora 
BASE_CONTAINER="http://localhost:8080/fcrepo6/rest"
# Total number of children to create
TOTAL_CONTAINERS=15000
# When to update creation time
MARK_AT=100
# When to update and move (so we don't overwrite this average creation time)
MARK_AND_MOVE_AT=500

# The messages
# -H"Content-type: text/turtle" -d" @prefix test: <http://www.example.org/ns/test#> . <> a test:Container ."
# -H"Content-type: application/ld+json" -d"@andrey.jsonld"

#### THE SCRIPT ####

TOTAL_TIME=0
MARK_TIME=0
MARK_AND_MOVE_TIME=0
MOVE_LAG=0
CURRENT_LAG=0
USE_TX=0
COMMIT_TIME=0
COMMIT_COUNT=0

if [ $# -ge 1 ]; then
  for opt in "$@"; do
    case $opt in
      "transactions")
        USE_TX=1
        ;;
      *)
        echo "Unknown option, only transactions is currently valid."
        exit 1
     esac
   done
fi

function getTransaction() {
  TX_LOCATION=$(curl -s -i -XPOST -ufedoraAdmin:fedoraAdmin $BASE_CONTAINER/fcr:tx | sed -e '/^Location:/!d' -e 's/^Location: //' | tr -d '\r')
}

function commitTransaction() {
  local RES=$(curl -s --write-out "%{http_code}~%{time_total}" -ufedoraAdmin:fedoraAdmin -XPUT $TX_LOCATION)
  local CODE=$(echo "$RES" | cut -d'~' -f1)
  local TIME=$(echo "$RES" | cut -d'~' -f2)
  if [[ "$CODE" != "204" ]]; then
    printf "\n\n%s\n" "Error received committing transaction $TX_LOCATION, got $CODE"
    exit 1
  fi
  COMMIT_TIME=$(echo "$COMMIT_TIME+$TIME" | bc )
  COMMIT_COUNT=$(expr $COMMIT_COUNT + 1)
}


NEW_BASE=$(curl -s -XPOST -ufedoraAdmin:fedoraAdmin -H"Content-type: text/turtle" -d" @prefix test: <http://www.example.org/ns/test#> . <> a test:Container ." $BASE_CONTAINER)
echo "new base is $NEW_BASE"

if [[ $USE_TX -eq 1 ]]; then
  getTransaction
  echo "Using transactions for each $MARK_AND_MOVE_AT containers"
fi

START_TIME=$(date +"%s")
for COUNTER in $(seq 1 $TOTAL_CONTAINERS ); do
  if [[ $USE_TX -eq 1 ]]; then
    # Done like this because shell quoting was breaking stuff
    RES=$(curl -sS --write-out "%{http_code}~%{time_total}" -H"Atomic-ID: $TX_LOCATION" -o "/dev/null" -ufedoraAdmin:fedoraAdmin -XPOST -H"Content-type: application/ld+json" -d"@andrey.jsonld" $NEW_BASE)
  else
    RES=$(curl -sS --write-out "%{http_code}~%{time_total}" -o "/dev/null" -ufedoraAdmin:fedoraAdmin -XPOST -H"Content-type: application/ld+json" -d"@andrey.jsonld" $NEW_BASE)
  fi
  CODE=$(echo "$RES" | cut -d'~' -f1)
  TIME=$(echo "$RES" | cut -d'~' -f2) 
  if [ "$CODE" != "201" ]; then
    printf "\n%s\n" "Error received code $CODE POSTing to $NEW_BASE"
    exit 1
  fi
  MARK_TIME=$(echo "$MARK_TIME+$TIME" | bc )
  if [ $(echo "$MARK_TIME==0" | bc ) -eq 0 -a $(echo "$COUNTER%$MARK_AT" | bc ) -eq 0 ]; then
    CURRENT_LAG=$(echo "scale=5;$MARK_TIME/$MARK_AT" | bc )
    TOTAL_TIME=$(echo "$TOTAL_TIME+$MARK_TIME" | bc )
    MARK_TIME=0
  fi
  MARK_AND_MOVE_TIME=$(echo "$MARK_AND_MOVE_TIME+$TIME" | bc )

  if [ $(echo "$MARK_AND_MOVE_TIME==0" | bc ) -eq 0 -a $(echo "$COUNTER%$MARK_AND_MOVE_AT" | bc ) -eq 0 ]; then
    LAST_RANGE=$( echo "$COUNTER-$MARK_AND_MOVE_AT+1" | bc )" to $COUNTER"
    MOVE_LAG=$(echo "scale=5;$MARK_AND_MOVE_TIME/$MARK_AND_MOVE_AT" | bc )
    MARK_AND_MOVE_TIME=0
    STRING="Average time per request ${MOVE_LAG} (secs) for requests ${LAST_RANGE}"
    # This is a mark and move, so print a newline
    printf "\r%s %50s\n" "$STRING" ""
    if [[ $USE_TX -eq 1 ]]; then
      #Commit and start a new transaction
      commitTransaction
      getTransaction
    fi 
  fi
  STRING="Creating ${COUNTER} child"
  if [ $(echo "$CURRENT_LAG!=0" | bc ) -eq 1 ]; then
    STRING="${STRING} -- average time per request ${CURRENT_LAG} (secs) for the last ${MARK_AT} requests"
  fi
  printf "\r%s" "$STRING"
done
if [[ $USE_TX -eq 1 ]]; then
  commitTransaction
fi
END_TIME=$(date +"%s")

TOTAL_AVG=$(echo "scale=5;${TOTAL_TIME}/${TOTAL_CONTAINERS}" | bc)
printf "\r%s\n" "Done creating $TOTAL_CONTAINERS objects in $TOTAL_TIME (secs), average ${TOTAL_AVG} (secs) per container."

if [[ $USE_TX -eq 1 ]]; then
  TOTAL_COMMIT_AVG=$(echo "scale=5;${COMMIT_TIME}/${COMMIT_COUNT}" | bc)
  printf "%s\n" "The above time is a combination of all requests, when using transactions we need runtime to account for commits"
  printf "%s\n" "There were $COMMIT_COUNT commits taking a total of $COMMIT_TIME seconds, for an average of $TOTAL_COMMIT_AVG"
fi

TOTAL_RUN_TIME=$(expr $END_TIME - $START_TIME)
TOTAL_RUN_AVG=$(echo "scale=5;${TOTAL_RUN_TIME}/${TOTAL_CONTAINERS}" | bc)
printf "%s\n" "Total run time was ${TOTAL_RUN_TIME} (secs), averaged across ${TOTAL_CONTAINERS} is ${TOTAL_RUN_AVG} (secs) per container"

TIME=$(curl -s --write-out "%{time_total}" -ufedoraAdmin:fedoraAdmin -H "Accept: application/n-triples" $NEW_BASE -o /dev/null)
printf "\nRetrieved $NEW_BASE to warm cache in $TIME seconds\n"
TIME=$(curl -s --write-out "%{time_total}" -ufedoraAdmin:fedoraAdmin -H "Accept: application/n-triples" -o "n-children.nt" $NEW_BASE )
printf "\nRetrieved $NEW_BASE in $TIME seconds\n"
if [ ! -f "n-children.nt" ]; then
  echo "Problem locating n-children.nt"
else
  CONTAINED=$(grep -c 'ldp#contains' n-children.nt)
  echo "Number of contained children: $CONTAINED"
fi
exit