Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#!/bin/sh
#Configuration Parameters
APACHE_LOG_DIR=<Your apache log dir with full path>
APACHE_LOG_FILE=<Your apache log file with full path>
SCRIPT_DIR=<Your scripts directory with full path>
SERVICES_LIST=<Space separated list of service URIs for which you want to assess performance>
FROM_EMAIL_ADDR=<From email address>
TO_EMAIL_ADDRESSES=<Comma separated list of to email addresses>
SERVICE_BASE_URI=<Your service base URI that can identify all your services>
DATA_SERVICES_NAME=<Name of your data services>
if [ $# -ne 1 ]
then
AS_OF_DATE=`date +%d/%b/%Y`
else
AS_OF_DATE=$1
fi
CURRENT_TIME=`date +%s`
CURRENT_DATE=`date +%Y%m%d`
TEMP_FILE_NAME=${APACHE_LOG_FILE}_${CURRENT_DATE}_${CURRENT_TIME}.log
BAD_PERFORMING_REQUESTS_FILE=${APACHE_LOG_FILE}_bottleneck_${CURRENT_DATE}_${CURRENT_TIME}.log
SERVICES_TIMEOUTS_FILE=${APACHE_LOG_DIR}/service_timeouts.log
RPT_FILE_NAME=${APACHE_LOG_FILE}_${CURRENT_DATE}_${CURRENT_TIME}.rpt
EMAIL_HDR_FILE_NAME=${SCRIPT_DIR}/getApachePerformanceStats.email
echo "From: ${FROM_EMAIL_ADDR}" > ${EMAIL_HDR_FILE_NAME}
echo "To: ${TO_EMAIL_ADDRESSES}" >> ${EMAIL_HDR_FILE_NAME}
echo "Subject: Apache Log Miner Stats Report as of ${AS_OF_DATE} run on `date`" >> ${EMAIL_HDR_FILE_NAME}
echo "Content-Type: text/html" >> ${EMAIL_HDR_FILE_NAME}
echo "<HTML>" >> ${EMAIL_HDR_FILE_NAME}
echo "<HEAD>" >> ${EMAIL_HDR_FILE_NAME}
echo "<STYLE>" >> ${EMAIL_HDR_FILE_NAME}
echo "th { background-color : #cccccc; text-align : left;}" >> ${EMAIL_HDR_FILE_NAME}
echo "</STYLE>" >> ${EMAIL_HDR_FILE_NAME}
echo "</HEAD>" >> ${EMAIL_HDR_FILE_NAME}
echo "<BODY>" >> ${EMAIL_HDR_FILE_NAME}
echo "<h1>Service-wise 90th Percentile</h1>" >> ${EMAIL_HDR_FILE_NAME}
echo "<table>" >> ${EMAIL_HDR_FILE_NAME}
echo "<tr>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Service Name" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Total Requests" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Successful Requests" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Failures" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "90th Percentile (micro seconds)" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Max(micro seconds)" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "<th>" >> ${EMAIL_HDR_FILE_NAME}
echo "Min(micro seconds)" >> ${EMAIL_HDR_FILE_NAME}
echo "</th>" >> ${EMAIL_HDR_FILE_NAME}
echo "</tr>" >> ${EMAIL_HDR_FILE_NAME}
grep "${AS_OF_DATE}" ${APACHE_LOG_FILE} | grep "${SERVICE_BASE_URI}" > ${TEMP_FILE_NAME}
for service in ${SERVICES_LIST}
do
#echo "Processing $service ..."
SAMPLE_SUCCESS_COUNT=$(grep "${service}" ${TEMP_FILE_NAME} | grep -c " 200 " )
SAMPLE_FAILURE_COUNT=$(grep "${service}" ${TEMP_FILE_NAME} | grep -cv " 200 " )
SAMPLE_TOTAL_COUNT=$(grep -c "${service}" ${TEMP_FILE_NAME} )
PERCENTILE_90=`grep "${service}" ${TEMP_FILE_NAME} | grep " 200 " | grep -Po "(\d+)\-\-$" | sed 's/--//g' |sort -n|awk 'BEGIN{i=0} {s[i]=$1; i++;} END{print s[int(NR*0.90-0.5)]}'`
MIN_TIME=`grep "${service}" ${TEMP_FILE_NAME} | grep " 200 " | grep -Po "(\d+)\-\-$" | sed 's/--//g' |sort -n|head -1`
MAX_TIME=`grep "${service}" ${TEMP_FILE_NAME} | grep " 200 " | grep -Po "(\d+)\-\-$" | sed 's/--//g' |sort -n|tail -1`
#echo "90th percentile for [$service] = [${PERCENTILE_90}]"
echo "<tr>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td>" >> ${EMAIL_HDR_FILE_NAME}
echo "${service}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td align=right>" >> ${EMAIL_HDR_FILE_NAME}
echo "${SAMPLE_TOTAL_COUNT}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td align=right>" >> ${EMAIL_HDR_FILE_NAME}
echo "${SAMPLE_SUCCESS_COUNT}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td align=right>" >> ${EMAIL_HDR_FILE_NAME}
echo "${SAMPLE_FAILURE_COUNT}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
if [ ${PERCENTILE_90:-0} -gt 2000000 ]
then
echo "<td align='right' style='background-color : orange'>" >> ${EMAIL_HDR_FILE_NAME}
else
echo "<td align='right' style='background-color : lightgreen'>" >> ${EMAIL_HDR_FILE_NAME}
fi
if [ ${PERCENTILE_90:-0} -gt 4000000 ]
then
echo "<TIMEOUT> ${service} ${SAMPLE_TOTAL_COUNT} ${SAMPLE_SUCCESS_COUNT} ${SAMPLE_FAILURE_COUNT} ${PERCENTILE_90:-0} ``date" >> ${SERVICES_TIMEOUTS_FILE}
fi
echo "${PERCENTILE_90}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td align=right>" >> ${EMAIL_HDR_FILE_NAME}
echo "${MAX_TIME}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "<td align=right>" >> ${EMAIL_HDR_FILE_NAME}
echo "${MIN_TIME}" >> ${EMAIL_HDR_FILE_NAME}
echo "</td>" >> ${EMAIL_HDR_FILE_NAME}
echo "</tr>" >> ${EMAIL_HDR_FILE_NAME}
done
echo "</table>" >> ${EMAIL_HDR_FILE_NAME}
echo "<h1>${DATA_SERVICES_NAME} - Overall Performance</h1>" >> ${EMAIL_HDR_FILE_NAME}
echo "<HR>" > ${RPT_FILE_NAME}
echo "<TABLE>" >> ${RPT_FILE_NAME}
echo "<TR>" >> ${RPT_FILE_NAME}
echo "<TH>" >> ${RPT_FILE_NAME}
echo "As of Date" >> ${RPT_FILE_NAME}
echo "<TD>" >> ${RPT_FILE_NAME}
echo "${AS_OF_DATE}" >> ${RPT_FILE_NAME}
echo "</TR>" >> ${RPT_FILE_NAME}
NUMBER_OF_FAILURES=`grep -cv " 200 " ${TEMP_FILE_NAME}`
TOTAL_SERVICE_CALLS=$(grep -Po "(\d+)\-\-$" ${TEMP_FILE_NAME} | sed 's/--//g' | sort -n | wc -l)
echo "<TR>" >> ${RPT_FILE_NAME}
echo "<TH>" >> ${RPT_FILE_NAME}
echo "Number of service calls" >> ${RPT_FILE_NAME}
echo "<TD>" >> ${RPT_FILE_NAME}
echo "${TOTAL_SERVICE_CALLS}" >> ${RPT_FILE_NAME}
echo "</TR>" >> ${RPT_FILE_NAME}
if [ ${NUMBER_OF_FAILURES} -gt 0 ]
then
echo "<TR>" >> ${RPT_FILE_NAME}
echo "<TH>" >> ${RPT_FILE_NAME}
echo "Number of failures" >> ${RPT_FILE_NAME}
echo "<TD style='background-color : red;'>" >> ${RPT_FILE_NAME}
echo "${NUMBER_OF_FAILURES}" >> ${RPT_FILE_NAME}
echo "</TR>" >> ${RPT_FILE_NAME}
echo "<TR>" >> ${RPT_FILE_NAME}
echo "<TH>" >> ${RPT_FILE_NAME}
echo "% of failures" >> ${RPT_FILE_NAME}
echo "<TD style='background-color : red;'>" >> ${RPT_FILE_NAME}
echo "$(echo "scale=6; (${NUMBER_OF_FAILURES}/${TOTAL_SERVICE_CALLS})*100" | bc)" >> ${RPT_FILE_NAME}
echo "</TR>" >> ${RPT_FILE_NAME}
fi
echo "</TABLE>" >> ${RPT_FILE_NAME}
echo "<TABLE>" >> ${RPT_FILE_NAME}
grep " 200 " ${TEMP_FILE_NAME} | grep -Po "(\d+)\-\-$" | sed 's/--//g' | xargs -I NUMBER expr NUMBER / 1000000 | sort -n | uniq -c| sed 's/ //g' | sed 's/^ //g' | sed 's/ /|/g' | awk -v totrows=${TOTAL_SERVICE_CALLS} -F'|' 'BEGIN{tgtpercentile=90; percent=0; tgtpercentilereached="false";printf "<TR><TH>No. of Requests<TH>Response time (secs)<TH>Total Rows<TH>Percentile</TR>"; backgroundColor="white";} { percent+=($1/totrows)*100; if(percent>tgtpercentile && tgtpercentilereached == "false") { tgtpercentilereached = "true"; backgroundColor = "lightgreen";} else {backgroundColor = "WHITE";} printf "<TR style=\"background-color : %s\"><TD>%15d<TD>%19d<TD>%10d<TD>%f</TR>",backgroundColor, $1,$2,totrows,percent} END{}' >> ${RPT_FILE_NAME}
echo "</TABLE>" >> ${RPT_FILE_NAME}
echo "<BR>" >> ${RPT_FILE_NAME}
echo "<HR>" >> ${RPT_FILE_NAME}
echo "<h1>Requests with response time greater than 2 seconds</h1>" >> ${RPT_FILE_NAME}
echo "<div style='font-size : 10px;'>" >> ${RPT_FILE_NAME}
grep "${AS_OF_DATE}" ${APACHE_LOG_FILE} | grep "${SERVICE_BASE_URI}" | awk -f ${SCRIPT_DIR}/getApachePerformanceStats.awk | tail -1000 > ${BAD_PERFORMING_REQUESTS_FILE}
cat ${BAD_PERFORMING_REQUESTS_FILE} >> ${RPT_FILE_NAME}
echo "</div>" >> ${RPT_FILE_NAME}
echo "<HR>" >> ${RPT_FILE_NAME}
echo "<h1>Failed Requests</h1>" >> ${RPT_FILE_NAME}
echo "<div style='font-size : 10px;'>" >> ${RPT_FILE_NAME}
echo "<OL>" >> ${RPT_FILE_NAME}
grep "${AS_OF_DATE}" ${APACHE_LOG_FILE} | grep "${SERVICE_BASE_URI}" | grep -v " 200 " | xargs -I % sh -c 'echo "<li>%</li>"' >> ${RPT_FILE_NAME}
echo "</OL>" >> ${RPT_FILE_NAME}
echo "</div>" >> ${RPT_FILE_NAME}
echo "<HR>" >> ${RPT_FILE_NAME}
hostname >> ${RPT_FILE_NAME}
echo "</BODY>" >> ${RPT_FILE_NAME}
echo "</HTML>" >> ${RPT_FILE_NAME}
echo "Please check ${RPT_FILE_NAME} for a copy of this report"
rm -f ${TEMP_FILE_NAME}
cat ${EMAIL_HDR_FILE_NAME} ${RPT_FILE_NAME} | /usr/sbin/sendmail -t
echo "Please check ${BAD_PERFORMING_REQUESTS_FILE} to drill down to the bad performers"
echo "***ALL DONE***"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment