Fetch the latest application log from the Spark cluster and target specific information

fetch-log

#!/bin/bash
# spark.hosts lists all the nodes in the cluster

SPARK_DIR=/home/yanzhe/spark-0.8.1-incubating

LOG_NUM=`ls -t $SPARK_DIR/work | head -n 1`

# tricks to highlight the application id
red='\e[0;31m'
NC='\e[0m' # No Color
echo -e Application ID: "${red}$LOG_NUM${NC}"

FW_DIR="$(cd `dirname $0`; pwd)"

for node in `cat $FW_DIR/spark.hosts`
do
        mkdir -p $node
        rsync -av yanzhe@$node:$SPARK_DIR/work/$LOG_NUM/[0-9]/ $FW_DIR/$node
done

parse-log

#!/bin/bash
# spark.hosts lists all the nodes in the cluster

if [ -z "$1" ]; then
  echo "Usage: run-mapmatch regex" >&2
  exit 1
fi

SPARK_DIR=/home/yanzhe/spark-0.8.1-incubating
FW_DIR="$(cd `dirname $0`; pwd)"

for node in `cat $FW_DIR/spark.hosts`
do
        grep -rHn --color=auto -e $1 $node
done