earl_earl_url.sh

#! /bin/bash

folder=$1
watchlist=$1/watchlist
sleep=$2


slack_post() {
  message=$1
  payload='payload={"channel": "#earl-earl-url", "username": "earlbot", "text": "'$message'", "icon_emoji": ":ghost:"}'

  curl -X POST --data-urlencode "$payload" $SLACK_URL
  echo done

}


while true; do
  for url in `cat "$watchlist"`; do 
    file=$folder/`echo "$url" | sed 's/[^0-9a-zA-Z\.]//g'`
    save_file="${file}_d`date +%Y%m%d-%H%M%S`.txt"
    last_file=`ls -tr ${file}_d* | tail -n 1`

    if [ "$last_file" != "" ]; then
      last_modified=`cat $last_file | grep Last-Modified | sed 's/Last-Modified://'`
    else
      slack_post "Now watching $url"
    fi

    if [ "$last_modified" == '' ]; then
      last_modified=" Fri, 09 Sep 2011 17:03:56 GMT"
    fi


    echo "Last Modified:$last_modified"
    curl --header "If-Modified-Since:$last_modified" -io "$save_file" "$url"
    not_modified=`cat $save_file | grep "HTTP/1.1 304 Not Modified" | wc -l`

    echo "CURL done!"
    if [ "$not_modified" -ne "0" ]; then
      echo no change
      rm "$save_file"
    elif [ -f "$last_file" ]; then
      echo "Diffing against ${last_file}"
      diff=`diff -u <(grep -v Date: $last_file) <(grep -v Date: $save_file)`
      diff_lines=`echo $diff | wc -l`
      
      if [ "$diff" == "" ]; then
        echo "No Change"
        rm $save_file
      else 
        echo "Change"
        echo $diff
        slack_post "$url has been updated"
      fi
    fi
    echo Cat Nap
    sleep 2;
  done
  echo Done with all URLs! Sleeping...
  sleep $sleep;
done