clone site recursively and generate hashes for documents

gistfile1.txt

#!/bin/bash
domain=$1
location=$2
if [[ $# -eq 0 ]] ; then
echo '-------------------------------'    
echo 'please supply url, e.g C2'
echo '-------------------------------'
    exit 0
fi
if [[ $# -eq 1 ]] ; then
echo '------------------------------------------------'    
echo 'please supply storage location without ending / '
echo '------------------------------------------------'
    exit 0
fi
wget --limit-rate=150k --no-clobber --convert-links --random-wait -r -p -E -e robots=off -U mozilla -t 6 -T 6 $client
cp -R $client $location/$client
rm -R $client
#finding files that could be malicious
find $location/$client -iname "*.exe" > $location/$client/orig.list
find $location/$client -iname "*.pdf" >> $location/$client/orig.list
find $location/$client -iname "*.xls" >> $location/$client/orig.list
find $location/$client -iname "*.xlsx" >> $location/$client/orig.list
find $location/$client -iname "*.doc" >> $location/$client/orig.list
find $location/$client -iname "*.docx" >> $location/$client/orig.list
find $location/$client -iname "*.odt" >> $location/$client/orig.list
find $location/$client -iname "*.ods" >> $location/$client/orig.list
find $location/$client -iname "*.opd" >> $location/$client/orig.list
while read line; do md5sum "$line" >> $location/$client/tmp.list; done <$location/$client/orig.list
sed 's/\s.*$//' $location/$client/tmp.list > $location/$client/md5sum.list
rm $location/$client/tmp.list
cat $location/$client/orig.list
cat $location/$client/md5sum.list
#then we can feed this list into a automated hash checker