Skip to content

Instantly share code, notes, and snippets.

@joshsmith01
Created September 4, 2019 18:00
Show Gist options
  • Save joshsmith01/d9aa3524c0c0a41232a84c18bf43df2b to your computer and use it in GitHub Desktop.
Save joshsmith01/d9aa3524c0c0a41232a84c18bf43df2b to your computer and use it in GitHub Desktop.
Find strings that appear on any website.
#!/bin/bash
# ----------- SET COLORS -----------
COLOR_RED=$'\e[31m'
COLOR_CYAN=$'\e[36m'
COLOR_YELLOW=$'\e[33m'
COLOR_GREEN=$'\e[32m'
COLOR_RESET=$'\e[0m'
COLOR_HIGHLIGHT_FOUND=$'\e[0;30;42m'
#DOMAIN=https://www.efficiencyofmovement.com
echo "Let's search a website for a specfici string of text"
read -rp "Enter the full URL: " DOMAIN
echo "Cool, I have the URL of: $DOMAIN"
read -rp "Now input the string you want to search for: " STRING
spin()
{
spinner="/|\\-/|\\-"
while :
do
for i in `seq 0 7`
do
echo -n "${spinner:$i:1}"
echo -en "\010"
sleep 1
done
done
}
spin &
SPIN_PID=$!
trap 'kill -9 $SPIN_PID' $(seq 0 15)
## get all the working 200 URLs from a website
wget --spider --force-html -r "$DOMAIN" 2>&1 |
grep '^--' | awk '{ print $3 }' |
grep -E -v '\.(css|js|json|map|xml|png|gif|jpg|jpeg|JPG|bmp|txt|pdf|webmanifest)(\?.*)?$' |
grep -E -v '\?(p|replytocom)=' |
grep -E -v '\/wp-content\/uploads\/' |
grep -E -v '\/feed\/' |
grep -E -v '\/category\/' |
grep -E -v '\/tag\/' |
grep -E -v '\/page\/' |
grep -E -v '\/widgets.php$' |
grep -E -v '\/wp-json\/' |
grep -E -v '\/xmlrpc' |
grep -E '[/]$' |
sort -u \
> log.txt
#input="log.txt"
while IFS= read -r line; do
if wget -q "$line" -O - | grep --color=always -ni -C 1 "$STRING"; then
echo ${COLOR_RESET}${COLOR_GREEN}"Found (a) match(es) on: ${COLOR_HIGHLIGHT_FOUND}$line"${COLOR_RESET}
else
echo ${COLOR_RESET}${COLOR_CYAN}Nothing found on: "$line"${COLOR_RESET}
fi
done < log.txt
kill -9 $SPIN_PID
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment