Find strings that appear on any website.

findstring.sh

#!/bin/bash
# -----------  SET COLORS  -----------
COLOR_RED=$'\e[31m'
COLOR_CYAN=$'\e[36m'
COLOR_YELLOW=$'\e[33m'
COLOR_GREEN=$'\e[32m'
COLOR_RESET=$'\e[0m'
COLOR_HIGHLIGHT_FOUND=$'\e[0;30;42m'
#DOMAIN=https://www.efficiencyofmovement.com


echo "Let's search a website for a specfici string of text"
read -rp "Enter the full URL: " DOMAIN
echo "Cool, I have the URL of: $DOMAIN"
read -rp "Now input the string you want to search for: " STRING


spin()
{
  spinner="/|\\-/|\\-"
  while :
  do
    for i in `seq 0 7`
    do
      echo -n "${spinner:$i:1}"
      echo -en "\010"
      sleep 1
    done
  done
}

spin &
SPIN_PID=$!
trap 'kill -9 $SPIN_PID' $(seq 0 15)
## get all the working 200 URLs from a website
wget --spider --force-html -r "$DOMAIN" 2>&1 |
  grep '^--' | awk '{ print $3 }' |
  grep -E -v '\.(css|js|json|map|xml|png|gif|jpg|jpeg|JPG|bmp|txt|pdf|webmanifest)(\?.*)?$' |
  grep -E -v '\?(p|replytocom)=' |
  grep -E -v '\/wp-content\/uploads\/' |
  grep -E -v '\/feed\/' |
  grep -E -v '\/category\/' |
  grep -E -v '\/tag\/' |
  grep -E -v '\/page\/' |
  grep -E -v '\/widgets.php$' |
  grep -E -v '\/wp-json\/' |
  grep -E -v '\/xmlrpc' |
  grep -E '[/]$' |
  sort -u \
    > log.txt

#input="log.txt"
while IFS= read -r line; do

  if wget -q "$line" -O - | grep --color=always -ni -C 1 "$STRING"; then
    echo ${COLOR_RESET}${COLOR_GREEN}"Found (a) match(es) on: ${COLOR_HIGHLIGHT_FOUND}$line"${COLOR_RESET}
  else
    echo ${COLOR_RESET}${COLOR_CYAN}Nothing found on: "$line"${COLOR_RESET}
  fi
done < log.txt
kill -9 $SPIN_PID