scripts_update.sh

#!/bin/bash

#######
# Creates a flat text file list of NSE scripts suitable for searching via command line
# Used in conjunction with nsesearch.sh
# Replaces nmap_scripts.list gist
# Dependencies: html2text
#######

YEL='\033[1;33m'
#RED='\033[1;31m'
#GRN='\033[1;32m'
NC='\033[0m' # No Color

echo -e "------------------------------"  
echo -e "     Performing updates:      "  
echo -e "------------------------------"
echo -e "${YEL}--${NC}Checking Dependencies"
#check for html2text
PKG_OK=$(dpkg-query -W --showformat='${Status}\n' html2text|grep "install ok installed")
# echo Checking for html2text: $PKG_OK
if [ "" == "$PKG_OK" ]; then
  echo "html2text not installed. Setting up html2text."
  apt --force-yes --yes install html2text
fi
echo -e "${YEL}--${NC}Moving File"
mv /usr/local/bin/nmap_scripts.list /usr/local/bin/nmap_scripts_old.list
echo -e "${YEL}--${NC}Fetching New List"
wget --quiet index.html https://nmap.org/nsedoc/ # Get the page containing the list of NSE Scripts
sed '/<h2>Scripts<\/h2>/,/<h2>Libraries<\/h2>/ !d' index.html > /usr/local/bin/working.txt # Remove unessesary headers/footers
rm index.html # Remove the index file we staged

echo -e "${YEL}--${NC}Formatting List"
# Yes, I know I could pipe these together, but breaking them apart allows for easier comprehension
sed -i 's|<li>|-|g' /usr/local/bin/working.txt # Replace <li> with a - (dash)
sed -i 's|</li>||g' /usr/local/bin/working.txt # The rest we are just removing
sed -i 's|<ul>||g' /usr/local/bin/working.txt
sed -i 's|</ul>||g' /usr/local/bin/working.txt
sed -i 's|<p>||g' /usr/local/bin/working.txt
sed -i 's|</p>||g' /usr/local/bin/working.txt
sed -i '1,1d' /usr/local/bin/working.txt # Remove first and last lines to cleanup a bit
sed -i '$d' /usr/local/bin/working.txt

html2text -width 9999 /usr/local/bin/working.txt > /usr/local/bin/nmap_scripts.list # Convirt file to text, long width to avoid line wrapping
rm /usr/local/bin/working.txt # Remove staging file
sed -i -e 's/  \+/ /g' /usr/local/bin/nmap_scripts.list # Replace all multiple spaces with single space
sed -i -r 's/\s+/ == /' /usr/local/bin/nmap_scripts.list # Replace first space in each line with our delimiter ( == )

echo -e "${YEL}--${NC}Update Summery" # Diffs the file, sed just adds some colored characters for readability. No output if no differences
diff -u /usr/local/bin/nmap_scripts_old.list /usr/local/bin/nmap_scripts.list | sed -n '1,2d;/^[-+]/p' | sed -e 's/^-/\x1b[31m-\x1b[0m/' | sed -e 's/^+/\x1b[32m+\x1b[0m/'
rm /usr/local/bin/nmap_scripts_old.list
echo -e "${YEL}--${NC}Fin"

Used in conjunction with nsesearch: https://gist.github.com/BeanBagKing/c2a4f9a498a086c1b7f9

Next step, bundle these two into one.