Shell script to keep the local InternetDB SQLite file up to date

update-internetdb.sh

#!/bin/sh
# update-internetdb.sh
# Run this script daily to make sure your local InternetDB SQLite file is always up to date.

API_KEY=""
DB_FOLDER="/usr/local/shodan"

# The script requires the "curl" command to be available
if ! type "curl" > /dev/null; then
    echo "Error: 'curl' command not found"
    exit 1
fi

# Make sure the folder exists
mkdir -p $DB_FOLDER
if [ $? -ne 0 ]; then
    exit 1
fi

# Grab the SHA1 value for the current InternetDB file
OLD_SHA1=""
if [ -e $DB_FOLDER/internetdb.sqlite.sha1 ]; then
    OLD_SHA1=`cat $DB_FOLDER/internetdb.sqlite.sha1 2> /dev/null`
fi

# Get the metadata about the latest InternetDB files
echo "Connecting to the Shodan API..."
INFO=`curl -X GET "https://api.shodan.io/shodan/data/internetdb?key=$API_KEY" 2> /dev/null`
echo $INFO | grep internetdb 1> /dev/null
if [ $? -ne 0 ]; then
    echo "Error: Permission denied"
    exit 1
fi

# Parse out the SQLite file info
INFO=`echo $INFO | sed -e 's/}, {/\n/' | grep sqlite`
URL=`echo $INFO | sed -e 's/, /\n/g' | grep url | sed -e 's/.*http/http/' -e 's/"//g'`
NEW_SHA1=`echo $INFO | sed -e 's/, /\n/g' | grep sha1 | sed -e 's/.* "//' -e 's/"//g'`

# Compare the SHA1 checksums to see if there's a new file available
if [ "$OLD_SHA1" = "$NEW_SHA1" ]; then
    echo "InternetDB is up to date"
    exit 0
fi

# A new version is available so lets download it
TMP_FILE="$DB_FOLDER/.internetdb.sqlite"
TMP_FILE_COMPRESSED="$TMP_FILE.bz2"

echo "Downloading latest file..."
curl -o $TMP_FILE_COMPRESSED "$URL"

# Calculate the SHA1 checksum to make sure it downloaded successfully
echo "Verifying file integrity..."
TMP_SHA1=`sha1sum -b $TMP_FILE_COMPRESSED | sed -e 's/ .*//'`
if [ "$TMP_SHA1" != "$NEW_SHA1" ]; then
    echo "Error: File doesn't match the expected SHA1 checksum"
    exit 1
fi

# Update the file that stores the SHA1 for the database
echo $NEW_SHA1 > $DB_FOLDER/internetdb.sqlite.sha1

# Uncompress the database so we can start using it
echo "Uncompressing file..."
bunzip2 $TMP_FILE_COMPRESSED

# Overwrite the old file with the new one
mv $TMP_FILE $DB_FOLDER/internetdb.sqlite

echo "Successfully updated InternetDB. Reload your application to take advantage of it."