./submit_urls_to_virustotal.sh - Submits URLs to Virustotal for scanning

submit_urls_to_virustotal.sh

#!/bin/bash

# in seconds time gap between individual requests to VT due to API limits
DEFAULT_TIME_GAP=30
USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36"
DEFAULT_OUT_FILE="out-vt-scan.csv"
DELIMITER="|"

if [ $# -lt 2 ]; then
    echo "[-] $0 <virustotal-api-key> <url-to-scan/file-to-scan> [out-file] [timegap]"
    echo 
    echo "Description:"
    echo "This script submits a single URL or multiple URLs in a file to virustotal for "
    echo "scanning. There is generally a timegap (in secs) due to limits imposed by the "
    echo "VT API in submission of URL and extraction of report for each URL"
    echo
    echo "Output:"
    echo "The output consists of Hits/Total number of Engines in which scan was run, scanID, "
    echo "Last Date when the scan was run (in UTC) which should be almost the current time, "
    echo "and permalink to VT (VT URL which provides detailed scan results)"
    echo
    echo "Arguments:"
    echo "virustotal-api-key: VT API key can be obtained from the VT account"
    echo "url-to-scan/file-to-scan: Specify either a single URL to scan OR a file with URLs to scan"
    echo "out-file: Output file for the results of the VT scan"
    echo "timegap: time gap (in seconds)" 
    exit
fi
vt_api_key="$1"
url_or_file_to_scan="$2"
out_file="$3"
time_gap="$4"

[ -z "$time_gap" ] && time_gap=$DEFAULT_TIME_GAP
echo "[+] time_gap: $time_gap"

[ -z "$out_file" ] && out_file="$DEFAULT_OUT_FILE"
echo "[+] out_file: $out_file"

function scan_single_url_in_vt {
    # Scan URL using Virustotal and writing the output to output file
    local vt_api_key="$1"
    local url_to_scan="$2"
    local out_file="$3"
    local time_gap="$4"

    echo "[*] Making CURL request to VT to scan URL: $url_to_scan"
    vt_url_scan_output=`curl -s --request POST \
            -A "$USER_AGENT" \
            --url 'https://www.virustotal.com/vtapi/v2/url/scan' \
            -d apikey="$vt_api_key" \
            -d "url=$url_to_scan"`

    if [ ! -z "`echo $vt_url_scan_output | egrep -io 'Scan request successfully queued'`" ]; then
        echo "[+] Scan for URL: $url_to_scan successfully submitted to VT"
    else
        echo "[-] Scan for URL: $url_to_scan not submitted to VT"
        echo "[-] Output from URL submission: $vt_url_scan_output"
        exit
    fi

    echo "[*] Sleeping for $time_gap secs before requesting report"
    sleep $time_gap

    echo "[*] Making CURL request to VT to get the report on URL: $url_to_scan"
    vt_url_report_output=`curl -s --request POST \
      -A "$USER_AGENT" \
      --url 'https://www.virustotal.com/vtapi/v2/url/report' \
      -d "apikey=$vt_api_key" \
      -d "resource=$url_to_scan"`

    echo "[*] Parsing relevant fields from VT report output on URL: $url_to_scan"
    scan_id=`echo "$vt_url_report_output" | egrep -io '"scan_id": "[^\",]*"' | cut -d" " -f2 | tr -d '"'` 
    scan_date=`echo "$vt_url_report_output" | egrep -io '"scan_date": "[^\",]*"' | cut -d":" -f2,3,4 | tr -d '"' `
    positives=`echo "$vt_url_report_output" | egrep -io '"positives": [^,]*' | cut -d" " -f2 | tr -d '"' `
    total=`echo "$vt_url_report_output" | egrep -io '"total": [^,]*' | cut -d" " -f2 | tr -d '"' `
    permalink=`echo "$vt_url_report_output" | egrep -io '"permalink": "[^\",]*"' | cut -d" " -f2 | tr -d '"' `

    echo "[*] Writing fields for URL: $url_to_scan to outfile: $out_file"
    echo "$positives/$total$DELIMITER$url_to_scan$DELIMITER$scan_id$DELIMITER$scan_date$DELIMITER$permalink" >> $out_file

    echo "[*] Sleeping for $time_gap secs before requesting next URL scan"
    sleep $time_gap
}

function create_out_file {
    # Add heading to output file and create the output file
    local out_file="$1"
    echo "positives/total"$DELIMITER"url_to_scan"$DELIMITER"scan_id"$DELIMITER"scan_date"$DELIMITER"permalink" > "$out_file"
}

create_out_file "$out_file"

echo "[*] Determining if we have URL or file to scan"
if [ ! -f "$url_or_file_to_scan" ]; then
    url_to_scan="$url_or_file_to_scan"

    echo "[*] Scanning URL: $url_to_scan"
    scan_single_url_in_vt "$vt_api_key" "$url_to_scan" "$out_file" "$time_gap"

else
    file_to_scan="$url_or_file_to_scan"

    echo "[*] Reading URLs from file: $file_to_scan"
    urls=`cat $file_to_scan`
    
    IFS=$'\n'
    for url_to_scan in `echo "$urls"`; do
        echo "[*] Scanning URL: $url_to_scan" 
        scan_single_url_in_vt "$vt_api_key" "$url_to_scan" "$out_file" "$time_gap"
    done
fi