URL Extraction

gistfile1.txt

http://n3t.awardspace.us/content/tcpdump-url-extraction

#!/bin/bash
#
 
# reset variables
myhost="";
myurl="";
 
tcpdump -s 0 -w - -l $@ | strings |
while read line;
  do 
 
# filter GET requests
	myurl=`echo $line | grep GET | sed -r "s/GET (.*) HTTP.*/\1/"`;
	if [ "$myurl" == "" ]; then myurl=$myoldurl; fi
 
# filter Host headers
	myhost=`echo $line | grep Host | sed -r "s/Host: (.*)/\1/"`;
	if [ "$myhost" == "" ]; then myhost=$myoldhost; fi
 
# once we have a data pair, put them together and echo
	if [ "$myhost" != "" ] 
		then
		url="http://$myhost$myurl";
		echo $url;
		myhost="";
		myurl="";
	fi
 
	myoldurl=$myurl;
	myoldhost=$myhost;
done