scrap_pdf

#!/usr/bin/env python
import urllib2
import re
import sys
from itertools import imap, ifilter

if __name__ == "__main__":
    in_url = sys.argv[1]
    current_dir = "/".join(in_url.split('/')[:-1])
    all_urls = re.findall(r'href=[\'"]?([^\'" >]+)', urllib2.urlopen(in_url).read())
    print "\n".join(imap(lambda x: x if x.startswith('http') else current_dir + "/" + x, ifilter(lambda x: x.endswith('pdf'),all_urls)))

# Download all .pdf files referenced in a web page
python scrap_pdf.py http://www.vldb.org/pvldb/vol8.html | parallel wget {}