external_links.py

#*-- coding: utf-8 -*-
"""Given an url extract all external links and return a list of tuples (title, url)"""
from mechanize import Browser
from urlparse import urlparse
import sys

if __name__ == "__main__":
    url_list = sys.argv[1:]

    br = Browser()

    for url in url_list:
        base_url = urlparse(url)[1] # get the netloc part of the url
        br.open(url)
        
        external_links = set()

        for link in br.links():
            if urlparse(link.absolute_url)[1] != base_url:
                external_links.add((link.text, link.absolute_url))

        for l in external_links:
            print l