A graph database (e.g. Neo4j) is an interesting way to look at the links between pages on a site.

createNeo4jLinkGraph.py

# A JSON file containing the url and html is loaded into a Neo4j graph database.

import json
from bs4 import BeautifulSoup
from py2neo import neo4j

graph_db = neo4j.GraphDatabaseService()
urls = graph_db.get_or_create_index(neo4j.Node, "Urls")
connectedTo = graph_db.get_or_create_index(neo4j.Relationship, "ConnectedTo")

for line in open('dpsk12_org.json', 'r').readlines():
	try:
		record = json.loads(line.strip())
		soup = BeautifulSoup(record['html'])
		url = record['url']
		for link in soup.findAll('a'):
			if link['href'].startswith('http://'):
				try:
					fromUrl = urls.get_or_create("url", url, {'url' : url})
					toUrl = urls.get_or_create("url", link['href'], {'url' : link['href']})
					path = fromUrl.get_or_create_path("connectedTo", toUrl)
				except:
					pass
	except:
		pass