notebook.scala

val r = 
RecordLoader.loadArc(arc, 
sc) 
.keepValidPages() 
.map(r => { 
val t = ExtractRawText(r.getBodyContent) 
val len = 100 
(r.getCrawldate, createClickableLink(r.getUrl, 
r.getCrawldate), if ( t.length > len ) t.substring(0, len) else t)}) 
.collect() 

val r = 
RecordLoader.loadArc(arc,
sc) 
.keepMimeTypes(Set("text/html")) 
.discardDate(null) 
.map(r => { 
val t = ExtractRawText(r.getBodyContent)
NER3Classifier("/Users/ianmilligan1/dropbox/ner/stanford-ner-2015-04-20/classifiers/english.all.3class.distsim.crf.ser.gz")
val entities = NER3Classifier.classify(t)
val len = 100
(r.getCrawldate, r.getMimeType, entities, r.getUrl, if ( t.length > len ) t.substring(0, 
len) else t)})
.collect()

def createClickableLink(url: String, date: String): String = { 
"<a href='http://web.archive.org/web/" + date + "/" + url + "'>" + 
url + "</a>" 
}