CrawlerService

CrawlerService.scala

package it.eng.scala.crawl

import org.jsoup.Jsoup
import scala.collection.JavaConversions._

object CrawlerService {

  val AbsolutePrefix = "http://en.wikipedia.org/wiki/"
  val RelativePrefix = "/wiki/"
  val AbsolutePath = "http://en.wikipedia.org"
  
  def scanLinks(address: String): List[(String, String)] = 
    Jsoup.
      connect(address)
      .get
      .select("a[href]")
      .iterator.toList
      .map(e => (address, e.attr("href")))
      .filter(p => p._2.startsWith(AbsolutePrefix) || p._2.startsWith(RelativePrefix))
      .map(p => 
        if(p._2.startsWith(RelativePrefix)) (p._1, AbsolutePath + p._2)
        else p
      )
  
}