scalding wordcount - top 25 words

WordCountJob.scala

package com.twitter.scalding.examples

import com.twitter.scalding._

class WordCountJob(args : Args) extends Job(args) {
  TextLine( args("input") )
    .flatMap('line -> 'word) { line : String => tokenize(line) }
    .groupBy('word){ _.size }
    .groupAll { _.sortedReverseTake[(Long, String)](( 'size, 'word) -> 'top, 25) }
    .flattenTo[(Long, String)]('top -> ('size, 'word))
    .write( Tsv( args("output") ) )

  // Split a piece of text into individual words.
  def tokenize(text : String) : Array[String] = {
    // Lowercase each word and remove punctuation.
    text.toLowerCase.replaceAll("[^a-zA-Z\\s]", "").split("\\s+")
  }
}