common-comment-words.scala

import scala.meta._
import java.nio.file._

object CommentWordCount extends App {

  val scalaDir = "/Users/chris/code/scala"
  val stdlibSrcDir = s"$scalaDir/src/library/scala"

  val scalaFileContents: Array[String] = Files.walk(Paths.get(stdlibSrcDir)).toArray.collect {
    case p: Path if p.toString.endsWith(".scala") => new String(Files.readAllBytes(p), "UTF-8")
  }

  val commentWords = scalaFileContents.flatMap{ string =>
    val tokens = string.parse[Source].get.tokens
    val commentTokens = tokens.filter(_.name == "comment")
    val commentsAsStrings = commentTokens.map(_.toString)
    commentsAsStrings.flatten(_.split(" ")).collect { case x if x.matches("""[a-zA-Z]{2,}""") => x.toLowerCase }
  }.toSeq

  println(commentWords.groupBy(l => l).map(t => (t._1, t._2.length)).toSeq.sortBy(_._2).reverse)
}