Skip to content

Instantly share code, notes, and snippets.

@bryce-anderson
Created December 15, 2014 20:45
Show Gist options
  • Save bryce-anderson/6998af7b5a39cb057931 to your computer and use it in GitHub Desktop.
Save bryce-anderson/6998af7b5a39cb057931 to your computer and use it in GitHub Desktop.
scalaz-stream nio performance tuning
package scalaz.stream
import java.io._
import scodec.bits.ByteVector
import scalaz._, Scalaz._, stream._
import scalaz.stream.nio._
object Main {
def main(args: Array[String]): Unit = {
createFile
println("Starting.")
time(nioProcess, "nio")
time(ioProcess, "io")
time(sourceTest, "scala.io.Source")
println("Finished.")
val testString = "or\nld"
println(text.splitLines(testString).map(_.replace("\n", "\\n")))
println(testString.split("\r\n|\n").toVector.map(_.replace("\n", "\\n").replace("\r", "\\r")))
}
def time[A](a: => A, name: String): A = {
val t = System.currentTimeMillis()
val aa = a
val diff = System.currentTimeMillis() - t
println(s"$name Duration: $diff ms")
aa
}
def nioProcess = {
val p =
file.linesR(bigPath) |> stream.text.utf8Encode to file.chunkW("/tmp/nio.txt")
p.run.run
}
def ioProcess = {
val p =
stream.io.linesR(bigPath) |> stream.text.utf8Encode to stream.io.fileChunkW("/tmp/io.txt")
p.run.run
}
def sourceTest = {
val output = new BufferedOutputStream(new FileOutputStream("/tmp/scalaio.txt"), 32*1024)
val p =
scala.io.Source.fromFile(bigPath).getLines().map(_.getBytes("UTF-8")).foreach(output.write)
output.close()
}
// def transform(line: String): String = {
// val fields = line.split("\\|")
// line+"|"//+fields(3).toInt
// }
def createFile = {
val printer = new PrintWriter(bigPath)
val line = "1|2|3|4|5\n"
0 until 500000 foreach(_ => printer.write(line))
printer.close()
}
val bigPath = "/tmp/big.txt"
}
///////////////////////////////////////////////////////////////
/** Output
Starting.
nio Duration: 10405 ms
io Duration: 14430 ms
scala.io.Source Duration: 88 ms
Finished.
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment