Skip to content

Instantly share code, notes, and snippets.

@travisbrown
Created September 11, 2012 19:34
Show Gist options
  • Save travisbrown/3701424 to your computer and use it in GitHub Desktop.
Save travisbrown/3701424 to your computer and use it in GitHub Desktop.
Using the HTRC Data API with Scala and Dispatch 0.9
import dispatch._
import java.io.ByteArrayInputStream
import java.util.zip.ZipInputStream
import scala.util.parsing.json.JSON.{ parseFull => parseJson }
val headers = Map("content-type" -> "application/x-www-form-urlencoded")
val tokenUrl = url("https://silvermaple.pti.indiana.edu:25443/oauth2/token")
val volumesUrl = url("https://silvermaple.pti.indiana.edu:25443/data-api/volumes/")
def tokenParams(id: String, secret: String) = Map(
"grant_type" -> "client_credentials",
"client_id" -> id,
"client_secret" -> secret
)
// Boring stuff: a simple zip archive iterator.
type Entry = (String, IndexedSeq[String])
def iterateZip(zis: ZipInputStream) = new Iterator[Entry] {
// Drop the directory entry.
zis.getNextEntry
var entry = zis.getNextEntry
def hasNext = entry != null
def next = {
val name = entry.getName
val source = scala.io.Source.fromInputStream(zis)
val lines = source.getLines.toIndexedSeq
entry = zis.getNextEntry
name -> lines
}
}
// This really ought to be provided by Dispatch.
implicit def validationGuarantor[A] = new Guarantor[
Either[Throwable, Promise[Either[Throwable, A]]],
Either[Throwable, A],
Promise[Either[Throwable, A]]
] {
def promise(collateral: Either[Throwable, Promise[Either[Throwable, A]]]) =
collateral.fold(e => Promise(Left(e)), identity)
}
// Most of the work happens here.
def getVolumes(id: String, secret: String, volumes: String):
//Promise[Either[Throwable, Either[Throwable, Iterator[Entry]]]] = {
Promise[Either[Throwable, Iterator[Entry]]] = {
val tokenReq = tokenUrl <:< headers <<? tokenParams(id, secret) << "null"
val token = Http(tokenReq OK as.String).map(
// This is how worthless the standard library JSON parser is.
parseJson(_).flatMap(
_.asInstanceOf[Map[String, Any]].get("access_token")
).asInstanceOf[Option[String]].toRight(
new RuntimeException("Invalid JSON response.")
)
)
def volumesReq(token: String, volumes: String) = volumesUrl <:< headers + (
"Authorization" -> ("Bearer " + token)
) << Map("volumeIDs" -> volumes)
token.flatMap(_.right.map(token =>
Http(volumesReq(token, volumes) OK as.Bytes).map(
bytes => iterateZip(new ZipInputStream(new ByteArrayInputStream(bytes)))
).either
))
}
// Usage example:
val result = getVolumes("whatever", "secret", "nyp.33433076050149")
result().right.get.foreach {
case (name, lines) => println(name + " " + lines.headOption.getOrElse(""))
}
// Prints:
// nyp.33433076050149/00000001.txt Thackeray's worksWilliam Makepeace Thackeray
// nyp.33433076050149/00000002.txt THE NEW YORK PUBLIC LIBRARY
// nyp.33433076050149/00000003.txt -•', -^ 7.', .. ...
// ...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment