Skip to content

Instantly share code, notes, and snippets.

@dhoss
Created July 15, 2015 20:43
Show Gist options
  • Save dhoss/b9fa2c105adf540ecff0 to your computer and use it in GitHub Desktop.
Save dhoss/b9fa2c105adf540ecff0 to your computer and use it in GitHub Desktop.
scala future issues
devin@debian:~/projects/blaster$ activator test
[info] Loading project definition from /home/devin/projects/blaster/project
[info] Set current project to blaster (in build file:/home/devin/projects/blaster/)
[info] Compiling 1 Scala source to /home/devin/projects/blaster/target/scala-2.11/classes...
[error] /home/devin/projects/blaster/app/clients/Stash.scala:77: inferred type arguments [List[scala.collection.immutable.Map[String,List[play.api.libs.json.JsValue]]],Array] do not conform to method sequence's type parameter bounds [A,M[X] <: TraversableOnce[X]]
[error] for ( p <- Future.sequence(projectCommits) )
[error] ^
[error] /home/devin/projects/blaster/app/clients/Stash.scala:77: type mismatch;
[error] found : Array[scala.concurrent.Future[List[scala.collection.immutable.Map[String,List[play.api.libs.json.JsValue]]]]]
[error] required: M[scala.concurrent.Future[A]]
[error] for ( p <- Future.sequence(projectCommits) )
[error] ^
[error] two errors found
[error] (compile:compileIncremental) Compilation failed
[error] Total time: 7 s, completed Jul 15, 2015 2:39:49 PM
"crawlProjectRepos" should {
"successfully hit each URL in the list" in new WithApplication {
val crawler = new Stash(25)
for (commitMap <- Await.result(crawler.crawlProjectRepos(project), Duration.Inf)) {
commitMap must not be empty
commitMap must have length(be_<=(25))
for ((repo, commitList) <- commitMap) {
commitList must not be empty
commitList must have length(be_<=(25))
}
}
}
}
"crawlMultiProject" should {
"hit every URL" in new WithApplication {
val projects = new Stash(25)
val projectString = "cb+blas+dev"
for ( p <- projects.crawlMultiProject(projectString).pp ) {
val project = Await.result(p, Duration.Inf)
project.pp must not be empty
}
}
}
// we want to be asynchronous here because we are doing several API calls
def crawlProjectRepos(project: String) = {
val repos = reposFor(project)
// push all of our Future requests into commit
// do a little bit of parsing for the JSON so we get something like this:
// [
// {
// "project-name": [
// {
// "id": "e58f47e0067a5b84532aa195cc3b7a7af427aadd",
// "displayId": ...
val commits = for (repo <- repos)
yield for (c <- callApi(buildCommitUrl(project, repo)))
// not going to worry about creating a Commit object right now
// this is faster and when it comes time to insert info
// into the db, we can just pull out the values needed
yield Map(repo -> (c \ "values").as[List[JsValue]])
// create a Future.sequence to make sure the requests are run in parallel
// and then run them, returning a Future
for ( c <- Future.sequence(commits))
yield c
}
def crawlMultiProject(multiProject: String) = {
val projectCommits = for ( u <- multiProjectUrl(multiProject) )
yield crawlProjectRepos(u)
for ( p <- Future.sequence(projectCommits) )
yield p
}
def multiProjectUrl(multiProject: String): Array[String] = {
// validate URL here
multiProject.split('+').map(url => buildProjectUrl(url))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment