Created
July 15, 2015 20:43
-
-
Save dhoss/b9fa2c105adf540ecff0 to your computer and use it in GitHub Desktop.
scala future issues
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
devin@debian:~/projects/blaster$ activator test | |
[info] Loading project definition from /home/devin/projects/blaster/project | |
[info] Set current project to blaster (in build file:/home/devin/projects/blaster/) | |
[info] Compiling 1 Scala source to /home/devin/projects/blaster/target/scala-2.11/classes... | |
[error] /home/devin/projects/blaster/app/clients/Stash.scala:77: inferred type arguments [List[scala.collection.immutable.Map[String,List[play.api.libs.json.JsValue]]],Array] do not conform to method sequence's type parameter bounds [A,M[X] <: TraversableOnce[X]] | |
[error] for ( p <- Future.sequence(projectCommits) ) | |
[error] ^ | |
[error] /home/devin/projects/blaster/app/clients/Stash.scala:77: type mismatch; | |
[error] found : Array[scala.concurrent.Future[List[scala.collection.immutable.Map[String,List[play.api.libs.json.JsValue]]]]] | |
[error] required: M[scala.concurrent.Future[A]] | |
[error] for ( p <- Future.sequence(projectCommits) ) | |
[error] ^ | |
[error] two errors found | |
[error] (compile:compileIncremental) Compilation failed | |
[error] Total time: 7 s, completed Jul 15, 2015 2:39:49 PM |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"crawlProjectRepos" should { | |
"successfully hit each URL in the list" in new WithApplication { | |
val crawler = new Stash(25) | |
for (commitMap <- Await.result(crawler.crawlProjectRepos(project), Duration.Inf)) { | |
commitMap must not be empty | |
commitMap must have length(be_<=(25)) | |
for ((repo, commitList) <- commitMap) { | |
commitList must not be empty | |
commitList must have length(be_<=(25)) | |
} | |
} | |
} | |
} | |
"crawlMultiProject" should { | |
"hit every URL" in new WithApplication { | |
val projects = new Stash(25) | |
val projectString = "cb+blas+dev" | |
for ( p <- projects.crawlMultiProject(projectString).pp ) { | |
val project = Await.result(p, Duration.Inf) | |
project.pp must not be empty | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// we want to be asynchronous here because we are doing several API calls | |
def crawlProjectRepos(project: String) = { | |
val repos = reposFor(project) | |
// push all of our Future requests into commit | |
// do a little bit of parsing for the JSON so we get something like this: | |
// [ | |
// { | |
// "project-name": [ | |
// { | |
// "id": "e58f47e0067a5b84532aa195cc3b7a7af427aadd", | |
// "displayId": ... | |
val commits = for (repo <- repos) | |
yield for (c <- callApi(buildCommitUrl(project, repo))) | |
// not going to worry about creating a Commit object right now | |
// this is faster and when it comes time to insert info | |
// into the db, we can just pull out the values needed | |
yield Map(repo -> (c \ "values").as[List[JsValue]]) | |
// create a Future.sequence to make sure the requests are run in parallel | |
// and then run them, returning a Future | |
for ( c <- Future.sequence(commits)) | |
yield c | |
} | |
def crawlMultiProject(multiProject: String) = { | |
val projectCommits = for ( u <- multiProjectUrl(multiProject) ) | |
yield crawlProjectRepos(u) | |
for ( p <- Future.sequence(projectCommits) ) | |
yield p | |
} | |
def multiProjectUrl(multiProject: String): Array[String] = { | |
// validate URL here | |
multiProject.split('+').map(url => buildProjectUrl(url)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment