Skip to content

Instantly share code, notes, and snippets.

@dhoss
Created July 17, 2015 21:05
Show Gist options
  • Save dhoss/7e6a1d23613a1af79721 to your computer and use it in GitHub Desktop.
Save dhoss/7e6a1d23613a1af79721 to your computer and use it in GitHub Desktop.
trying to transform JSON
[
{
"project: "cb",
"repositories": [
{
"repository":"7zip",
"commits": [
{
"id": "92dda788900b40e95ffd3a73c0fe39b64182787b",
"displayId": "92dda788900",
"author": {
"name": "****,
"emailAddress": "****"
},
"authorTimestamp": 1393230044000,
"message": "small change",
"parents": [
{
"id": "b2e291bfac77f91e9285446e386669fd4a200463",
"displayId": "b2e291bfac7"
}
]
},
{
"id": "b2e291bfac77f91e9285446e386669fd4a200463",
"displayId": "b2e291bfac7",
"author": {
"name": "admin",
"emailAddress": "***",
"id": 1701,
"displayName": "admin",
"active": true,
"slug": "admin",
"type": "NORMAL",
"link": {
"url": "/users/admin",
"rel": "self"
},
"links": {
"self": [
{
"href": "http://stash.yourmom.com/users/admin"
}
]
}
},
"authorTimestamp": 1391729606000,
"message": "added new",
"parents": []
}
]
},
{
"acomiis": [
{
"id": "8e7fb042caff0d32af02d8748fe67d988a217753",
"displayId": "8e7fb042caf",
"author": {
"name": "****",
"emailAddress": "***"
},
"authorTimestamp": 1416951204000,
"message": "updated gem to most current version",
"parents": [
{
"id": "3380d56add39e39dd28d6318a0babba5e006880e",
"displayId": "3380d56add3"
}
]
},
{
"id": "3380d56add39e39dd28d6318a0babba5e006880e",
"displayId": "3380d56add3",
"author": {
"name": "admin",
"emailAddress": "***",
"id": 1701,
"displayName": "admin",
"active": true,
"slug": "admin",
"type": "NORMAL",
"link": {
"url": "/users/admin",
"rel": "self"
},
"links": {
"self": [
{
"href": "http://stash.yourmom.com/users/admin"
}
]
}
},
"authorTimestamp": 1391729606000,
"message": "added new",
"parents": []
}
]
},
....
[
{
"cb": [
{
"7zip": [
{
"id": "92dda788900b40e95ffd3a73c0fe39b64182787b",
"displayId": "92dda788900",
"author": {
"name": "****,
"emailAddress": "****"
},
"authorTimestamp": 1393230044000,
"message": "small change",
"parents": [
{
"id": "b2e291bfac77f91e9285446e386669fd4a200463",
"displayId": "b2e291bfac7"
}
]
},
{
"id": "b2e291bfac77f91e9285446e386669fd4a200463",
"displayId": "b2e291bfac7",
"author": {
"name": "admin",
"emailAddress": "***",
"id": 1701,
"displayName": "admin",
"active": true,
"slug": "admin",
"type": "NORMAL",
"link": {
"url": "/users/admin",
"rel": "self"
},
"links": {
"self": [
{
"href": "http://stash.yourmom.com/users/admin"
}
]
}
},
"authorTimestamp": 1391729606000,
"message": "added new",
"parents": []
}
]
},
{
"acomiis": [
{
"id": "8e7fb042caff0d32af02d8748fe67d988a217753",
"displayId": "8e7fb042caf",
"author": {
"name": "****",
"emailAddress": "***"
},
"authorTimestamp": 1416951204000,
"message": "updated gem to most current version",
"parents": [
{
"id": "3380d56add39e39dd28d6318a0babba5e006880e",
"displayId": "3380d56add3"
}
]
},
{
"id": "3380d56add39e39dd28d6318a0babba5e006880e",
"displayId": "3380d56add3",
"author": {
"name": "admin",
"emailAddress": "***",
"id": 1701,
"displayName": "admin",
"active": true,
"slug": "admin",
"type": "NORMAL",
"link": {
"url": "/users/admin",
"rel": "self"
},
"links": {
"self": [
{
"href": "http://stash.yourmom.com/users/admin"
}
]
}
},
"authorTimestamp": 1391729606000,
"message": "added new",
"parents": []
}
]
},
....
// we want to be asynchronous here because we are doing several API calls
def crawlProjectRepos(project: String) = {
val repos = reposFor(project)
val commits = for (repo <- repos)
yield for (c <- callApi(buildCommitUrl(project, repo)))
// not going to worry about creating a Commit object right now
// this is faster and when it comes time to insert info
// into the db, we can just pull out the values needed
yield Map(repo -> (c \ "values").as[List[JsValue]])
// create a Future.sequence to make sure the requests are run in parallel
// and then run them, returning a Future
Future.sequence(commits)
}
def crawlMultiProject(multiProject: String) = {
val projectCommits = for ( project <- splitMultiProjectParams(multiProject) )
yield for (repo <- crawlProjectRepos(project))
yield Map(project -> repo)
Future.sequence(projectCommits)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment