Skip to content

Instantly share code, notes, and snippets.

@ruiwen
Created October 3, 2012 12:43
Show Gist options
  • Save ruiwen/3826726 to your computer and use it in GitHub Desktop.
Save ruiwen/3826726 to your computer and use it in GitHub Desktop.
Map-reduce in with MongoDB with PyMongo for Parse timing calls
'''
Sample object:
{
"url": "/1/functions/Klass3",
"_id": {
"$oid": "506c082701d5410ec7000002"
},
"log": [
{
"duration": 30.353700876235962,
"params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}",
"sent": {
"$date": 1349286055012
}
},
{
"duration": 0.5051431655883789,
"params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}",
"sent": {
"$date": 1349286212207
}
},
{
"duration": 0.5687680244445801,
"params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}",
"sent": {
"$date": 1349286393610
}
}
]
}
Sample results:
{
"counts": {
"input": 4,
"reduce": 4,
"emit": 21,
"output": 4
},
"timeMillis": 19,
"ok": 1.0,
"results": [
{
"_id": "/1/classes/Klass1",
"value": {
"count": 8.0,
"total": 5.25708794593811,
"avg": 0.6571359932422638
}
},
{
"_id": "/1/classes/Klass2",
"value": {
"count": 6.0,
"total": 1.4316394329071045,
"avg": 0.23860657215118408
}
},
{
"_id": "/1/functions/Klass3",
"value": {
"count": 3.0,
"total": 31.42761206626892,
"avg": 10.475870688756308
}
},
{
"_id": "/1/login/",
"value": {
"count": 4.0,
"total": 1.687263011932373,
"avg": 0.42181575298309326
}
}
]
}
'''
from pymongo import Connection
from bson.code import Code
from bson.son import SON
conn = Connection();
db = conn.******;
db.authenticate("**", "***");
coll = db.******
mapf = Code('''
function() {
var self = this;
self.log.forEach(function(l){
emit(self.url, {
total: l.duration,
count: 0,
avg: 0
});
});
} ''')
reducef = Code('''
function(key, values) {
var r = {
count: 0,
total: 0,
avg: 0
};
values.forEach(function(item) {
r.count += 1;
r.total += item.total;
});
return r;
} ''')
finalisef = Code('''
function(key, value) {
if(value.count > 0) {
value.avg = value.total / value.count;
}
return value;
}''')
coll.map_reduce(map=mapf, reduce=reducef, out=SON([('inline',1)]), finalize=finalisef)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment