Created
July 1, 2015 18:53
-
-
Save vibgy/22386a065215c64115a4 to your computer and use it in GitHub Desktop.
Intel Lab Data analysis using Mongo. Get the data from http://db.csail.mit.edu/labdata/labdata.html and you can use these scripts to import them in mongo and learn to write aggregation queries against mongo.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var mongoose = require ("mongoose"); // The reason for this demo. | |
mongoose.set('debug', true); | |
// Here we find an appropriate database to connect to, defaulting to | |
// localhost if we don't find one. | |
var uristring = | |
process.env.MONGOLAB_URI || | |
process.env.MONGOHQ_URL || | |
'mongodb://localhost/iot'; | |
var connection = mongoose.connect(uristring, function (err, res) { | |
if (err) { | |
console.log ('ERROR connecting to: ' + uristring + '. ' + err); | |
} else { | |
console.log ('Succeeded connected to: ' + uristring); | |
testRangeQuery(); | |
testAvgTempratureByMoteId(); | |
testAvgTempratureByMoteIdAndMinute(); | |
testStdDevLightByMoteId(); | |
} | |
}); | |
// Schema | |
var iotSchema = new mongoose.Schema({ | |
ts: {type: Date}, | |
epoch: {type: Number, min: 0}, | |
moteid: {type: Number, min: 1, max: 54}, | |
temperature: {type: Number}, | |
humidity: {type: Number}, | |
light: {type: Number}, | |
voltage: {type: Number} | |
}); | |
//var iotModel = connection.model('data', iotSchema); | |
var iotModel = mongoose.model('data', iotSchema); | |
var count = 0; | |
var readline = require('linebyline'); | |
rd = readline('./test.txt') | |
rd.on('line', function(line, linecount) { | |
var args = line.split(" "); | |
if (!args || args.length != 7) { | |
return; | |
} | |
var d = args[0]; | |
var t = args[1]; | |
var e = args[2]; | |
var m = args[3]; | |
var c = args[4]; | |
var h = args[5]; | |
var l = args[6]; | |
var v = args[7]; | |
var z = d + 'T' + t; | |
var k = z.substring(0, 23) | |
k = k+'Z'; | |
console.log(k); | |
var testdata = new iotModel({ | |
ts: new Date(k), | |
epoch: e, | |
moteid: m, | |
temperature: c, | |
humidity: h, | |
light: l, | |
voltage: v | |
}); | |
testdata.save(function(err) { | |
if (err) console.log(err); | |
}) | |
}); | |
rd.on('end', function() { | |
console.log("done!!"); | |
process.exit(); | |
}); | |
function testRangeQuery() { | |
iotModel.find({ts: | |
{ | |
$gt: new Date("2004-02-27T00:00:00.000Z"), | |
$lt: new Date("2004-02-29T00:00:00.000Z") | |
} | |
}).exec(function(err, values) { | |
console.log("records found " + values.length) | |
}); | |
} | |
function testAvgTempratureByMoteId() { | |
iotModel.aggregate([ | |
{ $match : | |
{ ts: | |
{ | |
$gt: new Date("2004-02-27T00:00:00.000Z"), | |
$lt: new Date("2004-02-29T00:00:00.000Z") | |
} | |
} | |
}, | |
{ $group : | |
{ _id : "$moteid", | |
avgTemp : { $avg : "$temperature" }, | |
sumLight : { $sum : "$light" }, | |
countRecords : { $sum : 1 } | |
} | |
} | |
], function(err, results) { | |
console.log(results); | |
}); | |
} | |
function testAvgTempratureByMoteIdAndMinute() { | |
iotModel.aggregate([ | |
{ $match : | |
{ ts: | |
{ | |
$gt: new Date("2004-02-28T00:00:00.000Z"), | |
$lt: new Date("2004-02-29T00:00:00.000Z") | |
} | |
} | |
}, | |
{ $group : | |
{ _id : {mid: "$moteid", hour: {$hour : "$ts"}, minute: {$minute : "$ts"}}, | |
avgTemp : { $avg : "$temperature" }, | |
sumLight : { $sum : "$light" }, | |
countRecords : { $sum : 1 } | |
} | |
} | |
], function(err, results) { | |
console.log(results); | |
}); | |
} | |
function testStdDevLightByMoteId() { | |
// this map reduce is copied from a github gist: https://gist.github.com/Pyrolistical/8139958 | |
function map() { | |
// use some constant like 1 as the key if you want to do it across the collection | |
emit(this.moteid, { | |
sum: this.light, // the field you want stats for | |
min: this.light, | |
max: this.light, | |
count: 1, | |
diff: 0 | |
}); | |
} | |
function reduce(key, values) { | |
return values.reduce(function reduce(previous, current, index, array) { | |
var delta = previous.sum/previous.count - current.sum/current.count; | |
var weight = (previous.count * current.count)/(previous.count + current.count); | |
return { | |
sum: previous.sum + current.sum, | |
min: Math.min(previous.min, current.min), | |
max: Math.max(previous.max, current.max), | |
count: previous.count + current.count, | |
diff: previous.diff + current.diff + delta*delta*weight | |
}; | |
}) | |
} | |
function finalize(key, value) { | |
value.average = value.sum / value.count; | |
value.population_variance = value.diff / value.count; | |
value.population_standard_deviation = Math.sqrt(value.population_variance); | |
value.sample_variance = value.diff / (value.count - 1); | |
value.sample_standard_deviation = Math.sqrt(value.sample_variance); | |
delete value.diff; | |
return value; | |
} | |
iotModel.mapReduce({map: map, reduce:reduce, finalize: finalize}, function(err, results) { | |
console.log(results); | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment