Skip to content

Instantly share code, notes, and snippets.

@vibgy
Created July 1, 2015 18:53
Show Gist options
  • Save vibgy/22386a065215c64115a4 to your computer and use it in GitHub Desktop.
Save vibgy/22386a065215c64115a4 to your computer and use it in GitHub Desktop.
Intel Lab Data analysis using Mongo. Get the data from http://db.csail.mit.edu/labdata/labdata.html and you can use these scripts to import them in mongo and learn to write aggregation queries against mongo.
var mongoose = require ("mongoose"); // The reason for this demo.
mongoose.set('debug', true);
// Here we find an appropriate database to connect to, defaulting to
// localhost if we don't find one.
var uristring =
process.env.MONGOLAB_URI ||
process.env.MONGOHQ_URL ||
'mongodb://localhost/iot';
var connection = mongoose.connect(uristring, function (err, res) {
if (err) {
console.log ('ERROR connecting to: ' + uristring + '. ' + err);
} else {
console.log ('Succeeded connected to: ' + uristring);
testRangeQuery();
testAvgTempratureByMoteId();
testAvgTempratureByMoteIdAndMinute();
testStdDevLightByMoteId();
}
});
// Schema
var iotSchema = new mongoose.Schema({
ts: {type: Date},
epoch: {type: Number, min: 0},
moteid: {type: Number, min: 1, max: 54},
temperature: {type: Number},
humidity: {type: Number},
light: {type: Number},
voltage: {type: Number}
});
//var iotModel = connection.model('data', iotSchema);
var iotModel = mongoose.model('data', iotSchema);
var count = 0;
var readline = require('linebyline');
rd = readline('./test.txt')
rd.on('line', function(line, linecount) {
var args = line.split(" ");
if (!args || args.length != 7) {
return;
}
var d = args[0];
var t = args[1];
var e = args[2];
var m = args[3];
var c = args[4];
var h = args[5];
var l = args[6];
var v = args[7];
var z = d + 'T' + t;
var k = z.substring(0, 23)
k = k+'Z';
console.log(k);
var testdata = new iotModel({
ts: new Date(k),
epoch: e,
moteid: m,
temperature: c,
humidity: h,
light: l,
voltage: v
});
testdata.save(function(err) {
if (err) console.log(err);
})
});
rd.on('end', function() {
console.log("done!!");
process.exit();
});
function testRangeQuery() {
iotModel.find({ts:
{
$gt: new Date("2004-02-27T00:00:00.000Z"),
$lt: new Date("2004-02-29T00:00:00.000Z")
}
}).exec(function(err, values) {
console.log("records found " + values.length)
});
}
function testAvgTempratureByMoteId() {
iotModel.aggregate([
{ $match :
{ ts:
{
$gt: new Date("2004-02-27T00:00:00.000Z"),
$lt: new Date("2004-02-29T00:00:00.000Z")
}
}
},
{ $group :
{ _id : "$moteid",
avgTemp : { $avg : "$temperature" },
sumLight : { $sum : "$light" },
countRecords : { $sum : 1 }
}
}
], function(err, results) {
console.log(results);
});
}
function testAvgTempratureByMoteIdAndMinute() {
iotModel.aggregate([
{ $match :
{ ts:
{
$gt: new Date("2004-02-28T00:00:00.000Z"),
$lt: new Date("2004-02-29T00:00:00.000Z")
}
}
},
{ $group :
{ _id : {mid: "$moteid", hour: {$hour : "$ts"}, minute: {$minute : "$ts"}},
avgTemp : { $avg : "$temperature" },
sumLight : { $sum : "$light" },
countRecords : { $sum : 1 }
}
}
], function(err, results) {
console.log(results);
});
}
function testStdDevLightByMoteId() {
// this map reduce is copied from a github gist: https://gist.github.com/Pyrolistical/8139958
function map() {
// use some constant like 1 as the key if you want to do it across the collection
emit(this.moteid, {
sum: this.light, // the field you want stats for
min: this.light,
max: this.light,
count: 1,
diff: 0
});
}
function reduce(key, values) {
return values.reduce(function reduce(previous, current, index, array) {
var delta = previous.sum/previous.count - current.sum/current.count;
var weight = (previous.count * current.count)/(previous.count + current.count);
return {
sum: previous.sum + current.sum,
min: Math.min(previous.min, current.min),
max: Math.max(previous.max, current.max),
count: previous.count + current.count,
diff: previous.diff + current.diff + delta*delta*weight
};
})
}
function finalize(key, value) {
value.average = value.sum / value.count;
value.population_variance = value.diff / value.count;
value.population_standard_deviation = Math.sqrt(value.population_variance);
value.sample_variance = value.diff / (value.count - 1);
value.sample_standard_deviation = Math.sqrt(value.sample_variance);
delete value.diff;
return value;
}
iotModel.mapReduce({map: map, reduce:reduce, finalize: finalize}, function(err, results) {
console.log(results);
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment