Data Compaction/Locality Example

gistfile1.js

> db.disktest_noorg.ensureIndex({"userid":-1});
> db.disktest_org.ensureIndex({"userid":-1});
> db.disktest_noorg.find().sort({userid:-1})   
{ "_id" : ObjectId("4dd2d82b6a2e502b3043ef33"), "userid" : 49999, "imageid" : 20, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "79357fb65ba7b87f2632dfe8e098400c" }
{ "_id" : ObjectId("4dd2d82e6a2e502b30442186"), "userid" : 49999, "imageid" : 14, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "43a98d693fc469b323f2c9033176cd97" }
{ "_id" : ObjectId("4dd2d8316a2e502b30448652"), "userid" : 49999, "imageid" : 18, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "810c9ca8b61eff972e0d514191874f22" }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d4d4"), "userid" : 49996, "imageid" : 39, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "e864723614597f09c6d09469aa3e3235" }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efcd"), "userid" : 49995, "imageid" : 24, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "5103c246d520e0239a2f6dd2a19f00f1" }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efda"), "userid" : 49995, "imageid" : 70, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "eee3daa33b31bd4cddbc66dac0ef1fa2" }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f2e5"), "userid" : 49995, "imageid" : 31, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "ed047fbd683437234248b08485170905" }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f3e1"), "userid" : 49995, "imageid" : 83, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "80b4b515586952083d96f517683ae8cf" }
{ "_id" : ObjectId("4dd2d8316a2e502b3044747d"), "userid" : 49995, "imageid" : 36, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "d1122068e5c58a4502b019c7ec453acc" }
{ "_id" : ObjectId("4dd2d8336a2e502b3044a6ff"), "userid" : 49995, "imageid" : 88, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "afbd6bc65bfe02fe3fc8470aba822969" }
{ "_id" : ObjectId("4dd2d82d6a2e502b304411a9"), "userid" : 49994, "imageid" : 80, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "2a304b4c9f488ae31f2c8b5801257bb6" }
{ "_id" : ObjectId("4dd2d82d6a2e502b3044132d"), "userid" : 49994, "imageid" : 26, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "a9124995f8b4d43f2b916bf8175c0c32" }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043dc22"), "userid" : 49992, "imageid" : 65, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "ff3f2c24c4c046fdaec8d830a0c9708b" }
{ "_id" : ObjectId("4dd2d8266a2e502b30437f31"), "userid" : 49991, "imageid" : 50, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "20a3e52f5d8b30fb898b285021823348" }
{ "_id" : ObjectId("4dd2d8326a2e502b30448a4f"), "userid" : 49991, "imageid" : 66, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "c481f2a2f2769cc9d500d62877b1359c" }
{ "_id" : ObjectId("4dd2d8296a2e502b3043a5d6"), "userid" : 49990, "imageid" : 57, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "e598f318663eb180238e082d032a071c" }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d57f"), "userid" : 49990, "imageid" : 21, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "0059125396901267051925304b7d2968" }
{ "_id" : ObjectId("4dd2d8246a2e502b304357ac"), "userid" : 49990, "imageid" : 72, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "bf7908466325d1b1071c0242fe1f1c11" }
{ "_id" : ObjectId("4dd2d8246a2e502b30435fd4"), "userid" : 49990, "imageid" : 32, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "cdb76d09b3f85cf0a54b6d7c46610da4" }
{ "_id" : ObjectId("4dd2d8326a2e502b30449234"), "userid" : 49990, "imageid" : 72, "img" : "www.kennygorman.com/foo.jpg", "title" : "This is a sample title", "data" : "dd3b3160f8c4708721c36dd9cacf8f28" }
has more


> db.disktest_noorg.find({}, {'$diskLoc': 1,'userid':1}).sort({userid:-1}).limit(20).showDiskLoc()
{ "_id" : ObjectId("4dd2d82b6a2e502b3043ef33"), "userid" : 49999, "$diskLoc" : { "file" : 0, "offset" : 52924692 } }
{ "_id" : ObjectId("4dd2d82e6a2e502b30442186"), "userid" : 49999, "$diskLoc" : { "file" : 0, "offset" : 55346696 } }
{ "_id" : ObjectId("4dd2d8316a2e502b30448652"), "userid" : 49999, "$diskLoc" : { "file" : 1, "offset" : 2062832 } }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d4d4"), "userid" : 49996, "$diskLoc" : { "file" : 0, "offset" : 51655504 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efcd"), "userid" : 49995, "$diskLoc" : { "file" : 0, "offset" : 52953644 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efda"), "userid" : 49995, "$diskLoc" : { "file" : 0, "offset" : 52956088 } }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f2e5"), "userid" : 49995, "$diskLoc" : { "file" : 0, "offset" : 53102540 } }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f3e1"), "userid" : 49995, "$diskLoc" : { "file" : 0, "offset" : 53149916 } }
{ "_id" : ObjectId("4dd2d8316a2e502b3044747d"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 1204612 } }
{ "_id" : ObjectId("4dd2d8336a2e502b3044a6ff"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 3635452 } }
{ "_id" : ObjectId("4dd2d82d6a2e502b304411a9"), "userid" : 49994, "$diskLoc" : { "file" : 0, "offset" : 54583228 } }
{ "_id" : ObjectId("4dd2d82d6a2e502b3044132d"), "userid" : 49994, "$diskLoc" : { "file" : 0, "offset" : 54656172 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043dc22"), "userid" : 49992, "$diskLoc" : { "file" : 0, "offset" : 52007064 } }
{ "_id" : ObjectId("4dd2d8266a2e502b30437f31"), "userid" : 49991, "$diskLoc" : { "file" : 0, "offset" : 61266396 } }
{ "_id" : ObjectId("4dd2d8326a2e502b30448a4f"), "userid" : 49991, "$diskLoc" : { "file" : 1, "offset" : 2254780 } }
{ "_id" : ObjectId("4dd2d8296a2e502b3043a5d6"), "userid" : 49990, "$diskLoc" : { "file" : 0, "offset" : 49393864 } }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d57f"), "userid" : 49990, "$diskLoc" : { "file" : 0, "offset" : 51687652 } }
{ "_id" : ObjectId("4dd2d8246a2e502b304357ac"), "userid" : 49990, "$diskLoc" : { "file" : 0, "offset" : 59364400 } }
{ "_id" : ObjectId("4dd2d8246a2e502b30435fd4"), "userid" : 49990, "$diskLoc" : { "file" : 0, "offset" : 59756944 } }
{ "_id" : ObjectId("4dd2d8326a2e502b30449234"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 2634728 } }
>                                                                                                 
> 
> 
> 
> var arr=db.disktest_noorg.find().sort({userid:-1})
> for(var i=0; i<arr.length(); i++) {
...     db.disktest_org.insert(arr[i]);
...  }
> db.disktest_org.find({}, {'$diskLoc': 1,'userid':1}).sort({userid:-1}).limit(20).showDiskLoc()
{ "_id" : ObjectId("4dd2d82b6a2e502b3043ef33"), "userid" : 49999, "$diskLoc" : { "file" : 1, "offset" : 41683632 } }
{ "_id" : ObjectId("4dd2d82e6a2e502b30442186"), "userid" : 49999, "$diskLoc" : { "file" : 1, "offset" : 41683820 } }
{ "_id" : ObjectId("4dd2d8316a2e502b30448652"), "userid" : 49999, "$diskLoc" : { "file" : 1, "offset" : 41684008 } }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d4d4"), "userid" : 49996, "$diskLoc" : { "file" : 1, "offset" : 41684196 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efcd"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41684384 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043efda"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41684572 } }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f2e5"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41684760 } }
{ "_id" : ObjectId("4dd2d82c6a2e502b3043f3e1"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41684948 } }
{ "_id" : ObjectId("4dd2d8316a2e502b3044747d"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41685136 } }
{ "_id" : ObjectId("4dd2d8336a2e502b3044a6ff"), "userid" : 49995, "$diskLoc" : { "file" : 1, "offset" : 41685324 } }
{ "_id" : ObjectId("4dd2d82d6a2e502b304411a9"), "userid" : 49994, "$diskLoc" : { "file" : 1, "offset" : 41685512 } }
{ "_id" : ObjectId("4dd2d82d6a2e502b3044132d"), "userid" : 49994, "$diskLoc" : { "file" : 1, "offset" : 41685700 } }
{ "_id" : ObjectId("4dd2d82b6a2e502b3043dc22"), "userid" : 49992, "$diskLoc" : { "file" : 1, "offset" : 41685888 } }
{ "_id" : ObjectId("4dd2d8266a2e502b30437f31"), "userid" : 49991, "$diskLoc" : { "file" : 1, "offset" : 41686076 } }
{ "_id" : ObjectId("4dd2d8326a2e502b30448a4f"), "userid" : 49991, "$diskLoc" : { "file" : 1, "offset" : 41686264 } }
{ "_id" : ObjectId("4dd2d8296a2e502b3043a5d6"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 41686452 } }
{ "_id" : ObjectId("4dd2d82a6a2e502b3043d57f"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 41686640 } }
{ "_id" : ObjectId("4dd2d8246a2e502b304357ac"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 41686828 } }
{ "_id" : ObjectId("4dd2d8246a2e502b30435fd4"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 41687016 } }
{ "_id" : ObjectId("4dd2d8326a2e502b30449234"), "userid" : 49990, "$diskLoc" : { "file" : 1, "offset" : 41687204 } }


> var noorg = new Array();
> var arr=db.disktest_noorg.find({}, {'$diskLoc':1, 'userid':1}).limit(20).showDiskLoc()
> for(var i=0; i<arr.length(); i++) {
...     var b=Math.round(arr[i].$diskLoc.offset/512);
...     printjson(arr[i].userid+" "+b);
...     noorg.push(b);
... }
"21672 113562"
"9623 113562"
"4004 113563"
"11031 113563"
"17770 113563"
"36297 113564"
"28403 113564"
"776 113564"
"9094 113565"
"35484 113565"
"10078 113566"
"22259 113566"
"46254 113566"
"35119 113567"
"35290 113567"
"44706 113567"
"32259 113568"
"16737 113568"
"3593 113568"
"17301 113569"
20

> // take a peek where things live organized
> var org = new Array();
> var arr=db.disktest_org.find({}, {'$diskLoc':1, 'userid':1}).limit(20).showDiskLoc()
> for(var i=0; i<arr.length(); i++) {
...     var b=Math.round(arr[i].$diskLoc.offset/512);
...     printjson(arr[i].userid+" "+b);
...     noorg.push(b);
... }
"49999 81413"
"49999 81414"
"49999 81414"
"49996 81414"
"49995 81415"
"49995 81415"
"49995 81416"
"49995 81416"
"49995 81416"
"49995 81417"
"49994 81417"
"49994 81417"
"49992 81418"
"49991 81418"
"49991 81418"
"49990 81419"
"49990 81419"
"49990 81420"
"49990 81420"
"49990 81420"
40
>