Skip to content

Instantly share code, notes, and snippets.

@bwaldvogel
Created September 19, 2011 16:34
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bwaldvogel/1226902 to your computer and use it in GitHub Desktop.
Save bwaldvogel/1226902 to your computer and use it in GitHub Desktop.
db.collection.stats()
{
"sharded" : true,
"flags" : 1,
"ns" : "database.collection",
"count" : 42821370,
"numExtents" : 80,
"size" : 38573270908,
"storageSize" : 43791466336,
"totalIndexSize" : 2008426224,
"indexSizes" : {
"_id_" : 2008426224
},
"avgObjSize" : 900.7948813407885,
"nindexes" : 1,
"nchunks" : 945,
"shards" : {
"rs1" : {
"ns" : "database.collection",
"count" : 20662481,
"size" : 20199884736,
"avgObjSize" : 977.6117754687832,
"storageSize" : 24042160032,
"numExtents" : 41,
"nindexes" : 1,
"lastExtentSize" : 2146426864,
"paddingFactor" : 1.009999999955934,
"flags" : 1,
"totalIndexSize" : 977416272,
"indexSizes" : {
"_id_" : 977416272
},
"ok" : 1
},
"rs2" : {
"ns" : "database.collection",
"count" : 22158889,
"size" : 18373386172,
"avgObjSize" : 829.1654952556511,
"storageSize" : 19749306304,
"numExtents" : 39,
"nindexes" : 1,
"lastExtentSize" : 2146426864,
"paddingFactor" : 1.0099999999444642,
"flags" : 1,
"totalIndexSize" : 1031009952,
"indexSizes" : {
"_id_" : 1031009952
},
"ok" : 1
}
},
"ok" : 1
}
import java.util.ArrayList;
import java.util.List;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
public class MultipleIDQueryTest {
private static final String MONGO_URL = "localhost";
private static final String DATABASE = "database";
private static final String COLLECTION = "collection";
private static final int MAX_NUM_IDS = 10000;
public static void main( String[] args ) throws Exception {
Mongo mongo = new Mongo(MONGO_URL);
try {
DBCollection collection = mongo.getDB(DATABASE).getCollection(COLLECTION);
long collectionSize = collection.count();
if ( collectionSize < MAX_NUM_IDS ) {
throw new RuntimeException("collection has too few ids: " + collectionSize);
}
List<Object> ids = collectSomeIds(collection);
for ( int queryLimit = 500; queryLimit <= MAX_NUM_IDS; queryLimit += 500 ) {
BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$in", ids.subList(0, queryLimit)));
DBCursor cursor = collection.find(query).limit(queryLimit);
int itCount = 0;
while ( cursor.hasNext() ) {
cursor.next();
itCount++;
}
int cursorCount = cursor.count();
System.out.printf("cursor.count(): %5d | itcount: %5d ", cursorCount, itCount);
if ( itCount != cursorCount || itCount != queryLimit ) {
System.out.println("FAIL!");
} else {
System.out.println("OK");
}
}
}
finally {
mongo.close();
}
}
private static List<Object> collectSomeIds( DBCollection collection ) {
List<Object> ids = new ArrayList<Object>();
for ( DBObject o : collection.find() ) {
ids.add(o.get("_id"));
if ( ids.size() >= MAX_NUM_IDS ) break;
}
return ids;
}
}
cursor.count(): 500 | itcount: 500 OK
cursor.count(): 1000 | itcount: 1000 OK
cursor.count(): 1500 | itcount: 1500 OK
cursor.count(): 2000 | itcount: 2000 OK
cursor.count(): 2500 | itcount: 1445 FAIL!
cursor.count(): 3000 | itcount: 3000 OK
cursor.count(): 3500 | itcount: 3500 OK
cursor.count(): 4000 | itcount: 4000 OK
cursor.count(): 4500 | itcount: 4500 OK
cursor.count(): 5000 | itcount: 5000 OK
cursor.count(): 5500 | itcount: 5500 OK
cursor.count(): 6000 | itcount: 6000 OK
cursor.count(): 6500 | itcount: 6500 OK
cursor.count(): 7000 | itcount: 7000 OK
cursor.count(): 7500 | itcount: 7500 OK
cursor.count(): 8000 | itcount: 7483 FAIL!
cursor.count(): 8500 | itcount: 6572 FAIL!
cursor.count(): 9000 | itcount: 6072 FAIL!
cursor.count(): 9500 | itcount: 5572 FAIL!
cursor.count(): 10000 | itcount: 5564 FAIL!
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment