Skip to content

Instantly share code, notes, and snippets.

@woodywang
Last active December 16, 2015 14:59
Show Gist options
  • Save woodywang/5452928 to your computer and use it in GitHub Desktop.
Save woodywang/5452928 to your computer and use it in GitHub Desktop.
对以下说法的质疑: $in 操作一次是快,但是在 list 量大的时候会造成全局的阻塞,造成大量的慢查询(查询查日志中都有记录)。因为目前一个库要对应的查询各种各样,每秒的读操作要上千或者几千,并且 mongodb 是单进程在跑,所以我们要遵循的原则就是单次请求要尽量快的完成。$in 操作,尽量少用,即使要用,list 中的元素也尽量的少。
package com.woodywang;
import com.mongodb.*;
public class Benchmark {
/**
* 使用线程模拟多个客户端并发请求
*/
static class Finder implements Runnable {
private int size;
private int loop;
private String id;
private DBCollection collection;
/**
* 构造执行 find 操作的客户端
*
* @param size $in 条件长度
* @param loop 重复执行次数(使统计结果更明显)
* @param id 线程标识
* @throws Exception
*/
public Finder(int size, int loop, String id) throws Exception {
this.size = size;
this.loop = loop;
this.id = id;
MongoClient client = new MongoClient("127.0.0.1", 27017);
DB db = client.getDB("test");
this.collection = db.getCollection("test");
}
private String[] generateIds() {
String[] ids = new String[this.size];
for (int i = 0; i < this.size; i++) {
ids[i] = String.format("%08d", (int) (Math.random() * 1000000));
}
return ids;
}
public void run() {
long totalTime = 0;
for (int i = 0; i < this.loop; i++) {
String[] ids = generateIds();
BasicDBList idList = new BasicDBList();
for (String id : ids) {
idList.add(id);
}
DBObject condition = new BasicDBObject();
condition.put("uid", new BasicDBObject("$in", idList));
long startTime = System.currentTimeMillis();
DBCursor cursor = collection.find(condition);
cursor.count();
long endTime = System.currentTimeMillis();
totalTime += endTime - startTime;
}
System.out.println(id + ": DataSize: " + (this.loop * this.size) + "; Time: " + totalTime);
}
}
public static void main(String[] args) throws Exception {
// 创建一个带有大量 ID 的 $in 条件查询测试,如执行 $in 条件有 2000 个ID,执行一次
Thread tLong = new Thread(new Finder(2000, 1, "LongThread"));
// 创建若干个小数据量的查询测试
int countShortThreads = 100;
Thread[] threads = new Thread[countShortThreads + 1];
for (int i = 0; i < countShortThreads; i++) {
// 每个小数据量的测试定义为:每次查询只使用 1 个 ID,反复执行 2000 次
threads[i] = new Thread(new Finder(1, 2000, "ShortThread-" + i));
}
threads[countShortThreads] = tLong;
for (Thread t : threads) {
t.start();
}
for (Thread t : threads) {
t.join();
}
}
}
package com.woodywang;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;
/**
* Created with IntelliJ IDEA.
* User: woody
* Date: 4/24/13
* Time: 8:16 PM
* To change this template use File | Settings | File Templates.
*/
public class Generator {
private static void fillData(DBCollection collection, int size) throws Exception {
for (int i = 0; i < size; i++) {
String uid = String.format("%08d", i);
BasicDBObject doc = new BasicDBObject("uid", uid);
doc.append("text", generateString(15));
collection.insert(doc);
}
}
private static String generateString(int size) {
String str = "abcdefghijklmnopqrstuvwxyz";
StringBuilder builder = new StringBuilder();
for (int i = 0; i < size; i++) {
int index = (int) (Math.random() * str.length());
char c = str.charAt(index);
builder.append(c);
}
return builder.toString();
}
public static void main(String[] args) throws Exception {
MongoClient client = new MongoClient("localhost");
DB db = client.getDB("test");
DBCollection collection = db.getCollection("test");
fillData(collection, 1000000);
}
}
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.woodywang</groupId>
<artifactId>mongodb-benchmark</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>mongodb-benchmark</name>
<url>http://maven.apache.org</url>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.11.1</version>
</dependency>
</dependencies>
</project>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment