Skip to content

Instantly share code, notes, and snippets.

@boj
Last active April 9, 2019 16:26
Show Gist options
  • Star 15 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save boj/5412538 to your computer and use it in GitHub Desktop.
Save boj/5412538 to your computer and use it in GitHub Desktop.
mgo insert loop - parallelized vs. standard - The point of this test isn't to check the throughput of inserting 10000 records using mgo, but to explore the difference between doing standard and parallel loops using an overblown common case. Out of curiosity I have added a bulk insert benchmark at the bottom, which is clearly the most powerful wa…
package test
import (
"fmt"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"testing"
)
const INSERT_COUNT int = 10000
type User struct {
Id bson.ObjectId `bson:"_id,omitempty" json:"_id"`
Email string `bson:"email" json:"email"`
}
func (self *User) Init() {
self.Id = bson.NewObjectId()
}
func BenchmarkFlatInsert(b *testing.B) {
b.StopTimer()
// Database
dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
if err != nil {
panic(err)
}
// Collections
uc := dbs.Clone().DB("").C("users")
defer dbs.Clone().DB("").Session.Close()
// Clear DB
uc.RemoveAll(bson.M{})
b.StartTimer()
for n := 0; n < b.N; n++ {
count := INSERT_COUNT
for i := 0; i < count; i++ {
loop_user := User{}
loop_user.Init()
loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
if err := uc.Insert(&loop_user); err != nil {
panic(err)
}
}
}
}
func BenchmarkParallelInsert(b *testing.B) {
b.StopTimer()
// Database
dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
if err != nil {
panic(err)
}
// Collections
uc := dbs.Clone().DB("").C("users")
defer dbs.Clone().DB("").Session.Close()
// Clear DB
uc.RemoveAll(bson.M{})
b.StartTimer()
for n := 0; n < b.N; n++ {
count := INSERT_COUNT
sem := make(chan bool, count)
for i := 0; i < count; i++ {
go func(i int) {
loop_user := User{}
loop_user.Init()
loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
if err := uc.Insert(&loop_user); err != nil {
panic(err)
}
sem <- true
}(i)
}
for j := 0; j < count; j++ {
<-sem
}
}
}
func BenchmarkBatchInsert(b *testing.B) {
b.StopTimer()
// Database
dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
if err != nil {
panic(err)
}
// Collections
uc := dbs.Clone().DB("").C("users")
defer dbs.Clone().DB("").Session.Close()
// Clear DB
uc.RemoveAll(bson.M{})
b.StartTimer()
for n := 0; n < b.N; n++ {
count := INSERT_COUNT
users := make([]User, count)
for i := 0; i < count; i++ {
loop_user := User{}
loop_user.Init()
loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
users[i] = loop_user
}
if err := uc.Insert(&users); err != nil {
panic(err)
}
}
}
go test -bench=".*" -cpu 1,2
BenchmarkFlatInsert 1 1224019000 ns/op
BenchmarkFlatInsert-2 1 1155095000 ns/op
BenchmarkParallelInsert 5 408693800 ns/op
BenchmarkParallelInsert-2 5 409177400 ns/op
BenchmarkBatchInsert 50 44279860 ns/op
BenchmarkBatchInsert-2 50 44368200 ns/op
@herhu
Copy link

herhu commented Apr 9, 2019

how can run this package?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment