mgo insert loop - parallelized vs. standard - The point of this test isn't to check the throughput of inserting 10000 records using mgo, but to explore the difference between doing standard and parallel loops using an overblown common case. Out of curiosity I have added a bulk insert benchmark at the bottom, which is clearly the most powerful wa…

benchmark.go

package test

import (
    "fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"testing"
)

const INSERT_COUNT int = 10000

type User struct {
	Id    bson.ObjectId `bson:"_id,omitempty" json:"_id"`
	Email string        `bson:"email" json:"email"`
}

func (self *User) Init() {
	self.Id = bson.NewObjectId()
}

func BenchmarkFlatInsert(b *testing.B) {
	b.StopTimer()
	// Database
	dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
	if err != nil {
		panic(err)
	}

	// Collections
	uc := dbs.Clone().DB("").C("users")
	defer dbs.Clone().DB("").Session.Close()

	// Clear DB
	uc.RemoveAll(bson.M{})
	b.StartTimer()

	for n := 0; n < b.N; n++ {

		count := INSERT_COUNT
		for i := 0; i < count; i++ {
			loop_user := User{}
			loop_user.Init()
			loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
			if err := uc.Insert(&loop_user); err != nil {
				panic(err)
			}
		}

	}
}

func BenchmarkParallelInsert(b *testing.B) {
	b.StopTimer()
	// Database
	dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
	if err != nil {
		panic(err)
	}

	// Collections
	uc := dbs.Clone().DB("").C("users")
	defer dbs.Clone().DB("").Session.Close()

	// Clear DB
	uc.RemoveAll(bson.M{})
	b.StartTimer()

	for n := 0; n < b.N; n++ {

		count := INSERT_COUNT
		sem := make(chan bool, count)
		for i := 0; i < count; i++ {
			go func(i int) {
				loop_user := User{}
				loop_user.Init()
				loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
				if err := uc.Insert(&loop_user); err != nil {
					panic(err)
				}
				sem <- true
			}(i)
		}
		for j := 0; j < count; j++ {
			<-sem
		}

	}
}

func BenchmarkBatchInsert(b *testing.B) {
	b.StopTimer()
	// Database
	dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
	if err != nil {
		panic(err)
	}

	// Collections
	uc := dbs.Clone().DB("").C("users")
	defer dbs.Clone().DB("").Session.Close()

	// Clear DB
	uc.RemoveAll(bson.M{})
	b.StartTimer()

	for n := 0; n < b.N; n++ {

		count := INSERT_COUNT
		users := make([]User, count)
		for i := 0; i < count; i++ {
			loop_user := User{}
			loop_user.Init()
			loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
			users[i] = loop_user
		}
		if err := uc.Insert(&users); err != nil {
			panic(err)
		}

	}
}

results.txt

go test -bench=".*" -cpu 1,2

BenchmarkFlatInsert           1	1224019000 ns/op
BenchmarkFlatInsert-2         1	1155095000 ns/op
BenchmarkParallelInsert       5	 408693800 ns/op
BenchmarkParallelInsert-2     5	 409177400 ns/op
BenchmarkBatchInsert          50  44279860 ns/op
BenchmarkBatchInsert-2        50  44368200 ns/op

Your batchinsert code does not do what you think, it inserts an array of structs into one document, so it inserts N times INSERT_COUNTS structs into N documents, instead of N times INSERT_COUNT documents.

Hey guys, I have used other function, the Bulk()

func BenchmarkBulkInsert(b *testing.B) {
	b.StopTimer()
	// Database
	dbs, err := mgo.Dial("mongodb://localhost/ac-bench")
	if err != nil {
		panic(err)
	}

	// Collections
	uc := dbs.Clone().DB("").C("usersBenchmarkBulk")
	defer dbs.Clone().DB("").Session.Close()

	// Clear DB
	uc.RemoveAll(bson.M{})
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		count := INSERT_COUNT
		users := make([]interface{}, count)
		for i := 0; i < count; i++ {
			loop_user := User{}
			loop_user.Init()
			loop_user.Email = fmt.Sprintf("report-%d@example.com", i)
			users[i] = loop_user
		}
		bulk := uc.Bulk()
		bulk.Unordered()
		bulk.Insert(users...)
		_, bulkErr := bulk.Run()
		if bulkErr != nil {
			panic(err)
		}
	}
}

go test -bench=".*" -cpu 1,2

BenchmarkFlatInsert                  100          12281273 ns/op
BenchmarkFlatInsert-2                100          12981239 ns/op
BenchmarkParallelInsert              200           6963727 ns/op
BenchmarkParallelInsert-2            200           7948616 ns/op
BenchmarkBulkInsert                 2000           1134782 ns/op
BenchmarkBulkInsert-2               2000           1072460 ns/op

how can run this package?