Skip to content

Instantly share code, notes, and snippets.

@gadelkareem
Last active February 12, 2018 13:31
Show Gist options
  • Save gadelkareem/753808025c92c4783e82b7e2dfbbe0ac to your computer and use it in GitHub Desktop.
Save gadelkareem/753808025c92c4783e82b7e2dfbbe0ac to your computer and use it in GitHub Desktop.
package main
import (
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"fmt"
"strconv"
"time"
"sync"
)
const Limit = 5
const TotalDocsNum = 1000
type TestDocument struct {
Id bson.ObjectId `bson:"_id"`
Name string
Data []byte
Arr []string
Text string
}
func main() {
session, err := mgo.Dial("")
check(err)
defer session.Close()
collection := session.DB("testDB").C("testCollection")
generateDB(collection)
//first page
paginateWithIds(collection, 1)
//last page
paginateWithIds(collection, TotalDocsNum/Limit-1)
//first page
paginateWithSkip(collection, 1)
//last page
paginateWithSkip(collection, TotalDocsNum/Limit-1)
}
func paginateWithIds(collection *mgo.Collection, page int) []*TestDocument {
start := time.Now()
var results []struct {
Id bson.ObjectId `bson:"_id"`
}
//Ids should be cached here based on the first query
err := collection.Find(nil).Sort("_id").Select(bson.M{"_id": 1}).All(&results)
check(err)
resultsLen := len(results)
if resultsLen == 0 {
fmt.Println("nothing found")
return nil
}
skip := (page - 1) * Limit
max := skip + Limit
if max > resultsLen {
max = resultsLen
}
results = results[skip:max]
var ids []bson.ObjectId
for _, v := range results {
ids = append(ids, v.Id)
}
if len(ids) == 0 {
fmt.Println("nothing found")
return nil
}
docs := make([]*TestDocument, Limit)
err = collection.Find(bson.M{"_id": bson.M{"$in": ids}}).Sort("_id").Limit(Limit).All(&docs)
check(err)
fmt.Printf("paginateWithIds -> page %d with docs from %s to %s in %s\n", page, docs[0].Name, docs[len(docs)-1].Name, time.Since(start))
return docs
}
func paginateWithSkip(collection *mgo.Collection, page int) []*TestDocument {
start := time.Now()
skip := (page - 1) * Limit
docs := make([]*TestDocument, Limit)
err := collection.Find(nil).Sort("_id").Skip(skip).Limit(Limit).All(&docs)
check(err)
fmt.Printf("paginateWithSkip -> page %d with docs from %s to %s in %s\n", page, docs[0].Name, docs[len(docs)-1].Name, time.Since(start))
return docs
}
func generateDB(collection *mgo.Collection) {
totalDocuments, err := collection.Find(nil).Count()
check(err)
if totalDocuments >= TotalDocsNum {
return
}
fmt.Println("Generating test DB")
var wg sync.WaitGroup
guard := make(chan struct{}, 100)
round := 10
for i := totalDocuments; i < TotalDocsNum; i += round {
guard <- struct{}{}
wg.Add(1)
go func(x int) {
docs := make([]interface{}, round)
for total := x + round; x < total && x < TotalDocsNum; x++ {
docs = append(docs, TestDocument{bson.NewObjectId(), "_" + strconv.Itoa(i), make([]byte, 1e4), make([]string, 1e3), string(make([]rune, 1e6))})
}
b := collection.Bulk()
b.Unordered()
b.Insert(docs...)
_, err = b.Run()
check(err)
<-guard
wg.Done()
}(i)
}
wg.Wait()
}
func check(err error) {
if err != nil {
panic(err)
}
}
> go run test.go
Generating test DB
paginateWithIds -> page 1 with docs from _330 to _490 in 333.27532ms
paginateWithIds -> page 199 with docs from _1000 to _1000 in 171.75227ms <---------
paginateWithSkip -> page 1 with docs from _330 to _490 in 120.980189ms
paginateWithSkip -> page 199 with docs from _1000 to _1000 in 14.603289714s <---------
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment