Faster MongoDB pagination https://gadelkareem.com/2018/02/11/workaround-slow-pagination-using-skip-mongodb/

paginate.go

package main

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"fmt"
	"strconv"
	"time"
	"sync"
)

const Limit = 5
const TotalDocsNum = 1000

type TestDocument struct {
	Id   bson.ObjectId `bson:"_id"`
	Name string
	Data []byte
	Arr  []string
	Text string
}

func main() {
	session, err := mgo.Dial("")
	check(err)
	defer session.Close()

	collection := session.DB("testDB").C("testCollection")
	generateDB(collection)

	//first page
	paginateWithIds(collection, 1)
	//last page
	paginateWithIds(collection, TotalDocsNum/Limit-1)

	//first page
	paginateWithSkip(collection, 1)
	//last page
	paginateWithSkip(collection, TotalDocsNum/Limit-1)

}

func paginateWithIds(collection *mgo.Collection, page int) []*TestDocument {

	start := time.Now()

	var results []struct {
		Id bson.ObjectId `bson:"_id"`
	}

	//Ids should be cached here based on the first query
	err := collection.Find(nil).Sort("_id").Select(bson.M{"_id": 1}).All(&results)
	check(err)

	resultsLen := len(results)
	if resultsLen == 0 {
		fmt.Println("nothing found")
		return nil
	}

	skip := (page - 1) * Limit
	max := skip + Limit
	if max > resultsLen {
		max = resultsLen
	}
	results = results[skip:max]

	var ids []bson.ObjectId
	for _, v := range results {
		ids = append(ids, v.Id)
	}

	if len(ids) == 0 {
		fmt.Println("nothing found")
		return nil
	}

	docs := make([]*TestDocument, Limit)
	err = collection.Find(bson.M{"_id": bson.M{"$in": ids}}).Sort("_id").Limit(Limit).All(&docs)
	check(err)

	fmt.Printf("paginateWithIds -> page %d with docs from %s to %s in %s\n", page, docs[0].Name, docs[len(docs)-1].Name, time.Since(start))
	return docs

}

func paginateWithSkip(collection *mgo.Collection, page int) []*TestDocument {

	start := time.Now()
	skip := (page - 1) * Limit

	docs := make([]*TestDocument, Limit)
	err := collection.Find(nil).Sort("_id").Skip(skip).Limit(Limit).All(&docs)
	check(err)

	fmt.Printf("paginateWithSkip -> page %d with docs from %s to %s in %s\n", page, docs[0].Name, docs[len(docs)-1].Name, time.Since(start))
	return docs

}

func generateDB(collection *mgo.Collection) {

	totalDocuments, err := collection.Find(nil).Count()
	check(err)

	if totalDocuments >= TotalDocsNum {
		return
	}
	fmt.Println("Generating test DB")

	var wg sync.WaitGroup
	guard := make(chan struct{}, 100)
	round := 10
	for i := totalDocuments; i < TotalDocsNum; i += round {
		guard <- struct{}{}
		wg.Add(1)
		go func(x int) {
			docs := make([]interface{}, round)
			for total := x + round; x < total && x < TotalDocsNum; x++ {
				docs = append(docs, TestDocument{bson.NewObjectId(), "_" + strconv.Itoa(i), make([]byte, 1e4), make([]string, 1e3), string(make([]rune, 1e6))})

			}
			b := collection.Bulk()
			b.Unordered()
			b.Insert(docs...)
			_, err = b.Run()
			check(err)
			<-guard
			wg.Done()
		}(i)
	}
	wg.Wait()
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

results.sh

> go run test.go 
Generating test DB
paginateWithIds -> page 1 with docs from _330 to _490 in 333.27532ms
paginateWithIds -> page 199 with docs from _1000 to _1000 in 171.75227ms      <---------
paginateWithSkip -> page 1 with docs from _330 to _490 in 120.980189ms
paginateWithSkip -> page 199 with docs from _1000 to _1000 in 14.603289714s   <---------