bulk.go

package main

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"
	"time"

	"github.com/Preetam/lm2"
)

func main() {
	c, err := lm2.OpenCollection("/tmp/lm2/bulk.lm2")
	if err != nil {
		log.Fatal(err)
	}

	keyStream := make(chan string, 100000)

	generateKeys := func() {
		i := 0
		for {
			select {
			case keyStream <- fmt.Sprintf("%019d-%019d-%019d-%019d-%019d-%019d-%019d-%019d",
				rand.Int63(), rand.Int63(), rand.Int63(), rand.Int63(),
				rand.Int63(), rand.Int63(), rand.Int63(), rand.Int63()):
			}
			i++
		}
	}

	go generateKeys()
	go generateKeys()

	runtime.LockOSThread()
	for {
		start := time.Now()
		wb := lm2.NewWriteBatch()
		const batchSize = 100000
		for i := 0; i < batchSize; i++ {
			key := <-keyStream
			wb.Set(key, key)
		}
		log.Println("creating batch took", time.Now().Sub(start))
		start = time.Now()
		if err := c.Update(wb); err != nil {
			log.Fatal(err)
		}
		log.Println("writing batch took", time.Now().Sub(start))
	}
}