Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Sample Go App to mass insert 1M keys in redis using goroutines and unix socket
package main
import (
"fmt"
"log"
"net"
"runtime"
"sync"
"time"
"github.com/garyburd/redigo/redis"
"github.com/twinj/uuid"
)
const numKeys = 1000000
const routines = 16
const portion = numKeys / routines
var uuids [numKeys]string
var wg sync.WaitGroup
var pool *redis.Pool
func prepareUUids() {
for i := 0; i < numKeys; i++ {
uuids[i] = uuid.NewV4().String()
}
}
func createPool() {
pool = &redis.Pool{
MaxIdle: 16,
MaxActive: 16,
IdleTimeout: 60 * time.Second,
Dial: func () (redis.Conn, error) {
c, err := net.Dial("unix", "/tmp/redis.sock")
if err != nil {
log.Fatal(err)
return nil, err
}
return redis.NewConn(c, 10*time.Second, 10*time.Second), nil
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
func massImport() {
wg.Add(routines)
for i := 0; i < routines; i++ {
go importRoutine(i, pool.Get())
}
wg.Wait()
}
func importRoutine(t int, client redis.Conn) {
defer client.Flush()
defer wg.Done()
for i := t * portion; i < (t + 1) * portion; i++ {
client.Send("SET", uuids[i], uuids[i])
}
}
func closePool() {
pool.Close()
}
func main() {
runtime.GOMAXPROCS(16)
prepareUUids()
createPool()
start := time.Now()
massImport()
elapsed := time.Since(start)
fmt.Println("Took ", elapsed)
closePool()
}
@MarshalOfficial

This comment has been minimized.

Copy link

@MarshalOfficial MarshalOfficial commented Jan 4, 2021

hello, how much time does it take for you with this code?

@agrison

This comment has been minimized.

Copy link
Owner Author

@agrison agrison commented Jan 4, 2021

On my MacBook Pro in 2015 it took like 3 sec, so roughly 329K updates / sec.
On my current machine (Ryzen 3900X, 64GB memory) on Windows via WSL2 it takes 1.22 sec

@agrison

This comment has been minimized.

Copy link
Owner Author

@agrison agrison commented Jan 4, 2021

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.