Skip to content

Instantly share code, notes, and snippets.

@nhocki
Last active April 3, 2018 14:05
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nhocki/8bb791fa45f4ea16142e60f150d585fe to your computer and use it in GitHub Desktop.
Save nhocki/8bb791fa45f4ea16142e60f150d585fe to your computer and use it in GitHub Desktop.
Modified https://github.com/stickermule/rump to use KEYS against an older Redis installation.
#!/usr/bin/env bash
VERSION="$1"
rm rump-*
GOOS=darwin GOARCH=amd64 go build -o rump-$VERSION-darwin-amd64 rump.go
GOOS=linux GOARCH=amd64 go build -o rump-$VERSION-linux-amd64 rump.go
GOOS=linux GOARCH=arm go build -o rump-$VERSION-linux-arm rump.go
GOOS=windows GOARCH=amd64 go build -o rump-$VERSION-windows-amd64 rump.go
package main
import (
"flag"
"fmt"
"os"
"time"
"github.com/garyburd/redigo/redis"
)
// Report all errors to stdout.
func handle(err error) {
if err != nil && err != redis.ErrNil {
fmt.Println(err)
os.Exit(1)
}
}
// Scan and queue source keys.
func get(conn redis.Conn, queue chan<- map[string]string) {
var (
// cursor int64
keys []string
)
start := time.Now()
fmt.Printf("Fetching keys: %s\n", start.String())
keys, err := redis.Strings(conn.Do("KEYS", "*"))
handle(err)
fmt.Printf("Took: %s\n", time.Now().Sub(start))
fmt.Printf("Total Keys: %d\n", len(keys))
batchSize := 10
for i := 0; i < len(keys); i += batchSize {
last := i + batchSize
if last > len(keys) {
last = len(keys)
}
// Get pipelined dumps.
for _, key := range keys[i:last] {
conn.Send("DUMP", key)
}
dumps, err := redis.Strings(conn.Do(""))
handle(err)
// Build batch map.
batch := make(map[string]string)
for j, key := range keys[i:last] {
batch[key] = dumps[j]
}
fmt.Printf(">")
queue <- batch
}
close(queue)
fmt.Printf("Total Time: %s\n", time.Now().Sub(start))
// for {
// // Scan a batch of keys.
// values, err := redis.Values(conn.Do("SCAN", cursor))
// handle(err)
// values, err = redis.Scan(values, &cursor, &keys)
// handle(err)
// // Get pipelined dumps.
// for _, key := range keys {
// conn.Send("DUMP", key)
// }
// dumps, err := redis.Strings(conn.Do(""))
// handle(err)
// // Build batch map.
// batch := make(map[string]string)
// for i := range keys {
// batch[keys[i]] = dumps[i]
// }
// // Last iteration of scan.
// if cursor == 0 {
// // queue last batch.
// select {
// case queue <- batch:
// }
// close(queue)
// break
// }
// fmt.Printf(">")
// // queue current batch.
// queue <- batch
// }
}
// Restore a batch of keys on destination.
// func put(conn redis.Conn, queue <-chan map[string]string) {
func put(conn redis.Conn, queue <-chan map[string]string) {
for batch := range queue {
for key, value := range batch {
conn.Send("RESTORE", key, "0", value)
}
_, err := conn.Do("")
handle(err)
fmt.Printf(".")
}
}
func main() {
from := flag.String("from", "", "example: redis://127.0.0.1:6379/0")
to := flag.String("to", "", "example: redis://127.0.0.1:6379/1")
flag.Parse()
source, err := redis.DialURL(*from)
handle(err)
destination, err := redis.DialURL(*to)
handle(err)
defer source.Close()
defer destination.Close()
// Channel where batches of keys will pass.
queue := make(chan map[string]string, 100)
// Scan and send to queue.
// go get(source, queue)
go get(source, queue)
// Restore keys as they come into queue.
put(destination, queue)
fmt.Println("Sync done.")
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment