prune-redis.go

// Launches 100 goroutines to mass delete keys that match certain criteria.
//
// Current results: ~3,600 keys/s (~193,000 keys/minute)
package main

import (
	"fmt"
	"os"
	"time"

	"gopkg.in/redis.v3"
)

var (
	address = "localhost:6379"
	db      = 7
)

func Connect() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr: address,
		DB:   int64(db),
	})

	return client
}

func main() {
	mainClient := Connect()

	// Scan all keys
	var cursor int64
	var err error

	workChan := make(chan []string, 0)

	for i := 0; i < 100; i++ {
		fmt.Printf("Creating worker %d\n", i)

		go worker(i, workChan)
	}

	for {
		var keys []string
		if cursor, keys, err = mainClient.Scan(cursor, "", 1000).Result(); err != nil {
			fmt.Printf("ERROR: %s\n", err)
			os.Exit(2)
		}

		workChan <- keys

		if cursor == 0 {
			break
		}
	}
}

func worker(workerID int, workChan chan []string) {
	client := Connect()

	for v := range workChan {
		start := time.Now()
		toDelete := make([]string, 0)

		for _, v := range v {
			duration, err := client.TTL(v).Result()
			if err != nil {
				continue
			}

			if duration.Hours() > 0 {
				toDelete = append(toDelete, v)
			}
		}

		client.Del(toDelete...)

		end := time.Now()

		diff := end.Sub(start)

		fmt.Printf("%d: Deleted %v keys in %v seconds\n", workerID, len(toDelete), diff.Seconds())
	}
}