Simple spring boot app with Jedis inserting 1M keys in redis using pipelining and multi-threading
package me.grison.redis.foo; | |
import org.springframework.boot.CommandLineRunner; | |
import org.springframework.boot.SpringApplication; | |
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; | |
import org.springframework.boot.autoconfigure.SpringBootApplication; | |
import org.springframework.context.annotation.Bean; | |
import org.springframework.context.annotation.ComponentScan; | |
import org.springframework.context.annotation.Configuration; | |
import redis.clients.jedis.Jedis; | |
import redis.clients.jedis.JedisPool; | |
import redis.clients.jedis.JedisPoolConfig; | |
import redis.clients.jedis.Pipeline; | |
import java.util.UUID; | |
import java.util.concurrent.ExecutorService; | |
import java.util.concurrent.Executors; | |
import java.util.concurrent.TimeUnit; | |
@ComponentScan | |
@Configuration | |
@SpringBootApplication | |
@EnableAutoConfiguration | |
public class RedisMassImport { | |
@Bean | |
JedisPool jedisPool() { | |
JedisPoolConfig poolConfig = new JedisPoolConfig(); | |
poolConfig.setTestOnBorrow(true); | |
poolConfig.setTestWhileIdle(true); | |
return new JedisPool(poolConfig, "localhost", 6379, 30); | |
} | |
@Bean | |
CommandLineRunner init(JedisPool pool) { | |
return args -> { | |
int numKeys = 1_000_000; | |
int availableProcessors = Runtime.getRuntime().availableProcessors() * 2; | |
ExecutorService exec = Executors.newFixedThreadPool(availableProcessors); | |
// pre-compute UUIDs | |
String[] uuids = new String[numKeys]; | |
for (int i = 0; i < numKeys; ++i) | |
uuids[i] = UUID.randomUUID().toString(); | |
// split | |
final int portion = numKeys / availableProcessors; | |
long time = System.nanoTime(); | |
for (int i = 0; i < availableProcessors; ++i) { | |
final int threadId = i; | |
exec.submit(() -> { | |
try (Jedis jedis = pool.getResource()) { | |
// throw the data at redis | |
Pipeline p = jedis.pipelined(); | |
for (int j = threadId * portion; j < (threadId + 1) * portion; ++j) | |
p.set(uuids[j], uuids[j]); | |
p.sync(); | |
} | |
}); | |
} | |
exec.shutdown(); | |
try { | |
exec.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); | |
long endTime = System.nanoTime(); | |
System.out.println("\tTook " + (endTime - time) + "ns to sync the pipeline."); | |
} catch (InterruptedException e) { | |
System.err.println("Execution interrupted: " + e.getMessage()); | |
} | |
}; | |
} | |
public static void main(String[] args) { | |
SpringApplication.run(RedisMassImport.class, args); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment