Code for Real-time Deduping at scala

buckets.rb

buckets = keys_per_minute / keys_per_bucket
buckets_per_min = 2,000,000 / 100 = 20,000
buckets_per_sec = 20,000 / 60 = 333

key_calculation.scala

(overhead + key_bytes + value_bytes) * keys_per_minute * 60 * 24
(64 + 36 + 13) * 2,000,000 * 60 * 24 = 300GB

lua.rb

HMGET(bucket, message_ids)

message_ids.each do
  if existing owner matches or not set
    not dupe
  else
    dupe
  end
end

HMSET(bucket, new_message_ids_and_values)

return dupes

memory.rb

((per_key_overhead + key_bytes + value_bytes) * keys_per_minute) * 60 * 24 +
((per_bucket_overhead + bucket_key_bytes) * buckets_per_minute * 60 * 24

((1 + 16 + 8) * 2,000,000) * 60 * 24 +
((64 + 14) * 20,000 * 60 * 24

= (72,000,000,000 + 2,246,400,000) = 69GB

naive.rb

message_ids.each do |message_id, owner_id|
  client.HSETNX(message_id, owner_id) # claim owner
  client.HGET(message_id) # determine winner
end

new_key.rb

(per_key_overhead + key_bytes + value_bytes) * keys_per_minute * 60 * 24
(64 + 16 + 8) * 2,000,000 * 60 * 24 = 230GB

owner_id.scala

ByteBuffer.allocate(2).putShort(partition.toShort).array ++ ByteBuffer.allocate(8).putLong(offset).array.slice(2, 8)

pipeline.rb

client.pipeline do |pipeline|
  message_ids.each do |message_id, owner_id|
    pipeline.HSETNX(message_id, owner_id) # claim owner
    pipeline.HGET(message_id) # determine winner
  end
end

uuid_conversion.scala

val uuid = java.util.UUID.fromString("ce059644-18a0-4f27-bc2b-c2a2d4d4e7bf")
val hi = uuid.getMostSignificantBits
val lo = uuid.getLeastSignificantBits
ByteBuffer.allocate(16).putLong(hi).putLong(lo).array
// => Array(-50, 5, -106, 68, 24, -96, 79, 39, -68, 43, -62, -94, -44, -44, -25, -65)