Measure notifications in redis with sorted set and Json data or multiple data structures (sroted set, plain data and hash structures): http://stackoverflow.com/questions/15806987/best-approach-for-storing-and-reading-notifications-in-redis-json-or-multiple-d

benchmark.py

import redis
import time
import json

data = "my text notification tests{0}"
notification_type = "achievement{0}"
user_key = "notifications:user:{0}"
notification_counter = "notifications:counter:{0}"
notification_hash = "notification:{0}:{1}"

# We are going to use the same connection, for the measure of big data store
# and retrieval I think is not so important
r = redis.StrictRedis(host='localhost', port=6379, db=0)


def store_with_json(users, notifications):
    start = time.clock()

    for i in range(users):
        key = user_key.format(i)
        for j in range(notifications):
            store_data = {
                'notification': data.format(j),
                'type': notification_type.format(j),
            }
            json_data = json.dumps(store_data)
            r.zadd(key, time.time(), json_data)

    return time.clock() - start


def retrieve_with_json(users, notifications):
    start = time.clock()

    for i in range(users):
        key = user_key.format(i)
        json_data_list = r.zrange(key, 0, users)
        for json_data in json_data_list:
            json.loads(json_data)

    return time.clock() - start


def store_without_json(users, notifications):
    start = time.clock()

    for i in range(users):
        key = user_key.format(i)
        for j in range(notifications):
            notification_id = r.incr("notifications:counter:{0}".format(i))
            hash_key = notification_hash.format(i, notification_id)
            store_data = {
                'notification': data.format(j),
                'type': notification_type.format(j),
            }

            # We are going to use a pipe
            p = r.pipeline()
            p.zadd(key, time.time(), hash_key)
            p.hmset(hash_key, store_data)
            p.execute()

    return time.clock() - start


def retrieve_without_json(users, notifications):
    start = time.clock()

    for i in range(users):
        key = user_key.format(i)
        hash_key_list = r.zrange(key, 0, users)
        for hash_key in hash_key_list:
            r.hgetall(hash_key)

    return time.clock() - start


def print_redis_info(write_t, read_t):
    print("Write time: {0}".format(write_t))
    print("Read time: {0}".format(read_t))
    print("dbsize (number of keys): {0}".format(r.dbsize()))
    info = r.info()
    print ("Memory: {0}".format(info['used_memory_human']))


def main():
    for i in range(100, 1001, 100):
        users = i
        notifications = i * 2

        print('=' * 80)
        print("Users: {0}".format(users))
        print("Notifications per user: {0}".format(notifications))
        print('-' * 20)
        print("#### With Json in ZSet data structure ####")
        r.flushdb()
        write_time = store_with_json(users, notifications)
        read_time = retrieve_with_json(users, notifications)
        print_redis_info(write_time, read_time)
        print("#### With plain data in Zset and hash data structures ####")
        r.flushdb()
        write_time = store_without_json(users, notifications)
        read_time = retrieve_without_json(users, notifications)
        print_redis_info(write_time, read_time)

if __name__ == "__main__":
    main()

results.txt

================================================================================
Users: 100
Notifications per user: 200
--------------------
#### With Json in Set structure ####
Write time: 2.2
Read time: 0.21
dbsize (number of keys): 100
Memory: 4.87M
#### With set and hash data structures ####
Write time: 10.2
Read time: 2.09
dbsize (number of keys): 20200
Memory: 6.97M
================================================================================
Users: 200
Notifications per user: 400
--------------------
#### With Json in Set structure ####
Write time: 7.64
Read time: 0.82
dbsize (number of keys): 200
Memory: 16.82M
#### With set and hash data structures ####
Write time: 29.69
Read time: 4.83
dbsize (number of keys): 80400
Memory: 25.20M
================================================================================
Users: 300
Notifications per user: 600
--------------------
#### With Json in Set structure ####
Write time: 22.26
Read time: 1.7
dbsize (number of keys): 300
Memory: 38.44M
#### With set and hash data structures ####
Write time: 71.17
Read time: 12.65
dbsize (number of keys): 180600
Memory: 56.95M
================================================================================
Users: 400
Notifications per user: 800
--------------------
#### With Json in Set structure ####
Write time: 31.14
Read time: 3.02
dbsize (number of keys): 400
Memory: 64.46M
#### With set and hash data structures ####
Write time: 122.27
Read time: 22.22
dbsize (number of keys): 320800
Memory: 97.84M
================================================================================
Users: 500
Notifications per user: 1000
--------------------
#### With Json in Set structure ####
Write time: 73.13
Read time: 4.47
dbsize (number of keys): 500
Memory: 99.20M
#### With set and hash data structures ####
Write time: 201.4
Read time: 34.43
dbsize (number of keys): 501000
Memory: 149.07M
================================================================================
Users: 600
Notifications per user: 1200
--------------------
#### With Json in Set structure ####
Write time: 93.0
Read time: 6.65
dbsize (number of keys): 600
Memory: 150.60M
#### With set and hash data structures ####
Write time: 367.72
Read time: 40.2
dbsize (number of keys): 721200
Memory: 224.17M
================================================================================
Users: 700
Notifications per user: 1400
--------------------
#### With Json in Set structure ####
Write time: 145.79
Read time: 8.54
dbsize (number of keys): 700
Memory: 196.46M
#### With set and hash data structures ####
Write time: 439.5
Read time: 79.8
dbsize (number of keys): 981400
Memory: 294.18M
================================================================================
Users: 800
Notifications per user: 1600
--------------------
#### With Json in Set structure ####
Write time: 198.52
Read time: 11.72
dbsize (number of keys): 800
Memory: 254.38M
#### With set and hash data structures ####
Write time: 489.91
Read time: 90.38
dbsize (number of keys): 1281600
Memory: 387.78M
================================================================================
Users: 900
Notifications per user: 1800
--------------------
#### With Json in Set structure ####
Write time: 181.5
Read time: 14.05
dbsize (number of keys): 900
Memory: 319.96M
#### With set and hash data structures ####
Write time: 631.21
Read time: 115.5
dbsize (number of keys): 1621800
Memory: 484.39M
================================================================================
Users: 1000
Notifications per user: 2000
--------------------
#### With Json in Set structure ####
Write time: 226.07
Read time: 18.08
dbsize (number of keys): 1000
Memory: 392.95M
#### With set and hash data structures ####
Write time: 684.69
Read time: 121.1
dbsize (number of keys): 2002000
Memory: 592.02M