redis test

redis_test.py

#!/usr/bin/env python

'''
@author David Stuebe <dstuebe@asascience.com>

To Run:
@todo - fill this in!
'''


import redis
import random
from uuid import uuid4
import time

def persist(data):
    with open('/tmp/storage','a') as f:
        for datum in data:
            f.write('%s ' % datum)
        f.write('\n-----------\n')
        

if __name__ == "__main__":
    #-------------- This is a process --------------#
    # 

    rserver = redis.StrictRedis(host='localhost', port=6379, db=0)
    
    COMPARESET = 'compareset'



    dsets = 1 # number of datasets

    dset_max_size = 15 # Number of elements to put in the list before aggregation and chop


    data_points = dset_max_size * 1000

    for i in xrange(data_points):


        new_data = str(uuid4())
        dataset_name = 'dataset_%d' % random.uniform(1, dsets)

        new_vals = rserver.sadd(COMPARESET, new_data)
        assert new_vals == 1, 'That UUID already existed!'


        current_length = rserver.lpush(dataset_name, new_data)

        salt = random.normalvariate(mu=0,sigma=1)


        packet_block = []
        if current_length % dset_max_size == 0:
            with rserver.pipeline() as pipe:
                random_queue = str(uuid4())[:8]
                while True:
                    try:
                        pipe.watch(dataset_name)
                        if salt > 2:
                            time.sleep(0.2)
                        pipe.multi()
                        for i in xrange(dset_max_size):
                            pipe.rpoplpush(dataset_name, random_queue)
                        packet_block = pipe.execute()
                        

                        
                        break
                        
                    except redis.WatchError:
                        print 'Watch Error'
                        
                        continue
            rserver.delete(random_queue)
            for datum in packet_block:
                rserver.srem(COMPARESET,datum)