redis_massive_insertion_1m_floats.py

# Massive insertion of 1 million float values using Josiah Carlson ROM
# (C) By Athanassios I. Hatzis
# 14 Sept 2018
# Written to explain issue: https://github.com/RedisLabsModules/RediSearch/issues/493
#
from redis import Redis
import numpy as np
import time
import sys

from rom.util import session
from rom.model import Model
from rom.columns import Float

class FF(Model):
    _conn = Redis(host='localhost', port=6379, db=9)
    val = Float(unique=True, index=True)

size = 1000000
interval = (-5000, 5000)
fltarr = (interval[1]-interval[0])*np.random.sample(size)+interval[0]
print(f'Size: {fltarr.size}')
print(f'Data Type: {fltarr.dtype}')
print(f'Memory Size: {fltarr.nbytes}')
sys.getsizeof(fltarr)

################################ Start Benchmark Test ####################################
t1_start = time.perf_counter()
t2_start = time.process_time()

for n in range(size): FF(val=fltarr[n]).save()

t1_stop = time.perf_counter()
t2_stop = time.process_time()

print(f"Elapsed time for {size} floats    : {int(round((t1_stop-t1_start)))} [sec]")
print(f"CPU process time for {size} floats: {int(round((t2_stop-t2_start)))} [sec]")

################################ End Benchmark Test ####################################

# Elapsed time for 1000000 floats    : 562 [sec]
# CPU process time for 1000000 floats: 449 [sec]

# used_memory_human :  313 MB
# used_memory_dataset: 266 MB

# Save RDB policy: after 600 sec (10 min) if at least 10 keys changed

# OS Environment: Linux Ubuntu x64
# Processor: Intel(R) Core(TM) i3 CPU 540  @ 3.07GHz
# Memory 16GB