Aimq messaging and storage

aimq_msg_store.py

"""

AIMQ messaging:

Start with a dictionary
Convert to raw Avro
Send raw Avro over the wire
Convert to dict on the other end

Use pytables for data storage - create a group for each message type - nested by message/path
- main table for: primitive types except strings and bytes
- vlarrays for: bytes, string, map, array, union, enum, record
   https://github.com/PyTables/PyTables/blob/develop/examples/vlarray2.py
- earrays for: numpy logical types and fixed
- schema is stored as an attribute of the group
"""


import tables
import time
from skimage.data import moon

NIMAGES = 500

arr = moon()

t0 = time.time()
fileh = tables.open_file('/tmp/test1.h5', 'w')
atom = tables.Atom.from_dtype(arr.dtype)

array_c = fileh.create_earray(fileh.root, 'array_c', atom, (0, arr.size))
for i in range(NIMAGES):
    array_c.append(arr.reshape(1, arr.size))
print(array_c[0].reshape(arr.shape), array_c[0].size)
fileh.close()
print('EArray', time.time() - t0)


t0 = time.time()
fileh = tables.open_file('/tmp/test2.h5', 'w')
atom = tables.Atom.from_dtype(arr.dtype)

for i in range(NIMAGES):
    ds = fileh.create_array(fileh.root, 'array%s' % i, arr)
print(fileh.root.array1[:], fileh.root.array1[:].size)
fileh.close()
print('Arrays', time.time() - t0)