Carbonara compressiong using double delta + RLE on timestamps and LZ4

bench.py

import math
import time
import random
import pandas
import datetime
from gnocchi import carbonara

points = 14400
sampling = 5
compress_times = 10

now = datetime.datetime(2015, 04, 03, 23, 11)


for title, values in [
        ("Simple contiguous range", range(points)),
        ("All 0", [float(0)] * points),
        ("All 1", [float(1)] * points),
        ("0 and 1", [0, 1] * (points / 2)),
        ("1 and 0", [1, 0] * (points / 2)),
        ("1 and 0 random", [random.randint(0, 1) for x in range(points)]),
        ("Small number random pos/neg", [random.randint(-100000, 10000) for x in range(points)]),
        ("Small number random pos", [random.randint(0, 20000) for x in range(points)]),
        ("Small number random neg", [random.randint(-20000, 0) for x in range(points)]),
        ("Sin(x)", map(math.sin, range(points))),
        ("random ", [random.random() for x in range(points)]),
]:
    pts = pandas.Series(values,
                        [now + datetime.timedelta(seconds=i*sampling)
                         for i in range(points)])
    ts = carbonara.AggregatedTimeSerie(ts=pts, sampling=sampling)

    t0 = time.time()
    for i in range(compress_times):
        s = ts.serialize()
    t1 = time.time()
    print(title)
    print(" Bytes per point: %.2f" % (len(s) / float(points)))
    print(" Compression time: %.2f seconds" % ((t1 - t0) / compress_times))


# Gnocchi 1.3
# Simple contiguous range
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# All 0
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# All 1
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# 0 and 1
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# 1 and 0
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# 1 and 0 random
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# Small number random pos/neg
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# Small number random pos
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# Small number random neg
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# Sin(x)
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds
# random
#  Bytes per point: 18.00
#  Compression time: 0.02 seconds

# Gnocchi 2.0-dev: double-delta RLE + LZ4 on all data
# Simple contiguous range
#  Bytes per point: 3.00
#  Compression time: 0.04 seconds
# All 0
#  Bytes per point: 0.05
#  Compression time: 0.04 seconds
# All 1
#  Bytes per point: 0.05
#  Compression time: 0.07 seconds
# 0 and 1
#  Bytes per point: 0.02
#  Compression time: 0.06 seconds
# 1 and 0
# Bytes per point: 0.02
#  Compression time: 0.04 seconds
# 1 and 0 random
#  Bytes per point: 0.62
#  Compression time: 0.04 seconds
# Small number random pos/neg
#  Bytes per point: 3.73
#  Compression time: 0.04 seconds
# Small number random pos
#  Bytes per point: 2.98
#  Compression time: 0.04 seconds
# Small number random neg
#  Bytes per point: 2.98
#  Compression time: 0.04 seconds
# Sin(x)
#  Bytes per point: 8.98
#  Compression time: 0.04 seconds
# random
#  Bytes per point: 9.02
#  Compression time: 0.04 seconds

# Gnocchi 2.0-dev: double-delta RLE + LZ4 timestamps + XOR encoding on values
# Simple contiguous range
#  Bytes per point: 1.61
#  Compression time: 0.09 seconds
# All 0
#  Bytes per point: 0.14
#  Compression time: 0.05 seconds
# All 1
#  Bytes per point: 0.14
#  Compression time: 0.05 seconds
# 0 and 1
#  Bytes per point: 1.51
#  Compression time: 0.09 seconds
# 1 and 0
#  Bytes per point: 1.51
#  Compression time: 0.09 seconds
# 1 and 0 random
#  Bytes per point: 0.83
#  Compression time: 0.07 seconds
# Small number random pos/neg
#  Bytes per point: 3.20
#  Compression time: 0.09 seconds
# Small number random pos
#  Bytes per point: 2.51
#  Compression time: 0.09 seconds
# Small number random neg
#  Bytes per point: 2.51
# Compression time: 0.08 seconds
# Sin(x)
#  Bytes per point: 8.41
#  Compression time: 0.11 seconds
# random
#  Bytes per point: 7.26
#  Compression time: 0.11 seconds