checksum.py

import zlib
import itertools

def get_signed_value(unsigned_value):
    bitsize = 32
    return unsigned_value if unsigned_value < (1 << bitsize-1) else unsigned_value - (1 << bitsize)

def create_checksum_string(bids, asks):
    bids = (f"{price}:{qty}" for price, qty, _, _ in bids)
    asks = (f"{price}:{qty}" for price, qty, _, _ in asks)
    merged = [item 
              for pair in itertools.zip_longest(bids, asks)
              for item in pair
              if item is not None]
    return ":".join(merged)

def check_checksum(item):
    bids = item['bids'][:25]
    asks = item['asks'][:25]
    string = create_checksum_string(bids, asks)
    checksum = item['checksum']
    unsigned = zlib.crc32(string.encode('utf8'))
    signed = get_signed_value(unsigned)
    return signed == checksum
  
def update_side(old_levels, new_levels, reverse):
    level_map = {k[0]: k for k in old_levels}

    for update in new_levels:
        price, qty, _, _ = update
        if qty == 0:
            level_map.pop(price, None)
        else:
            level_map[price] = update
    return [level_map[k] for k in sorted(level_map.keys(), reverse=reverse)]

  
# These examples taken from https://www.okex.com/docs/en/#futures_ws-checksum
even_example = {
    "instrument_id": "BTC-USD-170310",
    "asks": [[3366.8, 9, 10, 3],[ 3368,8,3,4 ]],
    "bids": [
        [3366.1, 7, 0, 3],[ 3366,6,3,4 ]
    ],
    "timestamp": "2018-12-04T09:38:36.300Z",
    "checksum": -1881014294
}
assert check_checksum(even_example)

odd_example = {
    "instrument_id": "BTC-USD-170310",
    "asks": [[3366.8, 9, 10, 3],[ 3368,8,3,4 ],[ 3372,8,3,4 ]],
    "bids": [[3366.1, 7, 0, 3]],
    "timestamp": "2018-12-04T09:38:36.300Z",
    "checksum": 831078360
}
assert check_checksum(odd_example)

increment_example = {
    "table": "futures/depth",
    "action": "update",
    "data": [{
        "instrument_id": "BTC-USD-SWAP",
        "asks": [],
        "bids": [
            [3983, 789, 0, 3]
        ],
        "timestamp": "2018-12-04T09:38:36.300Z",
        "checksum": -1200119424
    }]
}
# this fails
assert check_checksum(increment_example)

# actual first book update, this fails
message_200_levels = {'instrument_id': 'BTC-USD-190329',
 'asks': [[3782.85, 95, 0, 3],
  [3782.88, 53, 0, 1],
  [3782.9, 39, 0, 1],
  [3783.06, 92, 0, 4],
  [3783.09, 40, 0, 1],
  [3783.5, 17, 0, 1],
  [3783.89, 100, 0, 1],
  [3783.91, 3, 0, 1],
  [3783.95, 5, 0, 2],
  [3783.96, 131, 0, 3],
  [3783.97, 77, 0, 2],
  [3783.99, 28, 0, 2],
  [3784, 45, 0, 4],
  [3784.01, 100, 0, 1],
  [3784.02, 22, 0, 1],
  [3784.06, 2, 0, 1],
  [3784.1, 9, 0, 2],
  [3784.13, 33, 0, 1],
  [3784.18, 15, 0, 1],
  [3784.19, 7, 0, 1],
  [3784.22, 5, 0, 1],
  [3784.28, 11, 0, 1],
  [3784.37, 8, 0, 1],
  [3784.38, 8, 0, 1],
  [3784.4, 8, 0, 1],
  [3784.41, 11, 0, 1],
  [3784.46, 100, 0, 1],
  [3784.54, 17, 0, 2],
  [3784.55, 7, 0, 0],
  [3784.64, 3, 0, 2],
  [3784.69, 25, 0, 1],
  [3784.73, 8, 0, 1],
  [3784.74, 16, 0, 2],
  [3784.81, 8, 0, 1],
  [3784.87, 10, 0, 1],
  [3784.91, 7, 0, 1],
  [3784.96, 16, 0, 2],
  [3785, 1, 0, 1],
  [3785.06, 40, 0, 2],
  [3785.17, 100, 0, 1],
  [3785.21, 8, 0, 1],
  [3785.24, 8, 0, 1],
  [3785.32, 20, 0, 2],
  [3785.37, 12, 0, 1],
  [3785.38, 2, 0, 1],
  [3785.44, 7, 0, 1],
  [3785.45, 16, 0, 3],
  [3785.49, 15, 0, 2],
  [3785.52, 1, 0, 1],
  [3785.55, 3, 0, 1],
  [3785.56, 5, 0, 2],
  [3785.6, 8, 0, 0],
  [3785.61, 42, 0, 2],
  [3785.68, 3, 0, 1],
  [3785.75, 100, 0, 2],
  [3785.79, 5, 0, 1],
  [3785.86, 8, 0, 1],
  [3785.89, 8, 0, 1],
  [3785.93, 7, 0, 1],
  [3786, 1, 0, 1],
  [3786.01, 2, 0, 1],
  [3786.04, 9, 0, 1],
  [3786.08, 6, 0, 1],
  [3786.24, 8, 0, 1],
  [3786.28, 50, 0, 2],
  [3786.37, 10, 0, 1],
  [3786.5, 2, 0, 1],
  [3786.54, 43, 0, 1],
  [3786.56, 10, 0, 1],
  [3786.62, 40, 0, 1],
  [3786.79, 2, 0, 1],
  [3786.82, 13, 0, 1],
  [3786.83, 16, 0, 1],
  [3786.88, 12, 0, 1],
  [3786.89, 2, 0, 1],
  [3786.94, 16, 0, 2],
  [3786.95, 2, 0, 1],
  [3786.99, 30, 0, 1],
  [3787, 18, 0, 4],
  [3787.02, 1, 0, 1],
  [3787.09, 22, 0, 1],
  [3787.14, 30, 0, 2],
  [3787.16, 2, 0, 1],
  [3787.26, 17, 0, 1],
  [3787.29, 5, 0, 1],
  [3787.34, 2, 0, 1],
  [3787.37, 40, 0, 1],
  [3787.39, 1, 0, 1],
  [3787.46, 33, 0, 2],
  [3787.5, 2, 0, 1],
  [3787.51, 2, 0, 1],
  [3787.54, 16, 0, 1],
  [3787.58, 6, 0, 1],
  [3787.63, 74, 0, 1],
  [3787.68, 4, 0, 0],
  [3787.82, 9, 0, 1],
  [3787.83, 2, 0, 1],
  [3787.88, 89, 0, 1],
  [3787.92, 2, 0, 1],
  [3788, 4, 0, 3],
  [3788.02, 8, 0, 1],
  [3788.15, 11, 0, 2],
  [3788.24, 30, 0, 1],
  [3788.34, 1, 0, 1],
  [3788.39, 30, 0, 1],
  [3788.5, 2, 0, 1],
  [3788.54, 10, 0, 1],
  [3788.6, 13, 0, 1],
  [3788.65, 22, 0, 2],
  [3788.66, 2, 0, 1],
  [3788.69, 2, 0, 1],
  [3788.71, 1, 0, 1],
  [3788.73, 2, 0, 1],
  [3788.78, 35, 0, 3],
  [3788.8, 1, 0, 1],
  [3789, 2, 0, 1],
  [3789.07, 5, 0, 1],
  [3789.09, 1, 0, 1],
  [3789.16, 1, 0, 1],
  [3789.35, 8, 0, 1],
  [3789.36, 6, 0, 1],
  [3789.5, 2, 0, 1],
  [3789.84, 2, 0, 1],
  [3790, 39, 0, 2],
  [3790.08, 10, 0, 1],
  [3790.48, 1, 0, 1],
  [3790.5, 2, 0, 1],
  [3790.58, 27, 0, 1],
  [3790.95, 60, 0, 2],
  [3790.98, 50, 0, 1],
  [3791, 402, 0, 2],
  [3791.1, 50, 0, 1],
  [3791.16, 40, 0, 1],
  [3791.18, 30, 0, 1],
  [3791.42, 30, 0, 1],
  [3791.45, 12, 0, 1],
  [3791.47, 1, 0, 1],
  [3791.5, 2, 0, 1],
  [3791.76, 15, 0, 1],
  [3791.77, 40, 0, 1],
  [3791.8, 20, 0, 1],
  [3791.83, 20, 0, 1],
  [3792, 2, 0, 1],
  [3792.07, 24, 0, 1],
  [3792.5, 2, 0, 1],
  [3792.64, 24, 0, 1],
  [3792.69, 30, 0, 1],
  [3792.72, 1, 0, 1],
  [3792.84, 40, 0, 1],
  [3792.9, 37, 0, 1],
  [3792.95, 7, 0, 1],
  [3793, 2, 0, 1],
  [3793.06, 40, 0, 1],
  [3793.17, 1, 0, 1],
  [3793.21, 33, 0, 2],
  [3793.33, 40, 0, 1],
  [3793.5, 12, 0, 2],
  [3793.51, 40, 0, 1],
  [3793.53, 260, 0, 1],
  [3793.76, 50, 0, 1],
  [3793.82, 80, 0, 1],
  [3794, 26, 0, 2],
  [3794.1, 20, 0, 1],
  [3794.23, 20, 0, 1],
  [3794.5, 2, 0, 1],
  [3794.57, 24, 0, 1],
  [3794.74, 80, 0, 1],
  [3795, 123, 0, 6],
  [3795.24, 49, 0, 1],
  [3795.3, 44, 0, 1],
  [3795.5, 2, 0, 1],
  [3795.75, 100, 0, 1],
  [3796, 12, 0, 2],
  [3796.49, 13, 0, 1],
  [3796.5, 2, 0, 1],
  [3796.58, 2, 0, 1],
  [3796.7, 20, 0, 1],
  [3796.92, 150, 0, 1],
  [3797, 109, 0, 4],
  [3797.26, 100, 0, 1],
  [3797.3, 270, 0, 1],
  [3797.31, 100, 0, 1],
  [3797.33, 18, 0, 1],
  [3797.35, 1, 0, 1],
  [3797.5, 2, 0, 1],
  [3798, 30, 0, 3],
  [3798.1, 11, 0, 1],
  [3798.5, 2, 0, 1],
  [3798.83, 100, 0, 1],
  [3799, 108, 0, 2],
  [3799.1, 37, 0, 1],
  [3799.12, 30, 0, 2],
  [3799.5, 2, 0, 1],
  [3799.54, 74, 0, 1],
  [3800, 1353, 0, 18],
  [3800.1, 50, 0, 1],
  [3800.34, 1, 0, 1],
  [3800.5, 2, 0, 1],
  [3800.76, 15, 0, 1],
  [3800.97, 8, 0, 2]],
 'bids': [[3782.52, 2, 0, 1],
  [3782.45, 1, 0, 1],
  [3782.2, 41, 0, 2],
  [3782.18, 5, 0, 1],
  [3782.12, 3, 0, 1],
  [3782.11, 1, 0, 1],
  [3782, 1, 0, 1],
  [3781.97, 1, 0, 1],
  [3781.83, 1, 0, 1],
  [3781.66, 1, 0, 1],
  [3781.64, 7, 0, 1],
  [3781.63, 7, 0, 1],
  [3781.36, 14, 0, 1],
  [3781.35, 2, 0, 1],
  [3781.29, 50, 0, 1],
  [3781.27, 8, 0, 1],
  [3781.19, 15, 0, 1],
  [3781.16, 5, 0, 1],
  [3781.12, 1, 0, 1],
  [3781.05, 10, 0, 1],
  [3781.03, 1, 0, 1],
  [3781.02, 2, 0, 1],
  [3781.01, 13, 0, 1],
  [3781, 11, 0, 3],
  [3780.95, 1, 0, 1],
  [3780.94, 35, 0, 1],
  [3780.86, 9, 0, 1],
  [3780.73, 3, 0, 2],
  [3780.7, 1, 0, 1],
  [3780.68, 5, 0, 1],
  [3780.67, 100, 0, 2],
  [3780.55, 100, 0, 1],
  [3780.53, 10, 0, 1],
  [3780.5, 2, 0, 1],
  [3780.29, 27, 0, 1],
  [3780.18, 29, 0, 1],
  [3780.15, 1, 0, 1],
  [3780.05, 23, 0, 1],
  [3780, 3, 0, 2],
  [3779.85, 2, 0, 1],
  [3779.8, 7, 0, 1],
  [3779.55, 10, 0, 1],
  [3779.5, 2, 0, 1],
  [3779.49, 11, 0, 1],
  [3779.48, 33, 0, 1],
  [3779.36, 20, 0, 2],
  [3779.24, 2, 0, 1],
  [3779.22, 2, 0, 1],
  [3779.18, 20, 0, 1],
  [3779.17, 1, 0, 1],
  [3779.16, 31, 0, 1],
  [3779.14, 24, 0, 1],
  [3779.01, 2, 0, 1],
  [3779, 15, 0, 3],
  [3778.9, 5, 0, 1],
  [3778.79, 10, 0, 1],
  [3778.66, 2, 0, 1],
  [3778.6, 8, 0, 1],
  [3778.58, 28, 0, 2],
  [3778.52, 43, 0, 1],
  [3778.5, 2, 0, 1],
  [3778.42, 30, 0, 2],
  [3778.31, 24, 0, 1],
  [3778.27, 8, 0, 2],
  [3778.25, 18, 0, 1],
  [3778.22, 9, 0, 1],
  [3778.16, 90, 0, 4],
  [3778.14, 40, 0, 1],
  [3778, 3, 0, 2],
  [3777.81, 20, 0, 1],
  [3777.8, 34, 0, 1],
  [3777.55, 1, 0, 1],
  [3777.53, 22, 0, 1],
  [3777.5, 2, 0, 1],
  [3777.45, 2, 0, 1],
  [3777.35, 1, 0, 1],
  [3777.06, 30, 0, 1],
  [3777.03, 40, 0, 1],
  [3777, 23, 0, 3],
  [3776.95, 1, 0, 1],
  [3776.93, 30, 0, 1],
  [3776.82, 89, 0, 1],
  [3776.66, 20, 0, 1],
  [3776.58, 74, 0, 1],
  [3776.5, 2, 0, 1],
  [3776.45, 1, 0, 1],
  [3776.39, 50, 0, 1],
  [3776.36, 6, 0, 1],
  [3776.09, 37, 0, 2],
  [3776, 303, 0, 3],
  [3775.94, 4, 0, 1],
  [3775.84, 8, 0, 1],
  [3775.69, 30, 0, 1],
  [3775.64, 1, 0, 1],
  [3775.6, 30, 0, 1],
  [3775.56, 40, 0, 1],
  [3775.5, 2, 0, 1],
  [3775.42, 30, 0, 1],
  [3775.4, 56, 0, 1],
  [3775.23, 15, 0, 1],
  [3775.16, 1, 0, 1],
  [3775.14, 40, 0, 1],
  [3775, 60, 0, 4],
  [3774.96, 16, 0, 1],
  [3774.7, 16, 0, 1],
  [3774.5, 2, 0, 1],
  [3774.44, 31, 0, 2],
  [3774.34, 1, 0, 1],
  [3774.09, 30, 0, 1],
  [3774, 2, 0, 1],
  [3773.9, 30, 0, 1],
  [3773.88, 20, 0, 1],
  [3773.72, 38, 0, 1],
  [3773.69, 2, 0, 1],
  [3773.57, 17, 0, 2],
  [3773.52, 40, 0, 1],
  [3773.5, 2, 0, 1],
  [3773.31, 7, 0, 1],
  [3773.02, 30, 0, 1],
  [3773, 208, 0, 4],
  [3772.69, 758, 0, 1],
  [3772.54, 24, 0, 1],
  [3772.5, 2, 0, 1],
  [3772.38, 30, 0, 1],
  [3772.16, 1, 0, 1],
  [3772, 58, 0, 4],
  [3771.91, 20, 0, 1],
  [3771.7, 277, 0, 1],
  [3771.61, 27, 0, 1],
  [3771.59, 24, 0, 1],
  [3771.5, 2, 0, 1],
  [3771.35, 40, 0, 1],
  [3771.34, 40, 0, 1],
  [3771.3, 11, 0, 1],
  [3771.27, 2, 0, 1],
  [3771.21, 24, 0, 1],
  [3771.05, 30, 0, 1],
  [3771, 24, 0, 5],
  [3770.95, 17, 0, 1],
  [3770.86, 7, 0, 1],
  [3770.85, 20, 0, 1],
  [3770.83, 40, 0, 2],
  [3770.81, 1, 0, 1],
  [3770.72, 150, 0, 1],
  [3770.67, 80, 0, 1],
  [3770.5, 2, 0, 1],
  [3770.37, 10, 0, 1],
  [3770.31, 20, 0, 1],
  [3770.13, 6, 0, 1],
  [3770.1, 10, 0, 1],
  [3770.03, 11, 0, 1],
  [3770, 1724, 0, 19],
  [3769.91, 24, 0, 1],
  [3769.86, 20, 0, 1],
  [3769.85, 50, 0, 1],
  [3769.5, 2, 0, 1],
  [3769.43, 30, 0, 1],
  [3769.21, 80, 0, 1],
  [3769.02, 49, 0, 1],
  [3769, 78, 0, 4],
  [3768.8, 6, 0, 1],
  [3768.54, 24, 0, 1],
  [3768.5, 2, 0, 1],
  [3768.38, 3, 0, 1],
  [3768.34, 100, 0, 1],
  [3768.23, 100, 0, 1],
  [3768.02, 230, 0, 1],
  [3768, 10, 0, 3],
  [3767.94, 11, 0, 1],
  [3767.5, 2, 0, 1],
  [3767.47, 50, 0, 1],
  [3767.37, 1, 0, 1],
  [3767.31, 50, 0, 1],
  [3767, 265, 0, 2],
  [3766.82, 100, 0, 1],
  [3766.73, 100, 0, 1],
  [3766.5, 2, 0, 1],
  [3766.24, 74, 0, 1],
  [3766.01, 5, 0, 1],
  [3766, 204, 0, 6],
  [3765.5, 2, 0, 1],
  [3765.47, 50, 0, 1],
  [3765.24, 15, 0, 1],
  [3765.21, 100, 0, 1],
  [3765, 252, 0, 12],
  [3764.8, 75, 0, 1],
  [3764.59, 10, 0, 1],
  [3764.5, 2, 0, 1],
  [3764.34, 1, 0, 1],
  [3764, 202, 0, 2],
  [3763.85, 1500, 0, 1],
  [3763.76, 11, 0, 2],
  [3763.53, 222, 0, 1],
  [3763.5, 2, 0, 1],
  [3763, 2, 0, 1],
  [3762.5, 2, 0, 1],
  [3762, 5, 0, 2],
  [3761.5, 2, 0, 1],
  [3761, 106, 0, 5],
  [3760.9, 11, 0, 1]],
 'timestamp': '2019-03-02T17:51:54.418Z',
 'checksum': 1276686467}
assert check_checksum(message_200_levels)