Skip to content

Instantly share code, notes, and snippets.

@prdn
Last active February 25, 2022 04:52
Show Gist options
  • Save prdn/b9c9c24fb1bb38604dcd1e94ee89dd9e to your computer and use it in GitHub Desktop.
Save prdn/b9c9c24fb1bb38604dcd1e94ee89dd9e to your computer and use it in GitHub Desktop.
/*USAGE:
npm install ws lodash async moment crc-32
mkdir logs
node bfx_test_book.js BTCUSD
*/
const WS = require('ws')
const _ = require('lodash')
const async = require('async')
const fs = require('fs')
const moment = require('moment')
const CRC = require('crc-32')
const pair = process.argv[2]
const conf = {
wshost: "wss://api.bitfinex.com/ws/2"
}
const logfile = __dirname + '/logs/ws-book-raw.log'
const BOOK = {}
let connected = false
let connecting = false
let cli
let seq = null
function connect() {
if (connecting || connected) return
connecting = true
cli = new WS(conf.wshost, { /*rejectUnauthorized: false*/ })
cli.on('open', function open() {
console.log('WS open')
connecting = false
connected = true
BOOK.bids = {}
BOOK.asks = {}
BOOK.psnap = {}
BOOK.mcnt = 0
cli.send(JSON.stringify({ event: 'conf', flags: 65536 + 131072 }))
cli.send(JSON.stringify({ event: "subscribe", channel: "book", pair: pair, prec: "R0", len: 25 }))
})
cli.on('close', function open() {
seq = null
console.log('WS close')
connecting = false
connected = false
})
cli.on('message', function(msg) {
msg = JSON.parse(msg)
if (msg.event) return
if (msg[1] === 'hb') {
seq = +msg[2]
return
} else if (msg[1] === 'cs') {
seq = +msg[3]
const checksum = msg[2]
const csdata = []
const bids_keys = BOOK.psnap['bids']
const asks_keys = BOOK.psnap['asks']
for (let i = 0; i < 25; i++) {
if (bids_keys[i]) {
const eid = bids_keys[i]
const pp = BOOK.bids[eid]
csdata.push(pp.id, pp.amount)
}
if (asks_keys[i]) {
const eid = asks_keys[i]
const pp = BOOK.asks[eid]
csdata.push(pp.id, -pp.amount)
}
}
const cs_str = csdata.join(':')
console.log(cs_str)
const cs_calc = CRC.str(cs_str)
fs.appendFileSync(logfile, "[" + moment().format("YYYY-MM-DDTHH:mm:ss.SSS") + "] " + pair + " | " + JSON.stringify(["cs_string=" + cs_str, "cs_calc=" + cs_calc, "server_checksum=" + checksum]) + "\n")
if (cs_calc !== checksum) {
console.error("CHECKSUM_FAILED", checksum, cs_calc)
process.exit(-1)
}
return
}
fs.appendFileSync(logfile, "[" + moment().format("YYYY-MM-DDTHH:mm:ss.SSS") + "] " + pair + " | " + JSON.stringify(msg) + "\n")
if (BOOK.mcnt === 0) {
_.each(msg[1], function(pp) {
pp = { id: pp[0], price: pp[1], amount: pp[2] }
const side = pp.amount >= 0 ? 'bids' : 'asks'
pp.amount = Math.abs(pp.amount)
BOOK[side][pp.id] = pp
})
} else {
const cseq = +msg[2]
msg = msg[1]
if (!seq) {
seq = cseq - 1
}
if (cseq - seq !== 1) {
console.error('OUT OF SEQUENCE', seq, cseq)
process.exit()
}
seq = cseq
const pp = { id: msg[0], price: msg[1], amount: msg[2], ix: msg[3] }
if (!pp.price) {
let found = true
if (pp.amount > 0) {
if (BOOK['bids'][pp.id]) {
delete BOOK['bids'][pp.id]
} else {
found = false
}
} else if (pp.amount < 0) {
if (BOOK['asks'][pp.id]) {
delete BOOK['asks'][pp.id]
} else {
found = false
}
}
if (!found) {
fs.appendFileSync(logfile, "[" + moment().format() + "] " + pair + " | " + JSON.stringify(pp) + " BOOK_RAW delete fail found\n")
}
} else {
const side = pp.amount >= 0 ? 'bids' : 'asks'
pp.amount = Math.abs(pp.amount)
BOOK[side][pp.id] = pp
}
}
_.each(['bids', 'asks'], function(side) {
let sbook = BOOK[side]
let bentries = Object.keys(sbook)
let prices = bentries.sort(function(a, b) {
if (+sbook[a].price === +sbook[b].price) {
return a - b
}
if (side === 'bids') {
return +sbook[a].price >= +sbook[b].price ? -1 : 1
} else {
return +sbook[a].price <= +sbook[b].price ? -1 : 1
}
})
BOOK.psnap[side] = bentries
})
BOOK.mcnt++
})
}
setInterval(function() {
if (connected) return
connect()
}, 3500)
function saveBook() {
const now = moment.utc().format('YYYYMMDDHHmmss')
fs.writeFileSync(__dirname + "/logs/tmp-ws-book-raw-" + pair + '-' + now + '.log', JSON.stringify({ bids: BOOK.bids, asks: BOOK.asks}))
}
setInterval(function() {
saveBook()
}, 30000)
@Nono242
Copy link

Nono242 commented Apr 17, 2018

Hello Paolo,
There is a bug on the Bitfinex side. If one amount in the bid or ask books is below 1e-07 (like 0.0000008) the value reported in the CRC is wrong.
Regards,
Arnaud

@Nono242
Copy link

Nono242 commented Apr 21, 2018

Here is an example taken a few minutes ago:
Pair BTCUSD: Checksum not valid, Received from Bitfinex: 2664795953 <> Localy computed: 1550466235
The bid book

[[8743.9658738, 11101458585, 0.2], [8743.9, 11101455753, 0.83284087], [8743.9, 11101455842, 0.25788734], [8743.9, 11101455935, 2.3], [8743.9, 11101456363, 2.5], [8743.9, 11101456459, 0.28600039], [8743.9, 11101458181, 0.1], [8743.9, 11101458638, 2], [8743.9, 11101459498, 0.5], [8743.9, 11101459499, 0.5], [8743.9, 11101459500, 0.5], [8743.9, 11101459501, 0.5], [8743.9, 11101459504, 0.5], [8743.9, 11101459509, 0.2], [8743.9, 11101459523, 0.5], [8743.9, 11101459524, 0.5], [8743.9, 11101459525, 0.4], [8743.5, 11101456853, 0.5], [8743.4, 11101459512, 0.2], [8743.2, 11101455149, 0.86755981], [8742.9, 11101458903, 0.2], [8742.1, 11101459552, 7.92], [8742, 11101449969, 2.4], [8742, 11101453397, 0.335], [8741.99955681, 11101453620, 0.002]]

The ask book

[[8744, 11101457496, -7e-08], [8746.1, 11101457382, -0.022], [8746.7, 11101443091, -0.02], [8748.7, 11101445552, -0.02], [8748.9, 11101438733, -2], [8749.5, 11101438580, -0.02], [8750.3, 11101458905, -1.14], [8751.5, 11101456054, -0.05718469], [8752, 11101454021, -0.4], [8752.6, 11101458412, -0.5], [8752.7, 11101456245, -0.02], [8752.7, 11101458863, -0.02], [8752.9, 11101444398, -0.5], [8752.9, 11101459296, -0.5], [8753.79745876, 11101458511, -1.1409], [8754.6, 11101457822, -1], [8754.9, 11101439758, -0.05734873], [8755, 11101401849, -0.81803941], [8755, 11101420500, -0.5], [8756.2, 11101459569, -0.5014], [8756.3, 11101437200, -8], [8757.6, 11101457636, -0.23106522], [8758, 11101402918, -0.01221818], [8758.1, 11101433925, -1.01128782], [8758.4, 11101387597, -1]]

The list of data to be taken for CRC

[11101458585, 0.2, 11101457496, -7e-08, 11101455753, 0.83284087, 11101457382, -0.022, 11101455842, 0.25788734, 11101443091, -0.02, 11101455935, 2.3, 11101445552, -0.02, 11101456363, 2.5, 11101438733, -2, 11101456459, 0.28600039, 11101438580, -0.02, 11101458181, 0.1, 11101458905, -1.14, 11101458638, 2, 11101456054, -0.05718469, 11101459498, 0.5, 11101454021, -0.4, 11101459499, 0.5, 11101458412, -0.5, 11101459500, 0.5, 11101456245, -0.02, 11101459501, 0.5, 11101458863, -0.02, 11101459504, 0.5, 11101444398, -0.5, 11101459509, 0.2, 11101459296, -0.5, 11101459523, 0.5, 11101458511, -1.1409, 11101459524, 0.5, 11101457822, -1, 11101459525, 0.4, 11101439758, -0.05734873, 11101456853, 0.5, 11101401849, -0.81803941, 11101459512, 0.2, 11101420500, -0.5, 11101455149, 0.86755981, 11101459569, -0.5014, 11101458903, 0.2, 11101437200, -8, 11101459552, 7.92, 11101457636, -0.23106522, 11101449969, 2.4, 11101402918, -0.01221818, 11101453397, 0.335, 11101433925, -1.01128782, 11101453620, 0.002, 11101387597, -1]

The string of the data

11101458585:0.2:11101457496:-0.00000007:11101455753:0.83284087:11101457382:-0.022:11101455842:0.25788734:11101443091:-0.02:11101455935:2.3:11101445552:-0.02:11101456363:2.5:11101438733:-2:11101456459:0.28600039:11101438580:-0.02:11101458181:0.1:11101458905:-1.14:11101458638:2:11101456054:-0.05718469:11101459498:0.5:11101454021:-0.4:11101459499:0.5:11101458412:-0.5:11101459500:0.5:11101456245:-0.02:11101459501:0.5:11101458863:-0.02:11101459504:0.5:11101444398:-0.5:11101459509:0.2:11101459296:-0.5:11101459523:0.5:11101458511:-1.1409:11101459524:0.5:11101457822:-1:11101459525:0.4:11101439758:-0.05734873:11101456853:0.5:11101401849:-0.81803941:11101459512:0.2:11101420500:-0.5:11101455149:0.86755981:11101459569:-0.5014:11101458903:0.2:11101437200:-8:11101459552:7.92:11101457636:-0.23106522:11101449969:2.4:11101402918:-0.01221818:11101453397:0.335:11101433925:-1.01128782:11101453620:0.002:11101387597:-1

The binary of the string

b'11101458585:0.2:11101457496:-0.00000007:11101455753:0.83284087:11101457382:-0.022:11101455842:0.25788734:11101443091:-0.02:11101455935:2.3:11101445552:-0.02:11101456363:2.5:11101438733:-2:11101456459:0.28600039:11101438580:-0.02:11101458181:0.1:11101458905:-1.14:11101458638:2:11101456054:-0.05718469:11101459498:0.5:11101454021:-0.4:11101459499:0.5:11101458412:-0.5:11101459500:0.5:11101456245:-0.02:11101459501:0.5:11101458863:-0.02:11101459504:0.5:11101444398:-0.5:11101459509:0.2:11101459296:-0.5:11101459523:0.5:11101458511:-1.1409:11101459524:0.5:11101457822:-1:11101459525:0.4:11101439758:-0.05734873:11101456853:0.5:11101401849:-0.81803941:11101459512:0.2:11101420500:-0.5:11101455149:0.86755981:11101459569:-0.5014:11101458903:0.2:11101437200:-8:11101459552:7.92:11101457636:-0.23106522:11101449969:2.4:11101402918:-0.01221818:11101453397:0.335:11101433925:-1.01128782:11101453620:0.002:11101387597:-1'

Code in python

        # format the data to compute crc
        data = []
        if self.raws.get(channel): #raw channel
            for i in range(25):
                data.extend([book[0][i][1],book[0][i][2]])
                data.extend([book[1][i][1],book[1][i][2]])
        else:
            for i in range(25):
                data.extend([book[0][i][0],book[0][i][2]])
                data.extend([book[1][i][0],book[1][i][2]])

        data_str = ':'.join('{:.8f}'.format(x).rstrip('0').rstrip('.') for x in data)
        crc = binascii.crc32(data_str.encode('ascii'))
        
        # Convert signed checksum to unsigned
        if checksum < 0: 
            checksum = 4294967296 + checksum
        
        if crc == checksum:
            #print('Channel %s: Checksum is OK'%channel)
            # Replace the old working book with the new state of the book
            self.working_books[channel] = copy.deepcopy(book)
            self.callback_method(channel, False)
            return 0
        
        else:
            pair = self.getpair(channel)
            print('Pair %s: Checksum not valid, Received from Bitfinex: %s <> Localy computed: %s'%(pair, checksum, crc))
            print(book[0])
            print(book[1])
            print(data)
            print(data_str)
            print(data_str.encode('ascii'))
            #data_str = ':'.join(str(x) for x in data)
            #crc = binascii.crc32(data_str.encode('ascii'))
            #print(data_str)
            #print(crc)
            return 1

The code above works as a charm for 'normal' book. With raw book, it happens sometimes that the CRC doesn't match if there is an mount below 10E-7 or something around.

Arnaud

@Nono242
Copy link

Nono242 commented Apr 23, 2018

I've updated my python code to suit Bitfinex exponent nightmare rules.
` ########################################################################
# Check CRC32 received from Bitfinex with CRC32 computed from local book #
########################################################################

def checksum_compare(self, channel, checksum):
    # Take book from channel
    book = self.findbook(channel)

    # Sanity check size of books (Ask and bid)
    if len(book[0]) != 25 or len(book[1]) !=25:
        print('Size of book isnt 25 for bid (%s) or ask (%s)'%(len(book[0]),len(book[1])))
        return -1

    # format the data to compute crc
    """
    # Old school without exponent
    if self.raws.get(channel): #raw channel
        for i in range(25):
            data.extend([book[0][i][1],book[0][i][2]])
            data.extend([book[1][i][1],book[1][i][2]])
    data_str = ':'.join('{:.8f}'.format(x).rstrip('0').rstrip('.') for x in data)
    """

    data_str = ''
    for i in range(25):
        val = book[0][i][0] # Bid Offer_Id (For raw book) or Bid Price for 'normal' book
        data_str += '{:.8f}'.format(val).rstrip('0').rstrip('.') + ':'

        val = book[0][i][2] # Bid Offer amount
        val_str = str(val)
        if 'e' in val_str:
            mantissa, exp = val_str.split('e')
            if int(exp) < -6: #Only if exp is strictly below -6 (-7 or -8)
                data_str += mantissa + 'e' + str(int(exp))
            else:
                data_str += '{:.8f}'.format(val).rstrip('0').rstrip('.')
        else:
            data_str += val_str
        data_str += ':'

        val = book[1][i][0] # Ask Offer_Id (For raw book) or Ask Price for 'normal' book
        data_str += '{:.8f}'.format(val).rstrip('0').rstrip('.') + ':'

        val = book[1][i][2] # Ask Offer amount
        val_str = str(val)
        if 'e' in val_str:
            mantissa, exp = val_str.split('e')
            if int(exp) < -6: #Only if exp is strictly below -6 (-7 or -8)
                data_str += mantissa + 'e' + str(int(exp))
            else:
                data_str += '{:.8f}'.format(val).rstrip('0').rstrip('.')
        else:
            data_str += val_str
        data_str += ':'
    data_str = data_str[0:-1] # The last ':' at end of str must be removed

    crc = binascii.crc32(data_str.encode('ascii'))
    
    # Convert signed checksum to unsigned
    if checksum < 0: 
        checksum = 4294967296 + checksum
    
    if crc == checksum:
        #print('Channel %s: Checksum is OK'%channel)
        # Replace the old working book with the new state of the book
        self.working_books[channel] = copy.deepcopy(book)
        self.callback_method(channel, False)
        return 0
    
    else:
        pair = self.getpair(channel)
        print('Pair %s: Checksum not valid, Received from Bitfinex: %s <> Localy computed: %s'%(pair, checksum, crc))
        print(book[0])
        print(book[1])
        print(data_str)
        print(data_str.encode('ascii'))
        return 1`

It may useful to someone else
Arnaud

@mythichamster
Copy link

In the example above, there's a ask price level which is not on a standard tick increment: [8753.79745876, 11101458511, -1.1409]. How does this happen? Is this a special order type from the exchange?

@adrianog
Copy link

adrianog commented Dec 6, 2018

any update on this?

@15305512923
Copy link

I want to know,How to retrieve snapshot information after CHECKSUM fails?

@DaanVanVugt
Copy link

@prdn could you specify the string formatter used when calculating the CRC32 checksum? The version above by @Nono242 does not work for all books (TRX/BTC for instance), and the blog post about checksums does not say anything about this.

@Nono242
Copy link

Nono242 commented Sep 13, 2019

@prdn could you specify the string formatter used when calculating the CRC32 checksum? The version above by @Nono242 does not work for all books (TRX/BTC for instance), and the blog post about checksums does not say anything about this.

Hello Exteris,
Sometimes Bitfinex changes slightly their rules without going into details and before hand. I don't maintain this python code anymore. However, I've an updated version in C++ instead which is working perfectly. If you like I can't transmit to you. This C++ code works for Raw and 'Normal' book.

@ctubio
Copy link

ctubio commented Feb 16, 2021

the input values used for the checksum calculation on bitfinex side are a joke

with a bit of debug you can see all they do (and have some laughts while considering to do the same on your end)

hire someone at kraken please, the kraken api is a pleasure to sync with

@BlueFrye2
Copy link

Bitcoin

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment