bfx_test_book_raw.js

/*USAGE:
npm install ws lodash async moment crc-32
mkdir logs
node bfx_test_book.js BTCUSD
*/

const WS = require('ws')
const _ = require('lodash')
const async = require('async')
const fs = require('fs')
const moment = require('moment')
const CRC = require('crc-32')

const pair = process.argv[2]

const conf = {
  wshost: "wss://api.bitfinex.com/ws/2"
}

const logfile = __dirname + '/logs/ws-book-raw.log'

const BOOK = {}

let connected = false
let connecting = false
let cli
let seq = null

function connect() {
  if (connecting || connected) return
  connecting = true

  cli = new WS(conf.wshost, { /*rejectUnauthorized: false*/ })
   
  cli.on('open', function open() {
    console.log('WS open')
    connecting = false
    connected = true
    BOOK.bids = {}
    BOOK.asks = {}
    BOOK.psnap = {}
    BOOK.mcnt = 0
    cli.send(JSON.stringify({ event: 'conf', flags: 65536 + 131072 }))
    cli.send(JSON.stringify({ event: "subscribe", channel: "book", pair: pair, prec: "R0", len: 25 }))
  })

  cli.on('close', function open() {
    seq = null
    console.log('WS close')
    connecting = false
    connected = false
  })

  cli.on('message', function(msg) {
    msg = JSON.parse(msg)

    if (msg.event) return
    if (msg[1] === 'hb') {
      seq = +msg[2]
      return
    } else if (msg[1] === 'cs') {
      seq = +msg[3]

      const checksum = msg[2]
      const csdata = []
      const bids_keys = BOOK.psnap['bids']
      const asks_keys = BOOK.psnap['asks']

      for (let i = 0; i < 25; i++) {
        if (bids_keys[i]) {
          const eid = bids_keys[i]
          const pp = BOOK.bids[eid]
          csdata.push(pp.id, pp.amount)
        }
        if (asks_keys[i]) {
          const eid = asks_keys[i]
          const pp = BOOK.asks[eid]
          csdata.push(pp.id, -pp.amount)
        }
      }
 
      const cs_str = csdata.join(':')
      console.log(cs_str)
      const cs_calc = CRC.str(cs_str)
   
      fs.appendFileSync(logfile, "[" + moment().format("YYYY-MM-DDTHH:mm:ss.SSS") + "] " + pair + " | " + JSON.stringify(["cs_string=" + cs_str, "cs_calc=" + cs_calc, "server_checksum=" + checksum]) + "\n")
      if (cs_calc !== checksum) {
        console.error("CHECKSUM_FAILED", checksum, cs_calc)
        process.exit(-1)
      }
      return
    }
      
    fs.appendFileSync(logfile, "[" + moment().format("YYYY-MM-DDTHH:mm:ss.SSS") + "] " + pair + " | " + JSON.stringify(msg) + "\n")

    if (BOOK.mcnt === 0) {
      _.each(msg[1], function(pp) {
        pp = { id: pp[0], price: pp[1], amount: pp[2] }
        const side = pp.amount >= 0 ? 'bids' : 'asks'
        pp.amount = Math.abs(pp.amount)
        BOOK[side][pp.id] = pp
      })
    } else {
      const cseq = +msg[2]
      msg = msg[1]
      
      if (!seq) {
        seq = cseq - 1
      }

      if (cseq - seq !== 1) {
        console.error('OUT OF SEQUENCE', seq, cseq)
        process.exit()
      }

      seq = cseq

      const pp = { id: msg[0], price: msg[1], amount: msg[2], ix: msg[3] }
      
      if (!pp.price) {
        let found = true
        if (pp.amount > 0) {
          if (BOOK['bids'][pp.id]) {
            delete BOOK['bids'][pp.id]
          } else {
            found = false
          }
        } else if (pp.amount < 0) {
          if (BOOK['asks'][pp.id]) {
            delete BOOK['asks'][pp.id]
          } else {
            found = false
          }
        }
        if (!found) {
          fs.appendFileSync(logfile, "[" + moment().format() + "] " + pair + " | " + JSON.stringify(pp) + " BOOK_RAW delete fail found\n")
        }

      } else {
        const side = pp.amount >= 0 ? 'bids' : 'asks'
        pp.amount = Math.abs(pp.amount)
        BOOK[side][pp.id] = pp
      }
    }

    _.each(['bids', 'asks'], function(side) {
      let sbook = BOOK[side]
      let bentries = Object.keys(sbook)

      let prices = bentries.sort(function(a, b) {
        if (+sbook[a].price === +sbook[b].price) {
          return a - b
        }
        if (side === 'bids') {
          return +sbook[a].price >= +sbook[b].price ? -1 : 1
        } else {
          return +sbook[a].price <= +sbook[b].price ? -1 : 1
        }
      })

      BOOK.psnap[side] = bentries
    })

    BOOK.mcnt++
  })
}

setInterval(function() {
  if (connected) return
  connect()
}, 3500)

function saveBook() {
  const now = moment.utc().format('YYYYMMDDHHmmss')
  fs.writeFileSync(__dirname + "/logs/tmp-ws-book-raw-" + pair + '-' + now + '.log', JSON.stringify({ bids: BOOK.bids, asks: BOOK.asks}))
}

setInterval(function() {
  saveBook()
}, 30000)

I want to know,How to retrieve snapshot information after CHECKSUM fails?

@prdn could you specify the string formatter used when calculating the CRC32 checksum? The version above by @Nono242 does not work for all books (TRX/BTC for instance), and the blog post about checksums does not say anything about this.

@prdn could you specify the string formatter used when calculating the CRC32 checksum? The version above by @Nono242 does not work for all books (TRX/BTC for instance), and the blog post about checksums does not say anything about this.

Hello Exteris,
Sometimes Bitfinex changes slightly their rules without going into details and before hand. I don't maintain this python code anymore. However, I've an updated version in C++ instead which is working perfectly. If you like I can't transmit to you. This C++ code works for Raw and 'Normal' book.

the input values used for the checksum calculation on bitfinex side are a joke

with a bit of debug you can see all they do (and have some laughts while considering to do the same on your end)

hire someone at kraken please, the kraken api is a pleasure to sync with

Bitcoin