tweak.js

'use strict'

// process.env.DEBUG = '*'

const path = require('path')

const async = require('async')
const hypercore = require('hypercore')
const Hyperbee = require('hyperbee')

const replicate = require('@hyperswarm/replicator')

const hopts = {
  overwrite: true
}

const hbOpts = {
  valueEncoding: 'json'
}

const rawCandleData = [
  [1533919680000,6373,6374.5,6375.9,6369.2,38.58293517],
  [1533919620000,6408.8,6372.9,6408.9,6366,327.53682997],
  [1533919560000,6420,6409.6,6420.1,6406,42.38708092],
  [1533919500000,6428.1,6420,6428.1,6420,46.31485018],
  [1533919440000,6428.1,6428.1,6428.1,6428,1.01328572],
  [1533919380000,6428,6428,6428,6428,0.01],
  [1533919320000,6429.23156736,6428.1,6429.3,6428,1.58844826],
  [1533919260000,6432.54036992,6429.2,6432.54036992,6429.2,0.49441145],
  [1533919200000,6434.7,6433.5,6434.8,6433.5,1.01979856],
  [1533919140000,6434.6,6434.7,6434.70820608,6434.6,1.54670225],
  [1533919080000,6436.7,6434.6,6436.7,6434,1.60242713],
  [1533919020000,6435,6438.2,6438.2,6435,4.82548411]
]
rawCandleData.sort((a, b) => {
  return a[0] - b[0]
})

const kp = 'candles!5m!'
const from = rawCandleData[2][0]
const to = rawCandleData[4][0]

console.log('from:', from, 'to:', to)

let feed
let localFeed
let db
let localDb
async.auto({
  setupCores: (next) => {
    // server has 1000 entries
    feed = hypercore(path.join(__dirname, 'dbs', 'BTCUSD'), hopts)
    db = new Hyperbee(feed, hbOpts)

    // lets share it

    db.feed.ready(() => {
      replicate(feed, { announce: true, live: true, lookup: false })

      // localcopy will have a frame of it
      // lets sync some data from the middle
      localFeed = hypercore(path.join(__dirname, 'dbs', 'BTCUSD_frame'), feed.key, {
        ...hopts,
        sparse: true
      })

      replicate(localFeed, { lookup: true, live: false })

      localDb = new Hyperbee(localFeed, hbOpts)
      localDb.feed.ready(next)
    })
  },

  prepareCandles: async () => {
    // feed data into the data source / server

    let batch = db.batch()
    for (let i = 0; i < rawCandleData.length; i++) {
      const data = rawCandleData[i]
      const mts = data[0]

      const k = kp + mts
      await batch.put(k, JSON.stringify(data))
    }

    await batch.flush()
  },

  tweak: ['prepareCandles', 'setupCores', (res, next) => {
    console.log('remove the line with `return next` to make it work')
    console.trace()

    return next()
    // populate feed.length
    // also required for
    // https://github.com/bitfinexcom/hypercore-bisect/blob/master/index.js#L23
    localDb.feed.get(1, (er, el) => {
      next()
    })
  }],

  findIndexes: ['prepareCandles', 'setupCores', 'tweak', ({ findIndexes }, next) => {
    console.log("findIndexes", kp + from)

    const s = localDb.createReadStream({ gte: kp + from })
    let count = 0

    s.on('data', function (data) {
      count++
      console.log(count, data)
    })
  }]
}, (err, data) => {
  if (err) throw err

  console.log("finished")
})