superagent-throttle issue6

issue6.js

'use strict'
/**
 * ## tests
 *
 * I've never written any tests before, so I'm sure there's room for
 * criticism here.
 *
 */

const request       = require('superagent')
const _             = require('lodash')
const assert        = require('chai').assert
const Throttle      = require('../index')
const http          = require('http')


let log
let max
let mockServer
let respond

/**
 * ## mockServer
 *
 * a very fancy no-dep http server that just returns 'ok' to every request,
 * it seems to take about 3ms to do the response on my clunky dev machine
 */
mockServer = (delay, jitter) => {
  return http.createServer(
    (request, response) => {
      if (!delay) return respond(response)
      if (!jitter) jitter = 0
      setTimeout(
        () => respond(response),
        Math.floor((Math.random() * jitter) + delay)
      )
    }
  ).listen(3003)
}

respond = (response) => {
  response.writeHead(200)
  response.end()
}

/**
 * ## log
 *
 * a helper to write pretty tables when attached to Throttle events
 */
log = function(prefix) {
  let count = 0
  let start = Date.now()
  return (request) => {
    let rate
    let check = new Date(Date.now() - request.throttle.ratePer)
    rate = request.throttle._requestTimes.length - 1 - _.findLastIndex(
      request.throttle._requestTimes,
      (date) => (date < check)
    )
    count += 1
    console.log([
      '| ',
      _.padEnd(prefix, 10, ' '),
      '| ',
      _.padStart(count, 3, ' '),
      ' | ',
      _.padStart(Date.now() - start, 6, ' '),
      ' | conc: ',
      _.padStart(request.throttle._current, 3, ' '),
      ' | rate: ',
      _.padStart(rate, 3, ' '),
      ' | queued: ',
      _.padStart(request.throttle._buffer.length, 3, ' '),
      ' | ',
      request.serial
    ].join(''))
  }
}

/**
 * ## max
 *
 * collates various maximums, useful for tests
 */
max = function() {
  let count = 0
  let maxRate = 0
  let maxConcurrent = 0
  let maxBuffer = 0
  let start = Date.now()
  return (request) => {
    if (request) {
      let rate
      let check = new Date(Date.now() - request.throttle.ratePer)
      rate = request.throttle._requestTimes.length - 1 - _.findLastIndex(
        request.throttle._requestTimes,
        (date) => (date < check)
      )
      count += 1
      if (maxConcurrent < request.throttle._current)
        maxConcurrent = request.throttle._current
      if (maxRate < rate)
        maxRate = rate
      if (maxBuffer < request.throttle._buffer.length)
        maxBuffer = request.throttle._buffer.length
    }
    return {
      count,
      maxRate,
      maxConcurrent,
      maxBuffer
    }
  }
}

const throttle = new Throttle({
  rate: 10000, // how many requests can be sent every `ratePer`
  ratePer: 100, // number of ms in which `rate` requests may be sent
  concurrent: 2000 // how many requests can be sent concurrently
})
.on('sent', log('sent'))
.on('drain', () => server.close())
const make = (endpoint, data, cb) => {
  request.post(endpoint)
    .use(throttle.plugin())
    .send(data)
    .timeout(3000)
    .end((err, res) => {
      // retry
      if (err && (err.code === 'ECONNABORTED' || err.code === 'ECONNRESET')) {
        return make(endpoint, data, cb)
      }
      if (err) return cb(err)
      if (!res || !res.body) return cb(new Error('No response'))
      if (res.error || res.body.error) return cb(res.body)
      cb(null, res.body)
    })
}

let server = mockServer()
for (var i = 0; i < 20000; i++) make(
  'http://localhost:3003',
  { foo: 'bar' },
  (err, res) => {
    if (err) console.log(err)
  }
)