Instantly share code, notes, and snippets.

Embed
What would you like to do?
# frozen_string_literal: true
require 'net/http'
require 'fiber'
require 'uri'
require 'openssl'
require 'socket'
require 'pp'
require 'benchmark/ips'
require 'benchmark/memory'
require 'memory_profiler'
#rubocop:disable Metrics/LineLength
# :nodoc:
class Selector
def initialize
@readable = {}
end
def wait_readable(io)
Fiber.new do
@readable[io] = Fiber.current
Fiber.yield
yield
end.resume
end
def resume
readable, = IO.select(@readable.keys)
readable.each do |io|
fiber = @readable.delete(io)
fiber.resume
end
end
def run
resume until @readable.empty?
end
end
# :nodoc:
class AsyncHTTP
# :nodoc:
class Connection
BODY_SPLIT = "\r\n\r\n".freeze
attr_reader :hostname, :port, :ssl, :ready, :selector
def initialize(hostname, port, selector)
@selector = selector
@hostname = hostname
@port = port
@socket = TCPSocket.new hostname, port
@ssl = OpenSSL::SSL::SSLSocket.new(@socket)
@ready = false
@buffer = +''
@count = 0
@requests = 0
@chunked = false
end
def connect
return if ready
@ready = true
ssl.sync_close = true
ssl.connect
end
def start(request)
@requests += 1
ssl.write "GET #{request.request_uri} HTTP/1.1\r\nHost: #{request.hostname}\r\n\r\n"
end
def wait_response(&block)
loop do
break if @count == @requests
@buffer ||= +''
@buffer << ssl.read_nonblock(4096)
if @buffer.include?(BODY_SPLIT) && @chunk == false
header, @buffer = buf.split(BODY_SPLIT)
@chunked = true if header.include?('chunked')
end
if @chunk == true
if @buffer.include?("0#{BODY_SPLIT}")
@chunk = false
@count += 1
response, @buffer = @buffer.split("0#{BODY_SPLIT}")
# TODO: Combine Chunk
yield response
end
else
@count += 1
yield @buffer
end
end
rescue IO::WaitReadable
return if @count == @requests
selector.wait_readable(ssl) do
wait_response(&block)
end
end
def close
return unless ready
ssl.close
end
end
include Enumerable
attr_reader :selector
class << self
def start(&block)
new.start(&block)
end
end
def initialize
@selector = Selector.new
@pool = {}
end
def start(&block)
instance_exec(self, &block)
@pool.values.each(&:close)
end
def request(request)
conn = @pool[request.hostname] ||= Connection.new(request.hostname, request.port, selector)
conn.connect
conn.start(request)
end
def each(&block)
@pool.values.each do |conn|
conn.wait_response(&block)
end
selector.run
end
end
#rubocop:enable Metrics/LineLength
uris = [
'https://api.bitfinex.com/v1/pubticker/BTCUSD',
'https://api.bitfinex.com/v1/pubticker/ETHUSD',
'https://api.hitbtc.com/api/2/public/ticker/BTCUSD',
'https://api.hitbtc.com/api/2/public/ticker/ETHUSD',
'https://bittrex.com/api/v1.1/public/getmarketsummary?market=USD-BTC',
'https://bittrex.com/api/v1.1/public/getmarketsummary?market=USD-ETH',
'https://api.binance.com/api/v1/ticker/24hr?symbol=BTCUSDT',
'https://api.binance.com/api/v1/ticker/24hr?symbol=ETHUSDT'
]
Benchmark.ips do |x|
x.report('Fiber') do |c|
c.times do
AsyncHTTP.start do |http|
uris.each do |uri|
http.request(URI(uri))
end
http.each do |body|
# TODO
end
end
sleep 5
end
end
x.report('Net::HTTP') do |c|
c.times do
uris.each do |uri|
Net::HTTP.get(URI(uri))
end
sleep 5
end
end
x.report('[P] Net::HTTP') do |c|
c.times do
uris.map do |uri|
Thread.new { Net::HTTP.get(URI(uri)) }
end.each(&:join)
sleep 5
end
end
x.compare!
end
Benchmark.memory do |x|
x.report('Fiber') do
AsyncHTTP.start do |http|
uris.each do |uri|
http.request(URI(uri))
end
http.each do |body|
# TODO
end
end
end
sleep 5
x.report('Net::HTTP') do
uris.each do |uri|
Net::HTTP.get(URI(uri))
end
end
sleep 5
x.report('[P] Net::HTTP') do
uris.map do |uri|
Thread.new { Net::HTTP.get(URI(uri)) }
end.each(&:join)
end
x.compare!
end
# MemoryProfiler.report do
# AsyncHTTP.start do |http|
# uris.each do |uri|
# http.request(URI(uri))
# end
#
# http.each do |body|
# # TODO
# end
# end
# end.pretty_print
@elct9620

This comment has been minimized.

Owner

elct9620 commented Aug 17, 2018

Warming up --------------------------------------
               Fiber     1.000  i/100ms
           Net::HTTP     1.000  i/100ms
       [P] Net::HTTP     1.000  i/100ms
Calculating -------------------------------------
               Fiber      0.154  (± 0.0%) i/s -      1.000  in   6.505653s
           Net::HTTP      0.119  (± 0.0%) i/s -      1.000  in   8.376975s
       [P] Net::HTTP      0.161  (± 0.0%) i/s -      1.000  in   6.217517s

Comparison:
       [P] Net::HTTP:        0.2 i/s
               Fiber:        0.2 i/s - 1.05x  slower
           Net::HTTP:        0.1 i/s - 1.35x  slower

Calculating -------------------------------------
               Fiber    79.822k memsize (     1.162k retained)
                       279.000  objects (     9.000  retained)
                        45.000  strings (     9.000  retained)
           Net::HTTP     9.085M memsize (    12.521k retained)
                         5.829k objects (   119.000  retained)
                        50.000  strings (    50.000  retained)
       [P] Net::HTTP    17.515M memsize (    10.744k retained)
                         5.851k objects (   105.000  retained)
                        50.000  strings (    50.000  retained)

Comparison:
               Fiber:      79822 allocated
           Net::HTTP:    9084771 allocated - 113.81x more
       [P] Net::HTTP:   17515208 allocated - 219.43x more
@elct9620

This comment has been minimized.

Owner

elct9620 commented Aug 17, 2018

Warming up --------------------------------------
               Fiber     1.000  i/100ms
           Net::HTTP     1.000  i/100ms
       [P] Net::HTTP     1.000  i/100ms
Calculating -------------------------------------
               Fiber      0.152  (± 0.0%) i/s -      1.000  in   6.578475s
           Net::HTTP      0.116  (± 0.0%) i/s -      1.000  in   8.648963s
       [P] Net::HTTP      0.162  (± 0.0%) i/s -      1.000  in   6.175362s

Comparison:
       [P] Net::HTTP:        0.2 i/s
               Fiber:        0.2 i/s - 1.07x  slower
           Net::HTTP:        0.1 i/s - 1.40x  slower

Calculating -------------------------------------
               Fiber    83.919k memsize (     1.162k retained)
                       279.000  objects (     9.000  retained)
                        44.000  strings (     9.000  retained)
           Net::HTTP     9.085M memsize (    12.385k retained)
                         5.827k objects (   117.000  retained)
                        50.000  strings (    50.000  retained)
       [P] Net::HTTP    17.466M memsize (    11.139k retained)
                         5.849k objects (   110.000  retained)
                        50.000  strings (    50.000  retained)

Comparison:
               Fiber:      83919 allocated
           Net::HTTP:    9084994 allocated - 108.26x more
       [P] Net::HTTP:   17466200 allocated - 208.13x more
@elct9620

This comment has been minimized.

Owner

elct9620 commented Aug 17, 2018

Warming up --------------------------------------
               Fiber     1.000  i/100ms
           Net::HTTP     1.000  i/100ms
       [P] Net::HTTP     1.000  i/100ms
Calculating -------------------------------------
               Fiber      0.153  (± 0.0%) i/s -      1.000  in   6.524378s
           Net::HTTP      0.116  (± 0.0%) i/s -      1.000  in   8.619916s
       [P] Net::HTTP      0.161  (± 0.0%) i/s -      1.000  in   6.193903s

Comparison:
       [P] Net::HTTP:        0.2 i/s
               Fiber:        0.2 i/s - 1.05x  slower
           Net::HTTP:        0.1 i/s - 1.39x  slower

Calculating -------------------------------------
               Fiber    83.919k memsize (     1.162k retained)
                       279.000  objects (     9.000  retained)
                        44.000  strings (     9.000  retained)
           Net::HTTP     9.083M memsize (    11.887k retained)
                         5.803k objects (   116.000  retained)
                        50.000  strings (    50.000  retained)
       [P] Net::HTTP    17.451M memsize (    11.765k retained)
                         5.851k objects (   114.000  retained)
                        50.000  strings (    50.000  retained)

Comparison:
               Fiber:      83919 allocated
           Net::HTTP:    9083352 allocated - 108.24x more
       [P] Net::HTTP:   17450878 allocated - 207.95x more
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment