WIP version of streaming record batches with red-arrow

streaming.rb

require "arrow"

nrows = 0
batches = []

Net::HTTP.start(host, port) do |http|
  req = Net::HTTP::Get.new(url)

  # Note: This works but isn't actually streaming anything, it just looks like it
  http.request(req) do |res|
    StringIO.open(res.read_body) do |stringio_input|
      Gio::RubyInputStream.open(stringio_input) do |gio_input|
        Arrow::GIOInputStream.open(gio_input) do |arrow_input|
          reader = Arrow::RecordBatchStreamReader.new(arrow_input)

          p reader.schema

          reader.each do |batch|
            puts "Got batch of #{batch.length} rows"

            nrows += batch.length
            batches << batch
          end
        end
      end
    end
  end
end

puts "Streamed a total of #{batches.length} batches and #{nrows} total rows"