Dump a large elasticsearch index to a csv file

gistfile1.rb

require 'net/http'
require 'json'
require 'uri'

scroll_id = nil

counter = 0
while true
  $stderr.puts "Running #{counter}"
  if scroll_id
    url = "curl -X GET -s 'http://localhost:9200/_search/scroll?scroll=1m' -d '#{scroll_id}'"
  else
    url = "curl -X GET -s 'http://localhost:9200/logstash-2015.02.27/_search?scroll=1m&size=1000'"
  end
  res = %x{#{url}}.strip
  body = JSON.parse(res)
  scroll_id = body["_scroll_id"] unless scroll_id
  current = body['hits']['hits'].length
  break if current == 0
  counter += current
  body['hits']['hits'].each do |x|
    puts "#{x['_source']['host']};#{x['_source']['path']};#{x['_source']['type']};#{JSON.dump(x['_source']).size};#{x['_source']['message'].size};"
  end
end

$stderr.puts "End #{counter}