Using the Elasticsearch scroll API

parse.sh

#!/bin/bash

es_url=192.168.1.2:9200
index=twitch_v2

response=$(curl -s $es_url/$index/_search?scroll=1m -d @query.json)
scroll_id=$(echo $response | jq -r ._scroll_id)
hits_count=$(echo $response | jq -r '.hits.hits | length')
hits_so_far=hits_count
echo Got initial response with $hits_count hits and scroll ID $scroll_id

data=$(echo $response | jq -c '.hits.hits')
echo $data | jq -c '.[]._source' | while read content ; do
if [[ $content =~ .*(won|lost).*duckets.* ]]
then
#echo $content
curl -X POST -H "Content-Type: application/json" 192.168.1.2:3000/last -d "$content"
echo
fi
done

while [ "$hits_count" != "0" ]; do
  response=$(curl -s $es_url/_search/scroll?scroll=1m -d "$scroll_id")
  scroll_id=$(echo $response | jq -r ._scroll_id)
  hits_count=$(echo $response | jq -r '.hits.hits | length')
  hits_so_far=$((hits_so_far + hits_count))
  echo "Got response with $hits_count hits (hits so far: $hits_so_far), new scroll ID $scroll_id"
  
	data=$(echo $response | jq -c '.hits.hits')
	echo $data | jq -c '.[]._source' | while read content ; do
	if [[ $content =~ .*(won|lost).*duckets.* ]]
	then
	#echo $content
	curl -X POST -H "Content-Type: application/json" 192.168.1.2:3000/last -d "$content"
	echo
	fi
	done
done

echo Done!

query.json

{
  "sort": ["_doc"],
  "size": 100,
  "query": {
    "bool": {
      "must": {
        "match_all": {}
      },
      "filter": {
        ...
      }
    }
  }
}

query_my.json

{
    "query": {
	"match": { "user" : "hugo__bot"}
	},
    "size":  100,
    "sort": { "timestamp": { "order": "asc" }}
}

search.sh

#!/bin/bash

es_url=my-es-host:9200
index=my-index

response=$(curl -s $es_url/$index/_search?scroll=1m -d @query.json)
scroll_id=$(echo $response | jq -r ._scroll_id)
hits_count=$(echo $response | jq -r '.hits.hits | length')
hits_so_far=hits_count
echo Got initial response with $hits_count hits and scroll ID $scroll_id

# TODO process first page of results here

while [ "$hits_count" != "0" ]; do
  response=$(curl -s $es_url/_search/scroll -d "{ \"scroll\": \"1m\", \"scroll_id\": \"$scroll_id\" }")
  scroll_id=$(echo $response | jq -r ._scroll_id)
  hits_count=$(echo $response | jq -r '.hits.hits | length')
  hits_so_far=$((hits_so_far + hits_count))
  echo "Got response with $hits_count hits (hits so far: $hits_so_far), new scroll ID $scroll_id"
  
  # TODO process page of results
done

echo Done!