Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Frozen indices benchmark
#/bin/bash
echo $(date) "Create snapshot repositories"
curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm" -H 'Content-Type: application/json' -d'
{
"type": "fs",
"settings": {
"location": "/data/snapshots/elasticlogs-nofm"
}
}
'
curl -X PUT "localhost:9200/_snapshot/elasticlogs-fm" -H 'Content-Type: application/json' -d'
{
"type": "fs",
"settings": {
"location": "/data/snapshots/elasticlogs-fm"
}
}
'
echo $(date) "Snapshot not forcemerged data to the elasticlogs-nofm repository"
curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm/1?wait_for_completion=true" -H 'Content-Type: application/json' -d'
{
"indices": "elasticlogs",
"ignore_unavailable": true,
"include_global_state": false
}
'
echo $(date) "Forcemerge the index down to a single segment - wait for completion"
curl -X POST "localhost:9200/elasticlogs/_forcemerge?max_num_segments=1&flush=true"
COUNT=$(curl -s localhost:9200/_cat/segments | grep elasticlogs | wc -l)
while [ $COUNT -gt 2 ]
do
sleep 30
COUNT=$(curl -s localhost:9200/_cat/segments | grep elasticlogs | wc -l)
if [ $COUNT == 2 ]
then
echo $(date) "Forcemerge of elasticlogs index has completed."
else
echo $(date) "Forcemerge of elasticlogs index still not completed."
fi
done
echo $(date) "Snapshot forcemerged data to the elasticlogs-fm repository"
curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm/1?wait_for_completion=true" -H 'Content-Type: application/json' -d'
{
"indices": "elasticlogs",
"ignore_unavailable": true,
"include_global_state": false
}
'
echo $(date) "Snapshots completed"
#!/bin/bash
INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)
for i in $(seq 1 $INDEX_COUNT); do
echo "Freeze elasticlogs-$i"
curl -XPOST localhost:9200/elasticlogs-$i/_freeze
done
#!/bin/bash
INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)
INDEX_HALF_COUNT=$((INDEX_COUNT / 2))
HALF_START=$((INDEX_HALF_COUNT + 1))
for i in $(seq 1 $INDEX_HALF_COUNT); do
echo "Freeze elasticlogs-$i"
curl -XPOST localhost:9200/elasticlogs-$i/_freeze
done
for i in $(seq $HALF_START $INDEX_COUNT); do
echo "Unfreeze elasticlogs-$i"
curl -XPOST localhost:9200/elasticlogs-$i/_unfreeze
done
#!/bin/bash
INDEX_COUNT=20
echo "Delete existing indices"
curl -XDELETE localhost:9200/elasticlogs*
echo "restore 20 indices"
for i in $(seq 1 $INDEX_COUNT); do
./restore_snapshot.sh localhost:9200 elasticlogs-fm elasticlogs 1 elasticlogs-$i
done
#!/bin/bash
INDEX_COUNT=20
echo "Delete existing indices"
curl -XDELETE localhost:9200/elasticlogs*
echo "restore 20 indices"
for i in $(seq 1 $INDEX_COUNT); do
./restore_snapshot.sh localhost:9200 elasticlogs-nofm elasticlogs 1 elasticlogs-$i
done
#/bin/bash
TIMESTAMP=$(date +%s)
ES_HOST=$1
REPOSITORY=$2
INDEX_NAME=$3
SNAPSHOT_ID=$4
NEW_INDEX_NAME=$5
RESULT=$(curl -XPOST -s http://$ES_HOST/_snapshot/$REPOSITORY\/$SNAPSHOT_ID\/_restore -H 'Content-Type: application/json' -d "{
\"indices\": \"$INDEX_NAME\",
\"ignore_unavailable\": \"true\",
\"include_global_state\": false,
\"rename_pattern\": \"$INDEX_NAME\",
\"rename_replacement\": \"$NEW_INDEX_NAME\"
}")
if [ "$RESULT" == '{"accepted":true}' ]
then
echo $(date) "Snapshot initiated. Restoring" $INDEX_NAME "into" $NEW_INDEX_NAME "from repository" $REPOSITORY ". Await completion."
COUNT=0
while [ $COUNT == 0 ]
do
sleep 30
COUNT=$(curl -s http://$ES_HOST/_cat/indices | grep $NEW_INDEX_NAME | grep green | wc -l)
if [ $COUNT == 1 ]
then
echo $(date) "Snapshot restoration of" $NEW_INDEX_NAME "has completed."
else
echo $(date) "Snapshot restoration of" $NEW_INDEX_NAME "still not completed."
fi
done
else
echo $(date) "Initiation of snapshot restoration for" $NEW_INDEX_NAME "FAILED :" $RESULT
fi
#!/bin/bash
./unfreeze_all.sh
esrally --track-path=/home/ubuntu/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-querying --client-options="use_ssl:false,timeout:600,request_timeout:600,max_retries:0" --track-params="node_type:'cold',instance:'d2.xlarge',data_vol:'1TB',forcemerged:'yes',throttled_threads:1,frozen_ratio_pct:0" --telemetry=node-stats --telemetry-params="node-stats-include-indices:true,node-stats-sample-interval:10" --on-error=abort
./freeze_all.sh
esrally --track-path=/home/ubuntu/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-querying --client-options="use_ssl:false,timeout:600,request_timeout:600,max_retries:0" --track-params="node_type:'cold',instance:'d2.xlarge',data_vol:'1TB',forcemerged:'yes',throttled_threads:1,frozen_ratio_pct:100" --telemetry=node-stats --telemetry-params="node-stats-include-indices:true,node-stats-sample-interval:10" --on-error=abort
#/bin/bash
esrally --track-path=~/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-data-generation
#!/bin/bash
INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)
for i in $(seq 1 $INDEX_COUNT); do
echo "Unfreeze elasticlogs-$i"
curl -XPOST localhost:9200/elasticlogs-$i/_unfreeze
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.