Skip to content

Instantly share code, notes, and snippets.

@hayzem
Created July 19, 2018 14:44
Show Gist options
  • Save hayzem/600f7520cc521e1798d55c8081f85c66 to your computer and use it in GitHub Desktop.
Save hayzem/600f7520cc521e1798d55c8081f85c66 to your computer and use it in GitHub Desktop.
Bash script for redis key migration
#!/usr/bin/env bash
######
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
######
######
### Migrates the keys responding to the pattern specified on the command line, using DUMP/RESTORE, supports authentication differently from MIGRATE
######
PATTERN=$1
# add -a before passwords
SOURCE_PASSWORD=""
SOURCE_PORT=6379
SOURCE_SCHEMA=0
# add -a before passwords
TARGET_PASSWORD=""
TARGET_PORT=6379
TARGET_SCHEMA=4
# log file
LOG_FILE="redis-migrate.log"
# iteration
START=0
COUNT=10000
# ttl
TTL=7889400
WAIT_COUNT=$2
if [[ -z "$PATTERN" ]]; then
echo -e "Please provide a KEYS matcher, like *~cache"
exit 1
fi
echo "*** Migrating keys matching $PATTERN"
M_COUNT=0
while true; do
echo " ** starting from $START counts $COUNT"
redis-cli ${SOURCE_PASSWORD} -p ${SOURCE_PORT} SCAN ${START} MATCH ${PATTERN} COUNT ${COUNT} | while read KEY; do
[[ ${KEY} == ERR* ]] && echo "*** Keys matched with $PATTERN TTL updated to $TTL and migrated to destination db." && exit
[[ ${KEY} == 0 ]] && continue
# Get TTL for the key in the source
KEY_TTL=`redis-cli ${SOURCE_PASSWORD} -p ${SOURCE_PORT} TTL "$KEY"`
# Check if the key is immortal
if [[ ${KEY_TTL} -lt 1 ]]; then
echo " * set expiration and migrate: $KEY"
# Set TTL for the key in the source
redis-cli ${SOURCE_PASSWORD} -p ${SOURCE_PORT} EXPIRE "$KEY" ${TTL} >> ${LOG_FILE} 2>&1
redis-cli --raw -p ${SOURCE_PORT} -n ${SOURCE_SCHEMA} ${SOURCE_PASSWORD} DUMP "$KEY" | head -c -1 | redis-cli -x -p ${TARGET_PORT} -n ${TARGET_SCHEMA} ${TARGET_PASSWORD} RESTORE "$KEY" ${TTL} >> ${LOG_FILE} 2>&1
echo "Dump/Restore \"$KEY\", ttl $TTL" >> ${LOG_FILE} 2>&1
M_COUNT=$((M_COUNT+1))
fi
if [[ $((M_COUNT%WAIT_COUNT)) -eq 0 ]]; then
sleep 1
fi
done
START=$((COUNT + START))
done
echo "Exiting."
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment