Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Playbook to show some of orchestrator's capabilities, used in DB AMA presentation.
#!/bin/bash
# This playbook assumes you have cloned https://github.com/openark/orchestrator
# and ran: ./script/dock system
# which landed you in orchestrator's playground environment.
# Further information available on the welcome screen once you've ran the docker image.
# FYI, orchestrator's config file is at /etc/orchestrator.conf.json
orchestrator-client -c topology-tabulated -alias ci
orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'
# Refactoring:
orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10112
orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10111
orchestrator-client -c relocate-replicas -i 127.0.0.1:10111 -d 127.0.0.1:10112
orchestrator-client -c which-cluster-instances -alias ci
orchestrator-client -c which-cluster-master -i 127.0.0.1:10112
orchestrator-client -c which-master -i 127.0.0.1:10112
orchestrator-client -c which-master -i 127.0.0.1:10113
orchestrator-client -c which-replicas -i 127.0.0.1:10112
# Scripting samples:
orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111
done
orchestrator-client -c enable-semi-sync-master -i 127.0.0.1:10111
orchestrator-client -c which-replicas -i 127.0.0.1:10111 | while read i ; do
orchestrator-client -c enable-semi-sync-replica -i "$i"
done
orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
orchestrator-client -c disable-semi-sync-master -i "$i"
orchestrator-client -c disable-semi-sync-replica -i "$i"
done
# Examples: GTID, Errant GTIS
~/sandboxes/ci/s2 -e "flush slow logs"
orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'
orchestrator-client -c which-gtid-errant -i 127.0.0.1:10113
orchestrator-client -c replication-analysis
# Digress: use of API
orchestrator-client -c api -path all-instances
orchestrator-client -c api -path all-instances | jq -C . | less -r
orchestrator-client -c api -path all-instances | jq -C '.[] | {Port: .Key.Port, ServerUUID: .ServerUUID, ExecutedGtidSet: .ExecutedGtidSet, GtidErrant:.GtidErrant}'
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="")'
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="") | {Port: .Key.Port}'
# Digress: further API show off
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.SlaveHosts | length == 0) | .Key.Port'
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.MasterKey.Port != 0) | .Key.Port'
orchestrator-client -c api -path cluster/ci | jq -r '.[] | .Version' | grep -c "5.7.26"
# Back to GTID :)
# Assuming we've validated the errant transaction didn't change data:
orchestrator-client -c gtid-errant-inject-empty -i 127.0.0.1:10113
# or...
~/sandboxes/ci/s2 -e "flush slow logs"
sleep 3
orchestrator-client -c gtid-errant-reset-master -i 127.0.0.1:10113
# or...
~/sandboxes/ci/s2 -e "flush slow logs"
sleep 3
orchestrator-client -c api -path gtid-errant-reset-master/127.0.0.1/10113 | jq -C .
# Demonstrate how HAProxy routes 13306 traffic to current master:
mysql -uci -pci -h 127.0.0.1 --port 13306 -e "select @@hostname, @@port"
consul kv get -recurse mysql/master/ci
# Graceful takeovers:
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10112
orchestrator-client -c graceful-master-takeover-auto -alias ci
orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer
# Let orchestrator pick the promoted replica, expect it to be 10113
orchestrator-client -c graceful-master-takeover-auto -alias ci
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10111
# Expect the following to fail due to '"PreventCrossRegionMasterFailover": true' config
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10114
orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111
done
orchestrator-client -c ack-all-recoveries --reason demo
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'
# for this environment, orchestrator's config allows failovers 5 seconds apart.
# In production the config would be higher.
sleep 5
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
while sleep 0.5 ; do orchestrator-client -c replication-analysis ; done
redeploy-ci-env
# Failover:
# A complex scenario. We suggest to orchestrator that 127.0.0.1:10113 is preferred to be promoted in case of failover.
# But we set up the topology such that 127.0.0.1:10113 is the most lagging replica,
# and such that 127.0.0.1:10114 (which may not be promoted due to PreventCrossRegionMasterFailover) is the most up-to-date replica.
orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer
orchestrator-client -c stop-replica -i 127.0.0.1:10113
sleep 3
orchestrator-client -c stop-replica -i 127.0.0.1:10112
sleep 3
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
while sleep 0.5 ; do orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' ; echo ""; done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.