Skip to content

Instantly share code, notes, and snippets.

@bcarpio
Created March 30, 2012 15:35
Show Gist options
  • Save bcarpio/2252318 to your computer and use it in GitHub Desktop.
Save bcarpio/2252318 to your computer and use it in GitHub Desktop.
mongodb.rolling_upgrade Fabric Task
@task
def rolling_upgrade(hostname):
""" Rolling upgrade of mongodb """
users.root()
## Make initial connection to determine which member is the primary node
conn = Connection('%s'%(hostname))
db = conn.local
isMaster = db.command("isMaster")
primary = (isMaster['primary'])
primary = primary.replace(":27017","")
## Connect to the primary node to check health on the secondary nodes
conn = Connection('%s'%(primary))
db = conn.admin
replSet = db.command("replSetGetStatus")
## Loop through the secondary hosts
for secondary in replSet['members']:
stateStr = str(secondary['stateStr'])
host = str(secondary['name'])
host = host.replace(":27017","")
if stateStr == 'SECONDARY':
print (blue("Upgrading Host:" + blue(host)))
print " "
# Pull in updated binaries using puppet
execute(puppet.puppetd_test, host='%s' %(host))
# Restart mongodb
execute(mongod.restart, host='%s' %(host))
# Check to make sure the secondary node is recovered before moving on to the next node
# Sleep 5 just to make sure that the master has been notified that the node is down
time.sleep(5)
goodstatus = 'false'
count = 0
while goodstatus != 'true':
# Make sure that the secondary node actually recovers if it doesn't exit the python script and don't continue
if count < 100:
replSet = db.command("replSetGetStatus")
for sec in db.command("replSetGetStatus")['members']:
h = str(sec['name'])
h = h.replace(":27017","")
if h == host:
s = str(sec['state'])
if s == '2':
goodstatus = 'true'
else:
goodstatus = 'false'
count += 1
time.sleep(2)
else:
print (red("PROBLEM: Node '%s' Didn't Upgrade")%(host))
sys.exit(1)
for primary in replSet['members']:
stateStr = str(primary['stateStr'])
host = str(primary['name'])
host = host.replace(":27017","")
state = str(primary['state'])
if stateStr == 'PRIMARY':
execute(mongod.stop, host='%s' %(host))
execute(puppet.puppetd_test, host='%s' %(host))
execute(mongod.start, host='%s' %(host))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment