Skip to content

Instantly share code, notes, and snippets.

@kpavlovsky
Created October 4, 2020 03:44
Show Gist options
  • Save kpavlovsky/5606ddf029d7360295f620509ae291f2 to your computer and use it in GitHub Desktop.
Save kpavlovsky/5606ddf029d7360295f620509ae291f2 to your computer and use it in GitHub Desktop.
Deployments with temouts, abort, non-blocking with paramiko
def run_deployment_v3(deployment: Deployment):
"""Runs deployment as 'app' user with separate commands passing commands via base64 scripts,
with timeouts and possibility to abort deployment"""
if deployment.is_aborted:
print("deployment is aborted")
deployment.log("Deployment was aborted")
deployment.status = DEPLOYMENT_STATUS.aborted
deployment.save(update_fields=['status', ])
return "Aborted"
server = deployment.application.server # type: Server
rsa_key = server.user.private_key # type: str
deployment.log(f"ran run_deployment_v3 for deployment {deployment.id}")
username = 'app'
deployment.log(f'??? Connecting to {username}@{server.ip_address}')
try:
pkey = paramiko.RSAKey.from_private_key(StringIO(rsa_key))
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(server.ip_address, username=username, pkey=pkey, timeout=5)
build_script = base64_command(make_build_script(deployment=deployment))
deployment.stage = DEPLOYMENT_STAGE.build
deployment.save(update_fields=['stage', ])
last_exit_code = -1
# build step
start = time.time()
channel = client.get_transport().open_session()
channel.exec_command(build_script)
while time.time() - start < 30 * 60:
if deployment.is_aborted:
raise DeploymentAborted()
if channel.exit_status_ready():
print("received exit_status_ready()")
last_exit_code = channel.recv_exit_status()
break
rl, wl, xl = select.select([channel], [], [], 0.1)
if len(rl) > 0:
deployment.log(channel.recv(2048).decode())
time.sleep(0.1)
channel.close()
print(f"last_exit_code: {last_exit_code}")
# deploy step
if last_exit_code == 0:
deployment_script = base64_command(make_deployment_script(deployment=deployment))
deployment.stage = DEPLOYMENT_STAGE.deploy
deployment.save(update_fields=['stage', ])
start = time.time()
channel = client.get_transport().open_session()
channel.exec_command(deployment_script)
while time.time() - start < 20 * 60:
if deployment.is_aborted:
raise DeploymentAborted()
if channel.exit_status_ready():
print("received exit_status_ready()")
last_exit_code = channel.recv_exit_status()
break
rl, wl, xl = select.select([channel], [], [], 0.1)
if len(rl) > 0:
deployment.log(channel.recv(2048).decode())
time.sleep(0.1)
channel.close()
print(f"last_exit_code: {last_exit_code}")
# release step
if last_exit_code == 0:
if not deployment.application.skip_release_command:
release_script = base64_command(make_release_script(deployment=deployment))
deployment.stage = DEPLOYMENT_STAGE.release
deployment.save(update_fields=['stage', ])
start = time.time()
channel = client.get_transport().open_session()
channel.exec_command(release_script)
while time.time() - start < 20 * 60:
if deployment.is_aborted:
raise DeploymentAborted()
if channel.exit_status_ready():
print("received exit_status_ready()")
last_exit_code = channel.recv_exit_status()
break
rl, wl, xl = select.select([channel], [], [], 0.1)
if len(rl) > 0:
deployment.log(channel.recv(2048).decode())
time.sleep(0.1)
channel.close()
print(f"last_exit_code: {last_exit_code}")
# clean up step
if last_exit_code == 0:
cleanup_script = base64_command(make_cleanup_script())
deployment.stage = DEPLOYMENT_STAGE.cleanup
deployment.save(update_fields=['stage', ])
start = time.time()
channel = client.get_transport().open_session()
channel.exec_command(cleanup_script)
while time.time() - start < 1 * 60:
if deployment.is_aborted:
raise DeploymentAborted()
if channel.exit_status_ready():
print("received exit_status_ready()")
# We don't care about exit code of clean up process, since it has little to do with the app
# last_exit_code = channel.recv_exit_status()
break
rl, wl, xl = select.select([channel], [], [], 0.1)
if len(rl) > 0:
deployment.log(channel.recv(2048).decode())
time.sleep(0.1)
channel.close()
print(f"last_exit_code: {last_exit_code}")
client.close()
deployment.log(f"Connection closed to the server. Exit code: {last_exit_code}")
if last_exit_code == 0:
deployment.status = DEPLOYMENT_STATUS.deployed
elif last_exit_code == -1:
deployment.status = DEPLOYMENT_STATUS.timeout
else:
deployment.status = DEPLOYMENT_STATUS.failed
deployment.log(f'Deployment finished with exit_code: {last_exit_code}')
deployment.finished_dt = arrow.now().datetime
deployment.save()
except AuthenticationException as e:
deployment.log(f"!!! Authentication failed: {e}")
deployment.status = DEPLOYMENT_STATUS.failed
deployment.finished_dt = arrow.now().datetime
message = str(e)
exception_type = type(e).__name__
deployment.error_message = f"{exception_type}: {message}"
deployment.save()
return f"Failed {str(e)}"
except SSHException as e:
deployment.log(f"!!! SSHException: {e}")
deployment.status = DEPLOYMENT_STATUS.failed
deployment.finished_dt = arrow.now().datetime
message = str(e)
exception_type = type(e).__name__
deployment.error_message = f"{exception_type}: {message}"
deployment.save()
return f"Failed {str(e)}"
except DeploymentAborted:
deployment.log(f"!!! Deployment Aborted")
deployment.status = DEPLOYMENT_STATUS.aborted
deployment.finished_dt = arrow.now().datetime
deployment.error_message = f"Deployment aborted"
deployment.save()
return f"Aborted"
except Exception as e:
deployment.status = DEPLOYMENT_STATUS.failed
deployment.finished_dt = arrow.now().datetime
message = str(e)
exception_type = type(e).__name__
deployment.error_message = f"{exception_type}: {message}"
deployment.log(f"!!! Exception: {exception_type}: {message}")
deployment.save()
return f"Failed {str(e)}"
return "Succeeded"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment