Deployments with temouts, abort, non-blocking with paramiko

deployment_v3.py

def run_deployment_v3(deployment: Deployment):
    """Runs deployment as 'app' user with separate commands passing commands via base64 scripts,
    with timeouts and possibility to abort deployment"""
    if deployment.is_aborted:
        print("deployment is aborted")
        deployment.log("Deployment was aborted")
        deployment.status = DEPLOYMENT_STATUS.aborted
        deployment.save(update_fields=['status', ])
        return "Aborted"
    server = deployment.application.server  # type: Server
    rsa_key = server.user.private_key  # type: str
    deployment.log(f"ran run_deployment_v3 for deployment {deployment.id}")
    username = 'app'
    deployment.log(f'??? Connecting to {username}@{server.ip_address}')
    try:
        pkey = paramiko.RSAKey.from_private_key(StringIO(rsa_key))
        client = paramiko.SSHClient()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        client.connect(server.ip_address, username=username, pkey=pkey, timeout=5)

        build_script = base64_command(make_build_script(deployment=deployment))
        deployment.stage = DEPLOYMENT_STAGE.build
        deployment.save(update_fields=['stage', ])
        last_exit_code = -1

        # build step
        start = time.time()
        channel = client.get_transport().open_session()
        channel.exec_command(build_script)
        while time.time() - start < 30 * 60:
            if deployment.is_aborted:
                raise DeploymentAborted()
            if channel.exit_status_ready():
                print("received exit_status_ready()")
                last_exit_code = channel.recv_exit_status()
                break
            rl, wl, xl = select.select([channel], [], [], 0.1)
            if len(rl) > 0:
                deployment.log(channel.recv(2048).decode())
            time.sleep(0.1)
        channel.close()
        print(f"last_exit_code: {last_exit_code}")

        # deploy step
        if last_exit_code == 0:
            deployment_script = base64_command(make_deployment_script(deployment=deployment))
            deployment.stage = DEPLOYMENT_STAGE.deploy
            deployment.save(update_fields=['stage', ])
            start = time.time()
            channel = client.get_transport().open_session()
            channel.exec_command(deployment_script)
            while time.time() - start < 20 * 60:
                if deployment.is_aborted:
                    raise DeploymentAborted()
                if channel.exit_status_ready():
                    print("received exit_status_ready()")
                    last_exit_code = channel.recv_exit_status()
                    break
                rl, wl, xl = select.select([channel], [], [], 0.1)
                if len(rl) > 0:
                    deployment.log(channel.recv(2048).decode())
                time.sleep(0.1)
            channel.close()
            print(f"last_exit_code: {last_exit_code}")

        # release step
        if last_exit_code == 0:

            if not deployment.application.skip_release_command:
                release_script = base64_command(make_release_script(deployment=deployment))
                deployment.stage = DEPLOYMENT_STAGE.release
                deployment.save(update_fields=['stage', ])
                start = time.time()
                channel = client.get_transport().open_session()
                channel.exec_command(release_script)
                while time.time() - start < 20 * 60:
                    if deployment.is_aborted:
                        raise DeploymentAborted()
                    if channel.exit_status_ready():
                        print("received exit_status_ready()")
                        last_exit_code = channel.recv_exit_status()
                        break
                    rl, wl, xl = select.select([channel], [], [], 0.1)
                    if len(rl) > 0:
                        deployment.log(channel.recv(2048).decode())
                    time.sleep(0.1)
                channel.close()
                print(f"last_exit_code: {last_exit_code}")

        # clean up step
        if last_exit_code == 0:
            cleanup_script = base64_command(make_cleanup_script())
            deployment.stage = DEPLOYMENT_STAGE.cleanup
            deployment.save(update_fields=['stage', ])
            start = time.time()
            channel = client.get_transport().open_session()
            channel.exec_command(cleanup_script)
            while time.time() - start < 1 * 60:
                if deployment.is_aborted:
                    raise DeploymentAborted()
                if channel.exit_status_ready():
                    print("received exit_status_ready()")
                    # We don't care about exit code of clean up process, since it has little to do with the app
                    # last_exit_code = channel.recv_exit_status()
                    break
                rl, wl, xl = select.select([channel], [], [], 0.1)
                if len(rl) > 0:
                    deployment.log(channel.recv(2048).decode())
                time.sleep(0.1)
            channel.close()
            print(f"last_exit_code: {last_exit_code}")
        client.close()
        deployment.log(f"Connection closed to the server. Exit code: {last_exit_code}")
        if last_exit_code == 0:
            deployment.status = DEPLOYMENT_STATUS.deployed
        elif last_exit_code == -1:
            deployment.status = DEPLOYMENT_STATUS.timeout
        else:
            deployment.status = DEPLOYMENT_STATUS.failed
        deployment.log(f'Deployment finished with exit_code: {last_exit_code}')
        deployment.finished_dt = arrow.now().datetime
        deployment.save()
    except AuthenticationException as e:
        deployment.log(f"!!! Authentication failed: {e}")
        deployment.status = DEPLOYMENT_STATUS.failed
        deployment.finished_dt = arrow.now().datetime
        message = str(e)
        exception_type = type(e).__name__
        deployment.error_message = f"{exception_type}: {message}"
        deployment.save()
        return f"Failed {str(e)}"
    except SSHException as e:
        deployment.log(f"!!! SSHException: {e}")
        deployment.status = DEPLOYMENT_STATUS.failed
        deployment.finished_dt = arrow.now().datetime
        message = str(e)
        exception_type = type(e).__name__
        deployment.error_message = f"{exception_type}: {message}"
        deployment.save()
        return f"Failed {str(e)}"
    except DeploymentAborted:
        deployment.log(f"!!! Deployment Aborted")
        deployment.status = DEPLOYMENT_STATUS.aborted
        deployment.finished_dt = arrow.now().datetime
        deployment.error_message = f"Deployment aborted"
        deployment.save()
        return f"Aborted"
    except Exception as e:
        deployment.status = DEPLOYMENT_STATUS.failed
        deployment.finished_dt = arrow.now().datetime
        message = str(e)
        exception_type = type(e).__name__
        deployment.error_message = f"{exception_type}: {message}"
        deployment.log(f"!!! Exception: {exception_type}: {message}")
        deployment.save()
        return f"Failed {str(e)}"
    return "Succeeded"