deploy.py

#!/bin/python
import argparse
import tempfile
import shutil
import subprocess
import sys
import traceback
import requests
import logging
import re
from six import StringIO

DEFAULT_ANSIBLE_SOURCE = \
    "https://gitlab.local/eng-ops/ansible.git"
DEFAULT_INTEGRATION_TESTS_SOURCE = \
    "https://gitlab.local/osbs/osbs-integration-tests.git"
ANSIBLE_PATHS = {
    "stage": {
        "brew": "playbooks/brew/stage",
        "osbs": "playbooks/osbs/qa",
        "osbs-rcm": "playbooks/osbs/qa/rcm"
    },
    "prod": {
        "brew": "playbooks/brew/prod",
        "osbs": "playbooks/osbs/prod",
        "osbs-rcm": "playbooks/osbs/prod/rcm"
    }
}
BREW_GROUP = {
    "stage": "brew-stage-osbs",
    "prod": "brew-prod-osbs"
}
BREW_BINARIES = {
    "stage": "brew-stage",
    "prod": "brew"
}
OSBS_BOX_BUILDERS = {
    "stage": "stage.local",
    "prod": "prod.local"
}
OSBS_BOX_CLUSTERS = {
    "stage": "stage-osbs.local",
    "prod": "prod-osbs.local"
}
OSBS_BOX_WORKERS = {
    "stage": "stage-worker.local",
    "prod": "prod-worker.local"
}

TOOLS = {
    "ansible": {
        "version_command": ["ansible", "--version"],
        "version_regexp": r"ansible (.+?)\s",
        "minimum_version": "2.3"
    },
    "docker": {
        "version_command": ["docker", "version"],
        "version_regexp": r"Version\:\s+(.+?)\s",
        "minimum_version": "1.10"
    },
    "oc": {
        "version_command": ["oc", "version"],
        "version_regexp": r"oc v(.+?)\s",
        "minimum_version": "1.5"
    },
    "python-dns": {
        "version_command":
            ["rpm", "-q", "--whatprovides", "python2dist(dnspython)"],
        "version_regexp": r"python[2]*-dns-(.+?)-",
        "minimum_version": "1.15.0"
    },
    "libselinux-python": {
        "version_command": ["rpm", "-qa", "--provides", "libselinux-python"],
        "version_regexp": r"libselinux-python \= (.+?)-",
        "minimum_version": "2.5"
    }
}


ansible_cli = ["ansible", "-v"]
ansible_playbook_cli = ["ansible-playbook", "-v"]

settings = {}

# Print logging to output and logging
logging_stream = StringIO()
stdout_handler = logging.StreamHandler()
stream_handler = logging.StreamHandler(logging_stream)
log = logging.getLogger('deploy')
log.setLevel(logging.DEBUG)
log.addHandler(stdout_handler)
log.addHandler(stream_handler)


def _wait_for_confirmation():
    if settings["is_osbs_box"]:
        return True

    yes = set(['yes', 'y'])
    no = set(['no', 'n'])

    choice = raw_input().lower()
    if choice in yes:
        return True
    elif choice in no:
        return False
    else:
        sys.stdout.write("Please respond with 'yes' or 'no'")


def _run(cmd, cwd=None, print_output=True, ignore_exitcode=None):
    kwargs = {}
    if cwd:
        kwargs['cwd'] = cwd
    log.info("Running '{}'".format(" ".join(cmd)))

    output = ''
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT, **kwargs)
    while True:
        line = proc.stdout.readline()
        if line != b'':
            decoded_line = line.rstrip().decode('utf-8')
            if print_output:
                log.debug(decoded_line)
            output += decoded_line
        else:
            break
    # Run poll to set returncode
    proc.wait()
    ignored_exitcodes = [0, ignore_exitcode]
    if proc.returncode not in ignored_exitcodes:
        raise RuntimeError('Command {0} failed with exitcode {1}'.format(
            cmd, proc.returncode))

    return output


def _rename_container(src, dest):
    # Check that container "dest" doesn't exist
    try:
        cmd = ["sudo", "docker", "inspect", dest]
        _run(cmd, print_output=False)
    except RuntimeError:
        pass
    else:
        return

    # Check that container "src" doesn't exist
    cmd = ["sudo", "docker", "inspect", src]
    _run(cmd, print_output=False)

    # Rename container
    cmd = ["sudo", "docker", "rename", src, dest]
    _run(cmd, print_output=False)


def _send_to_hastebin(content):
    url = 'http://haste.local/'
    response = requests.post("{}documents".format(url), data=content)
    pastebin_url = "{0}{1}".format(url, response.json()['key'])
    log.info("Logs are available at {}".format(pastebin_url))
    log.info("")


def parse_arguments():
    global ansible_cli, ansible_playbook_cli
    parser = argparse.ArgumentParser()
    parser.add_argument("platform", choices=["stage", "prod"])
    parser.add_argument("--osbs-box", action="store_true")
    parser.add_argument("--ansible-remote", default=DEFAULT_ANSIBLE_SOURCE)
    parser.add_argument("--ansible-branch", default="master")
    parser.add_argument("--fake-brew-stage", action="store_true")
    parser.add_argument("--testbuild", action="store_true")
    parser.add_argument("--integration-tests", action="store_true")
    parser.add_argument("--integration-tests-remote",
                        default=DEFAULT_INTEGRATION_TESTS_SOURCE)
    parser.add_argument("--integration-tests-branch", default="master")
    parser.add_argument("--integration-tests-params", default="")
    parsed = parser.parse_args()
    log.info("Running deploy on {0}".format(parsed.platform))

    if parsed.osbs_box:
        ansible_cli = ["sudo", "ansible", "-c", "docker"]
        ansible_playbook_cli = ["sudo", "ansible-playbook", "-c", "docker"]

    if parsed.integration_tests_remote != DEFAULT_INTEGRATION_TESTS_SOURCE or \
            parsed.integration_tests_branch != "master":
        parsed.integration_tests = True
    return {"platform": parsed.platform,
            "is_osbs_box": parsed.osbs_box,
            "ansible-remote": parsed.ansible_remote,
            "ansible-branch": parsed.ansible_branch,
            "fake-brew-stage": parsed.fake_brew_stage,
            "testbuild": parsed.testbuild,
            "integration-tests": parsed.integration_tests,
            "integration-tests-remote": parsed.integration_tests_remote,
            "integration-tests-branch": parsed.integration_tests_branch,
            "integration-tests-params": parsed.integration_tests_params,
            }


def check_no_venv():
    if hasattr(sys, 'real_prefix'):
        raise RuntimeError(
            "venv detected, sys.prefix: {}, sys.real_prefix: {}".format(
                sys.prefix, sys.real_prefix))


def check_tool_versions():
    for tool, tool_details in TOOLS.items():
        log.info("Checking tool '{}'".format(tool))

        # Run version command
        cmd = tool_details["version_command"]
        version_output = _run(cmd, ignore_exitcode=1)

        match = re.search(tool_details["version_regexp"], version_output)
        if not match:
            raise RuntimeError(
                "Failed to find version by regexp '{0}' in '{1}'".format(
                    tool_details["version_regexp"], version_output))

        actual_version = match.group(1)
        if actual_version < tool_details["minimum_version"]:
            raise RuntimeError(
                "Found version '{}', but expected to be at least '{}'".format(
                    actual_version, tool_details["minimum_version"]))
        else:
            log.info("Version '{}' fits a minumum requirement of '{}'".format(
                actual_version, tool_details["minimum_version"]))


def check_brew_hello():
    if settings["is_osbs_box"]:
        log.info("OSBS Box, skipping brew hello check")
        return

    brew_bin = BREW_BINARIES[settings["platform"]]
    log.info("Checking that '{} hello' works".format(brew_bin))
    _run([brew_bin, "hello"])


def generate_temp_dir_name():
    result = tempfile.mkdtemp()
    log.info("Temporary directory is {}".format(result))
    return result


def git_clone_ansible_repo(tempdir):
    cmd = ["git", "clone",
           "-b", settings["ansible-branch"],
           settings["ansible-remote"], tempdir]
    _run(cmd)

    cmd = ["git", "log", "--pretty=oneline", "--decorate", "-1"]
    _run(cmd, cwd=tempdir)


def fetch_ansible_roles(tempdir):
    for path in ANSIBLE_PATHS[settings["platform"]].values():
        if settings["fake-brew-stage"]:
            rolesfile_path = "{}/rolesfile.yml".format(path)

            # Replace brew-stage with fake brew repo
            cmd = ["sed", "-i",
                   "s;shared-access/brew-stage.git;"
                   "vrutkovs/fake-brew-stage.git"
                   "\\n  name: brew-stage;g",
                   rolesfile_path]
            _run(cmd, cwd=tempdir, print_output=False)

        cmd = ["ansible-galaxy", "install", "-r",
               "{}/rolesfile.yml".format(path),
               "-p", "{}/roles".format(path)]
        log.info("")
        _run(cmd, cwd=tempdir)
    log.info("")


def get_builders():
    if settings["is_osbs_box"]:
        log.info("OSBS Box, skipping builders state fetch")

        # Rename koji_builder_1 container to a hostname of the builder
        builder_name = OSBS_BOX_BUILDERS[settings["platform"]]
        _rename_container("osbsbox_koji-builder_1", builder_name)
        return builder_name, "foo"
    else:
        raise NotImplementedError


def update_packages_on_builder(builder, tempdir):
    cmd = ansible_cli + [
        BREW_GROUP[settings["platform"]], "--limit={}".format(builder),
        "-i", "inventory/brew-infra",
        "-m", "yum", "-a", "state=latest "
        "name=osbs-client,"
        "python-osbs-client,"
        "koji-containerbuild,"
        "koji-containerbuild-builder"
    ]
    _run(cmd, cwd=tempdir)

    cmd = ansible_cli + [
        BREW_GROUP[settings["platform"]], "--limit={}".format(builder),
        "-m", "command", "-a", "rpm -qa osbs-client python-osbs-client "
        "koji-containerbuild koji-containerbuild-builder"
    ]
    _run(cmd, cwd=tempdir)

    log.info("Verify versions are correct and type yes/y or no/n")
    assert _wait_for_confirmation()


def run_builder_playbook(builder):
    path = ANSIBLE_PATHS[settings["platform"]]["brew"]
    cmd = ansible_playbook_cli + [
        "--limit={}".format(builder),
        "{}/setup-brew-builders-osbs.yml".format(path)
    ]
    _run(cmd, cwd=tempdir)
    log.info("")


def set_builder_state(builder, state):
    log.info("")
    if settings["is_osbs_box"]:
        log.info("OSBS Box, skipping builders state change")
    else:
        raise NotImplementedError


def move_builder_between_channels(builder, src, dst):
    log.info("")
    if settings["is_osbs_box"]:
        log.info("OSBS Box, skipping builders channel moving")
    else:
        raise NotImplementedError


def setup_orchestrator_namespace():
    if settings["is_osbs_box"]:
        # Rename koji_clientr_1 container to a cluster hostname
        cluster_name = OSBS_BOX_CLUSTERS[settings["platform"]]
        _rename_container("osbsbox_koji-client_1", cluster_name)

        cmd = ["sudo", "docker", "exec", cluster_name,
               "oc", "login", "https://172.17.0.1:8443",
               "-u", "osbs", "-p", "osbs", "--insecure-skip-tls-verify=true"]
        _run(cmd)
        cmd = ["sudo", "docker", "exec", cluster_name,
               "mkdir", "-p", "/var/lib/origin"]
        _run(cmd)

    path = ANSIBLE_PATHS[settings["platform"]]["osbs"]
    cmd = ansible_playbook_cli + \
        ["{}/setup_orchestrator_namespace.yml".format(path)]
    if settings["is_osbs_box"]:
        cmd += ["-e", "osbs_kubeconfig_path=/root/.kube/config"]
    _run(cmd, cwd=tempdir)
    log.info("")


def run_worker_playbook():
    path = ANSIBLE_PATHS[settings["platform"]]["osbs"]
    cmd = ansible_playbook_cli
    if settings["is_osbs_box"]:
        cmd += ["--limit='{}'".format(OSBS_BOX_WORKERS)]
    cmd += ["{}/setup_worker_namespace.yml".format(path)]
    _run(cmd, cwd=tempdir)
    log.info("")


def await_verification():
    log.info("Please wait for verification to complete and type yes/y or no/n")
    assert _wait_for_confirmation()


def update_ppc64le_builder():
    if settings["is_osbs_box"]:
        log.info("OSBS Box, skipping ppc64le builder update")
    else:
        raise NotImplementedError


def adjust_configs_for_osbs_box():
    if not settings["is_osbs_box"]:
        return

    builder_name = OSBS_BOX_BUILDERS[settings["platform"]]
    cmd = ["sudo", "docker", "exec", builder_name,
           "koji", "add-tag", "osbs-test-1.0-rhel-7-docker-build",
           "--arches=x86_64"]
    _run(cmd, print_output=False, ignore_exitcode=1)

    cmd = ["sudo", "docker", "exec", builder_name,
           "koji", "add-tag", "osbs-test-1.0-rhel-7-docker-candidate",
           "--arches=x86_64"]
    _run(cmd, print_output=False, ignore_exitcode=1)

    cmd = ["sudo", "docker", "exec", builder_name,
           "koji", "add-target", "osbs-test-1.0-rhel-7-docker-candidate",
           "osbs-test-1.0-rhel-7-docker-build",
           "osbs-test-1.0-rhel-7-docker-candidate"]
    _run(cmd, print_output=False, ignore_exitcode=1)

    cmd = ["sudo", "docker", "exec", builder_name,
           "koji", "add-pkg", "osbs-test-1.0-rhel-7-docker-candidate",
           "osbs-test-sandwich-docker",
           "--owner", "kojiadmin"]
    _run(cmd, print_output=False, ignore_exitcode=1)

    cmd = ["sudo", "docker", "exec", builder_name,
           "koji", "add-pkg", "osbs-test-1.0-rhel-7-docker-candidate",
           "osbs-test-hamburger-docker",
           "--owner", "kojiadmin"]
    _run(cmd, print_output=False, ignore_exitcode=1)

    # Fetch token from client
    cluster_name = OSBS_BOX_CLUSTERS[settings["platform"]]
    cmd = ["sudo", "docker", "exec", cluster_name,
           "oc", "login", "--insecure-skip-tls-verify=true",
           "-u", "osbs", "-p", "osbs",
           "https://172.17.0.1:8443/"]
    _run(cmd, print_output=False)

    cmd = ["sudo", "docker", "exec", cluster_name, "oc", "whoami", "-t"]
    token = _run(cmd, print_output=False)

    cmd = ["sudo", "docker", "exec", cluster_name,
           "oc", "label", "node", "--all", "orchestrator=true",
           "--overwrite=true"]
    _run(cmd, print_output=False)

    # Store token on builder
    cmd = ["sudo", "docker", "exec", builder_name,
           "sed", "-i", "1c{}".format(token), "/etc/osbs/koji.token"]
    _run(cmd, print_output=False)

    # Update orchestrator URL on builder
    cmd = ["sudo", "docker", "exec", builder_name,
           "sed", "-i",
           "s;openshift_uri.*;openshift_uri = https://172.17.0.1:8443;g",
           "/etc/osbs.conf"]
    _run(cmd, print_output=False)

    if settings["testbuild"]:
        cmd = ["sudo", "docker", "exec", cluster_name,
               "koji-containerbuild", "container-build",
               "osbs-test-1.0-rhel-7-docker-candidate",
               "git://git.local"
               "/rpms/osbs-test-sandwich-docker"
               "#a5eb921e5216856569120837427f92ebbf096bc0",
               "--git-branch", "buildroot-smoketest",
               "--scratch",
               "--wait"]
        _run(cmd)
    else:
        log.info("Skipping test build")


def run_integration_tests():
    if not settings['integration-tests']:
        log.info("Skipping integration tests")
    else:
        cluster_name = OSBS_BOX_CLUSTERS[settings["platform"]]
        cmd = ["sudo", "docker", "exec", cluster_name,
               "rm", "-rf", "/tmp/tests"]
        _run(cmd)

        cmd = ["sudo", "docker", "exec", cluster_name,
               "env", "GIT_SSL_NO_VERIFY=1",
               "git", "clone", "-b", settings['integration-tests-branch'],
               settings['integration-tests-remote'], "/tmp/tests"]
        _run(cmd)

        cmd = ["sudo", "docker", "exec", cluster_name,
               "/tmp/tests/osbs-box.sh", settings["integration-tests-params"]]
        _run(cmd)


def pastebin_logs_and_exception(exception=False):
    content = logging_stream.getvalue()
    if exception:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        exception_list = traceback.format_exception(exc_type, exc_value,
                                                    exc_traceback)
        content += ''.join(exception_list)
    _send_to_hastebin(content)


def cleanup_dir(tempdir):
    log.info("Removing {}".format(tempdir))
    shutil.rmtree(tempdir)


settings = parse_arguments()
tempdir = generate_temp_dir_name()

try:
    check_no_venv()
    check_tool_versions()
    check_brew_hello()
    git_clone_ansible_repo(tempdir)
    fetch_ansible_roles(tempdir)
    idle_builder, live_builder = get_builders()
    update_packages_on_builder(idle_builder, tempdir)
    run_builder_playbook(idle_builder)
    set_builder_state(live_builder, False)
    move_builder_between_channels(live_builder, src="container", dst="testing")
    setup_orchestrator_namespace()
    #run_worker_playbook()
    move_builder_between_channels(idle_builder, src="container", dst="testing")
    set_builder_state(idle_builder, True)
    await_verification()
    move_builder_between_channels(idle_builder, src="testing", dst="container")
    update_ppc64le_builder()
    await_verification()
    adjust_configs_for_osbs_box()
    run_integration_tests()
    cleanup_dir(tempdir)
    pastebin_logs_and_exception(exception=False)
except Exception:
    pastebin_logs_and_exception(exception=True)
    raise