provision_web

cb_provision_web.py

import datetime
import pathlib
import sys
import threading
import re
import subprocess
from pathlib import Path
from zipfile import ZipFile, ZIP_DEFLATED

from bottle import request, route, run, static_file
from daemon import Daemon
from pii_utils import PIIRedactionUtil

CLOUDINIT_OUTPUT_LOG_FILE = '/var/log/cloud-init-output.log'
PORT = 80
DEBUG = False
PROVISIONING_WEB_PATH = '/'
CB_PATH = Path('/opt/cloudbridge')
VARIABLE_DIR = CB_PATH / 'var'
RELEASE_FILE = CB_PATH / 'etc/cloudbridge_release.txt'

PASSWD_FILE = CB_PATH / 'var/cloudbridge_admin_password'
PASSWD_FILE_ALT = Path('/root/cloudbridge_ui_password')

TEMPLATE_FILE = CB_PATH / 'var/cloudbridge_template_name'
TEMPLATE_FILE_ALT = Path('/root/cloudbridge_template_name')

cloudinit_events_history = []
cloudinit_error_messages = []
version = ''
admin_passwd = ''

def cloudinit_log_scan_threadfn():
    regex = re.compile(r'^!!(\w+)!!\s+\[([-\w\.]*)\]\s+\[([A-Z]*)\]\s+\[(.+?)\](.*)$')

    pipe = subprocess.Popen(['tail', '-F', CLOUDINIT_OUTPUT_LOG_FILE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
    for raw_line in iter(pipe.stdout.readline,''):
        line = raw_line.rstrip().decode('UTF-8')

        m = regex.match(line)
        if not m:
            continue

        level = m.group(1).upper()
        event = m.group(4).strip()
        if level == 'STEP':
            if DEBUG:
                print('cloudinit_log_scan_thread found event: {0}'.format(event))
            event_payload = {
                'code': m.group(2),
                'event': event,
                'time': datetime.datetime.now().strftime('%c'),
            }
            cloudinit_events_history.append(event_payload)

            if 'F' in m.group(3):
                if DEBUG:
                    print('cloudinit_log_scan_thread reached final step')
                cloudinit_events_history[len(cloudinit_events_history)-1]['last'] = True
                break

        else:
            if 'U' not in m.group(3):
                continue

            message_payload = {
                'level': level,
                'code': m.group(2),
                'message': event,
                'details': m.group(5).strip(),
                'time': datetime.datetime.now().strftime('%c'),
            }
            cloudinit_error_messages.append(message_payload)

def home():
    return static_file('home.html', root=pathlib.Path().absolute())


def get_latest_event():
    client_pos = int(request.query['pos']) if 'pos' in request.query else 0
    if client_pos < len(cloudinit_events_history):
        return {
            'error': False,
            'events': cloudinit_events_history[
                client_pos : len(cloudinit_events_history)
            ],
            'time': datetime.datetime.now().strftime('%c'),
        }
    else:
        return {
            'error': True,
            'error_msg': 'not yet found next event',
            'time': datetime.datetime.now().strftime('%c'),
        }

def get_latest_error_message():
    client_pos = int(request.query['pos']) if 'pos' in request.query else 0
    if client_pos < len(cloudinit_error_messages):
        return {
            'error': False,
            'messages': cloudinit_error_messages[
                client_pos : len(cloudinit_error_messages)
            ],
            'time': datetime.datetime.now().strftime('%c'),
        }
    else:
        return {
            'error': True,
            'error_msg': 'message index out of range',
            'time': datetime.datetime.now().strftime('%c'),
        }

TEMPLATE_MUTITENANT = 'Multitenant'
TEMPLATE_PRIVATELIFT = 'Private Lift'
TEMPLATE_REGULAR = 'Regular'
def get_template_name():
    try:
        template_file = TEMPLATE_FILE if TEMPLATE_FILE.exists() else TEMPLATE_FILE_ALT
        with open(template_file) as fin:
            line = fin.readline().strip()
            if 'multitenant' in line:
                return TEMPLATE_MUTITENANT
            elif 'private_lift' in line:
                return TEMPLATE_PRIVATELIFT
            else:
                return TEMPLATE_REGULAR
    except Exception as e:
        print('Error in reading file {0}'.format(e))
    return ''

def get_version():
    global version
    if version == '':
        # get version
        short_version = ''
        with open(RELEASE_FILE) as fin:
            for line in fin:
                m = re.match(r'^\s*(\w+):\s+(\S+)', line)
                if m and m.group(1).upper() == 'VERSION':
                    short_version = m.group(2)
                    break

        # get template name
        template_name = get_template_name()
        if short_version != '' and template_name != '':
            if template_name in {TEMPLATE_MUTITENANT, TEMPLATE_PRIVATELIFT}:
                version = f'{short_version} ({template_name})'
            else:
                version = short_version

    return {'version': version}

def get_subdomain():
    subdomain = read_variable_data('cloudbridge_subdomain')
    if subdomain and len(subdomain) > 0:
        return {
            'error': False,
            'subdomain': read_variable_data('cloudbridge_subdomain'),
        }
    else:
        return {'error': True, 'subdomain': ''}


def read_variable_data(name):
    try:
        with open(VARIABLE_DIR / name, 'r') as f:
            data = f.read().strip()
            return data
    except Exception as e:
        print('Error in reading file {0}'.format(e))
    return ''

def get_admin_passwd():
    global admin_passwd
    if len(admin_passwd) < 3:
        try:
            passwd_file = PASSWD_FILE if PASSWD_FILE.exists() else PASSWD_FILE_ALT
            with open(passwd_file) as fin:
                for line in fin:
                    print('line in {0}: {1}'.format(passwd_file, line))
                    admin_passwd = line.strip()
                    break
        except Exception as e:
            print('Error in reading file {0}'.format(e))

    return admin_passwd

def download_log_cloud_init():
    raw_log_file = Path(CLOUDINIT_OUTPUT_LOG_FILE)
    log_file = Path('/tmp/cloud-init-output.log')
    try:
        pii_util = PIIRedactionUtil()

        with open(log_file, 'w') as fileout:
            with open(raw_log_file) as filein:
                for line in filein.readlines():
                    print(pii_util.redact(line.strip()), file=fileout)

    except Exception as e:
        print('failed to redact log file {}'.format(e))
        log_file = raw_log_file

    download_file = Path('/tmp/cloud-init-log.zip')
    try:
        # 7z zip with encryption
        command = ['7z', 'a', '-mem=AES256', '-y']
        admin_passwd = get_admin_passwd()
        if len(admin_passwd) > 3:
            command += ['-p{}'.format(admin_passwd)]
        command += [str(download_file), str(log_file)]
        subprocess.run(command, capture_output=True, check=True, timeout=30)
    except Exception as e:
        print('failed to invoke 7z: {}'.format(e))
        # fall back to python zip without encryption
        with ZipFile(download_file, mode='w', compression=ZIP_DEFLATED) as newzip:
            newzip.write(log_file, log_file.name)

    return static_file(
        download_file.name, root=download_file.parents[0].absolute(), download=True
    )


def start():
    route(PROVISIONING_WEB_PATH, 'GET', home)
    route(PROVISIONING_WEB_PATH + '/get_version', 'GET', get_version)
    route(PROVISIONING_WEB_PATH + '/get_latest_event', 'GET', get_latest_event)
    route(PROVISIONING_WEB_PATH + '/get_latest_error_message', 'GET', get_latest_error_message)
    route(PROVISIONING_WEB_PATH + '/get_subdomain', 'GET', get_subdomain)
    route(
        PROVISIONING_WEB_PATH + '/download_log_cloud_init',
        'GET',
        download_log_cloud_init,
    )
    threading.Thread(target=cloudinit_log_scan_threadfn).start()
    run(host='0.0.0.0', port=PORT, debug=DEBUG)


class MyDaemon(Daemon):
    def run(self):
        start()


if __name__ == '__main__':
    daemon = MyDaemon(
        str(pathlib.Path('').absolute()),
        str(pathlib.Path('./cb_provision_web.pid').absolute()),
    )
    if len(sys.argv) >= 2:
        if 'start' == sys.argv[1]:
            PROVISIONING_WEB_PATH = sys.argv[2]
            daemon.start()
        elif 'start_dev' == sys.argv[1]:
            PORT = 8080
            DEBUG = True
            PROVISIONING_WEB_PATH = sys.argv[2]
            start()
        elif 'stop' == sys.argv[1]:
            daemon.stop()
        elif 'restart' == sys.argv[1]:
            daemon.restart()
        else:
            print('Unknown command')
            sys.exit(2)
        sys.exit(0)
    else:
        print('usage: %s start|stop|restart' % sys.argv[0])
        sys.exit(2)

home.html

<!DOCTYPE html>
<html>

<head>
    <title>Conversions API Gateway Provision Status</title>
    <!-- todo@david6: replace with png  -->
    <link href="data:image/x-icon;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAABGdBTUEAALGPC/xhBQAAACBjSFJN
AAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAY1BMVEUAAABxfZVxfZRyfpVz
fZRxfpRxf5RzfpVyfZRxfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVy
fpVyfpVyfpVyfpVyfpVyfpVyfpVyfpVyfpX///+xysWEAAAAH3RSTlMAAAAAAAAAAAAACxsfBBhz
v9zg4sW+ci3A/Rx8/n3yDB72YgAAAAFiS0dEILNrPYAAAAAJcEhZcwAACxMAAAsTAQCanBgAAAAH
dElNRQflCRAWFi4PsXKlAAAAo0lEQVQ4y+2S2Q6CMBBFCxcXaCll2Nf5/7+0bdQAQYlPxsTzNMmc
dLYK8ecTgjCRSimZAqkPkgiL9Ak6MzlRUUpAlgVRXtUaT+WMpu3YQQpQ5MOubXC5C9At81ZgNto/
YQv2w7gnjFNvk4IsM+8JPLuc4CVrwXMsHJZwTU77TQ6+ST+meTOm5fpqUeFjlTF0Xa1XbTKNeHEN
RNtjhcG3P9CPcQNb3B8yFsEi6AAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMS0wOS0xNlQyMjoyMjo0
NiswMDowMCvxHpIAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjEtMDktMTZUMjI6MjI6NDYrMDA6MDBa
rKYuAAAAV3pUWHRSYXcgcHJvZmlsZSB0eXBlIGlwdGMAAHic4/IMCHFWKCjKT8vMSeVSAAMjCy5j
CxMjE0uTFAMTIESANMNkAyOzVCDL2NTIxMzEHMQHy4BIoEouAOoXEXTyQjWVAAAAAElFTkSuQmCC" rel="icon" type="image/x-icon">
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/yegor256/tacit@gh-pages/tacit-css-1.5.3.min.css" />
    <script src="https://unpkg.com/@babel/standalone/babel.min.js"></script>
    <script crossorigin src="https://unpkg.com/react@17/umd/react.production.min.js"></script>
    <script crossorigin src="https://unpkg.com/react-dom@17/umd/react-dom.production.min.js"></script>
    <link rel="stylesheet" href="https://maxbeier.github.io/text-spinners/spinners.css">
    <style>
        .events li {
            display: flex;
            color: #999;
        }

        .events li:last-child {
            color: #555;
        }

        .events time {
            position: relative;
            padding: 0 1.5em;
            font-size: 0.7em;
            white-space: nowrap;
        }

        .events time::after {
            content: "";
            position: absolute;
            z-index: 2;
            right: 0;
            top: 10px;
            transform: translateX(50%);
            border-radius: 50%;
            background: #fff;
            border: 1px #ccc solid;
            width: .8em;
            height: .8em;
        }

        .events span {
            padding: 0 1.5em 1.5em 1.5em;
            position: relative;
        }

        .events span::before {
            content: "";
            position: absolute;
            z-index: 1;
            left: 0;
            top: 10px;
            height: 100%;
            border-left: 1px #ccc solid;
        }

        .events li:last-child span::before {
            display: none;
        }

        .events strong {
            display: block;
            font-weight: bolder;
        }

        .events {
            margin: 1em;
            width: 90%;
        }

        .events,
        .events *::before,
        .events *::after {
            box-sizing: border-box;
            font-family: arial;
        }

        .messages li {
            display: flex;
            color: #999;
        }

        .messages time {
            position: relative;
            padding: 0 1.5em;
            font-size: 0.7em;
            white-space: nowrap;
        }

        .messages em {
            display: block;
            color: red;
            width: 60ch;
            word-wrap: break-word;
        }

        .messages {
            margin: 1em;
            width: 90%;
        }

        .messages,
        .messages *::before,
        .messages *::after {
            box-sizing: border-box;
            font-family: arial;
        }

        .fblogo {
            left: 68px;
            top: 0px;
            position: absolute;
            padding-top: 80px;
            font-variant: all-small-caps;
            color: #aaa;
        }

        .transparent {
            opacity: 0;
        }

        .textbox {
            display: block;
            width: 80ch;
            word-wrap: break-word;
        }

        .titlebox {
            display: block;
            white-space: nowrap;
        }
    </style>
</head>

<body>
    <div id="root-container"></div>
    <script type="text/babel" data-presets="env,stage-3,react">

        function CloudBridgeProvisionUIRedirect(props) {
            return (
                <div className="textbox">
                    <h2>Conversions API Gateway installation is finished.</h2>
                    <h3>Please setup your DNS records as explained in the CallToAction, then go back to Events Manager to complete the setup.</h3>
                </div>
            );
        }

        class CloudBridgeProvisionFailed extends React.Component {
            constructor(props) {
                super(props);
                this.state = {
                    downloaded: false
                };
            }

            download() {
                let a = document.createElement('A');
                a.href = location.href + '/download_log_cloud_init';
                a.download = 'cloud-init-log.zip';
                a.click();
                this.setState({ downloaded: true });
            }

            render() {
                return (
                    <div>
                        <h2>Conversions API Gateway Installation Failed</h2>
                        <button onClick={this.download.bind(this)}>Download Installation Log</button>
                        {
                            this.state.downloaded
                                ? <div><span>use admin password to unzip the file</span></div>
                                : ''
                        }
                    </div>
                );
            }
        }

        function CloudBridgeProvisionEvents(props) {
            return (
                <ul className="events">
                    {props.rawEvents.map(eventObj => (
                        <li key={eventObj.time}>
                            <time datetime={eventObj.time}>{eventObj.time}</time>
                            <span><strong>{eventObj.event}</strong></span>
                        </li>
                    ))}
                </ul>
            );
        }

        function CloudBridgeProvisionTitle(props) {
            if (!props.version)
                return "Conversions API Gateway is provisioning";
            else
                return "Conversions API Gateway V" + props.version + " is provisioning";
        }

        function CloudBridgeProvisionErrorMessages(props) {
            if (!props.errMessages.length)
                return "";

            return (
                <div>
                    <hr />
                    <ul className="messages">
                        {props.errMessages.map(msgObj => (
                            <li key={msgObj.time}>
                                <time datetime={msgObj.time}>{msgObj.time}</time>
                                <span><em>{msgObj.message}</em></span>
                            </li>
                        ))}
                    </ul>
                    <hr />
                </div>
            )
        }

        class CloudBridgeProvisionEventsContainer extends React.Component {
            constructor(props) {
                super(props);
                this.state = {
                    version: '',
                    rawEvents: [],
                    subDomain: '',
                    shouldShowRedirectionButton: false,
                    errMessages: [],
                    installationFailed: false,
                };
            }

            componentDidMount() {
                const fetchNewEvent = (callback) => {
                    fetch(location.href + '/get_latest_event?pos=' + this.state.rawEvents.length)
                        .then(response => response.json())
                        .then(data => {
                            if (data.error) {
                                console.log(data);
                            } else {
                                for (var eventObj of data.events) { // Please refer to return value of get_latest_event() in cb_provision_web.py
                                    if (eventObj.event) {
                                        const newRawEvents = [].concat(this.state.rawEvents);
                                        newRawEvents.push(eventObj); // Add the event to the new list
                                        this.setState({ rawEvents: newRawEvents });
                                        if (eventObj.last) { // In case of the last event arrival, show the button if subdomain has been set as well.
                                            enableRedirectionButtonIfSubDomainAndLastEventReady();
                                            return false; // Stop the fetch loop
                                        }
                                    }
                                }
                            }
                            return true;
                        })
                        .then((loop) => {
                            loop && callback && callback();
                        })
                        .catch((error) => {
                            console.error('Provisioning event fetch failed - ' + error);
                            // In case that browser is online and subDomain is ready, but fetch failed in the middle of provisioining,
                            if (navigator.onLine && this.state.rawEvents.length > 0 && this.state.subDomain.length > 0) {
                                // Provisioning web server may stopped by conversions_api_gateway_install_v2.sh to reassign 80 port to the hub.
                                // So, stop fetching loop and show the click to FSH button with given guard time.

                                // any network glitch will trigger the following logic and cause misleading result.
                                // comment out for now
                                // console.log('Conversions API Gateway may be ready soon. Stop fetching events');
                                // this.setState({ shouldShowRedirectionButton: true });
                                // return false;
                            }
                            return true; // Otherwise, keep trying to fetch
                        });
                };

                const fetchErrorMessage = (callback) => {
                    fetch(location.href + '/get_latest_error_message?pos=' + this.state.errMessages.length)
                        .then(response => response.json())
                        .then(data => {
                            if (data.error) {
                                console.log(data);
                            } else {
                                for (var msgObj of data.messages) { // Please refer to return value of get_latest_error_message() in cb_provision_web.py
                                    const newErrMessages = [].concat(this.state.errMessages);
                                    newErrMessages.push(msgObj); // Add the message to the new list
                                    this.setState({ errMessages: newErrMessages });
                                    if (msgObj.level == "ERROR") { // installation failed, show installation failed message
                                        this.setState({ installationFailed: true });
                                    }
                                }
                            }
                            return true;
                        })
                        .then((loop) => {
                            loop && callback && callback();
                        })
                        .catch((error) => {
                            console.error('Failed to fetch error messages - ' + error);
                            return true; // keep fetching
                        });
                };

                const enableRedirectionButtonIfSubDomainAndLastEventReady = () => {
                    const numRawEvents = this.state.rawEvents.length;
                    if (numRawEvents == 0) {
                        return;
                    }
                    const lastRawEvent = this.state.rawEvents[numRawEvents - 1];
                    const subDomain = this.state.subDomain;
                    if (lastRawEvent.last && subDomain.length > 0) {
                        this.setState({ shouldShowRedirectionButton: true });
                    }
                }

                const fetchSubdomain = (callback) => {
                    fetch(location.href + '/get_subdomain')
                        .then(response => response.json())
                        .then(data => {
                            if (data.error) {
                                console.log(data);
                            } else {
                                this.setState({ subDomain: data.subdomain });
                                // Show the button if the last event has arrived also.
                                enableRedirectionButtonIfSubDomainAndLastEventReady();
                                return false;
                            }
                            return true;
                        })
                        .then((loop) => {
                            loop && callback && callback();
                        });
                };

                const fetchVersion = (callback) => {
                    fetch(location.href + '/get_version')
                        .then(response => response.json())
                        .then(data => {
                            if (data.version) {
                                this.setState({ version: data.version });
                                return false;
                            }
                            return true;
                        })
                        .then((loop) => {
                            loop && callback && callback();
                        });
                }

                const fetchNewEventWithInterval = (interval, delay = 0) => {
                    if (delay) {
                        setTimeout(() => { fetchNewEventWithInterval(interval); }, delay);
                    }
                    else {
                        fetchNewEvent(() => {
                            setTimeout(() => { fetchNewEventWithInterval(interval); }, interval);
                        });
                    }
                };

                const fetchNewErrorMessageWithInterval = (interval, delay = 0) => {
                    if (delay) {
                        setTimeout(() => { fetchNewErrorMessageWithInterval(interval); }, delay);
                    }
                    else {
                        fetchErrorMessage(() => {
                            setTimeout(() => { fetchNewErrorMessageWithInterval(interval); }, interval);
                        })
                    }
                }

                const fetchSubDomainWithInterval = (interval, delay = 0) => {
                    if (delay) {
                        setTimeout(() => { fetchSubDomainWithInterval(interval); }, delay);
                    }
                    else {
                        fetchSubdomain(() => {
                            setTimeout(() => { fetchSubDomainWithInterval(interval); }, interval);
                        });
                    }
                };

                const fetchVersionWithInterval = (interval, delay) => {
                    if (delay) {
                        setTimeout(() => { fetchVersionWithInterval(interval); }, delay);
                    }
                    else {
                        fetchVersion(() => {
                            setTimeout(() => { fetchVersionWithInterval(interval); }, interval);
                        });
                    }
                }

                fetchNewEventWithInterval(2000, 1000);
                fetchNewErrorMessageWithInterval(2000, 1000);
                fetchSubDomainWithInterval(2000, 2000);
                fetchVersionWithInterval(2000);
            }

            render() {
                return (
                    <div>
                        <div>
                            <div className="fblogo">Facebook</div>
                            <h1 className="titlebox">{
                                this.state.shouldShowRedirectionButton
                                    ? <span className="loading dots transparent" />
                                    : <span className="loading dots"></span>
                            } <CloudBridgeProvisionTitle version={this.state.version} /></h1>
                        </div>
                        <hr />
                        <CloudBridgeProvisionEvents rawEvents={this.state.rawEvents} />
                        <CloudBridgeProvisionErrorMessages errMessages={this.state.errMessages} />
                        {
                            this.state.installationFailed
                                ? <CloudBridgeProvisionFailed />
                                : ''
                        }
                        {
                            this.state.shouldShowRedirectionButton
                                ? <CloudBridgeProvisionUIRedirect subDomain={this.state.subDomain} />
                                : ''
                        }
                    </div>
                );
            }
        }

        ReactDOM.render(
            <CloudBridgeProvisionEventsContainer />,
            document.getElementById('root-container')
        );
    </script>

</body>

</html>