guedressel/freistilbox-deploy-check.py

## freistilbox-deploy-check.py
#!/usr/bin/env python3

import os
import sys
import re
import argparse
from datetime import timedelta
from datetime import datetime


deploy_log_line_pattern = re.compile(r'^[A-Z], \[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})#(\d+)\] \s*([\w]+) \S+ : (.+)$')

job_success_msg = 'Deployment finished.'
job_fail_msg = 'Aborting deployment.'

JOB_STATUS_RUNNING = 0
JOB_STATUS_SUCCEEDED = 1
JOB_STATUS_FAILED = 2
JOB_STATUS_LABEL = {
  JOB_STATUS_RUNNING: 'running',
  JOB_STATUS_FAILED: 'fail',
  JOB_STATUS_SUCCEEDED: 'success'
}


def read_file(file_path, continuous=False):

  if not continuous:
    with open(file_path,'r') as file_handle:
      for line in file_handle:
        line = line.strip()
        if line:
          yield line
  else:
    with open(file_path,'r') as file_handle:
      while not file_handle.closed:
        line = file_handle.readline().strip()
        if line:
          yield line


class ParsingError(Exception):
  def __init__(self, raw_line):
        super(ParsingError, self).__init__("Parsing error!! Raw input line: %s" % raw_line)


class DeployJob:

  def __init__(self, job_id, status, begin, end, log_entries):
    """

    """
    self.job_id = job_id
    self.status = status
    self.begin = begin
    self.end = end
    self.log = log_entries


  @property
  def completed(self) -> bool:
    return self.status in [JOB_STATUS_FAILED, JOB_STATUS_SUCCEEDED]

  @property
  def duration(self) -> timedelta:
    if self.begin and self.end:
      return self.end - self.begin
    else:
      return None


def read_last_job(file_path, since=None, commit_hash=None, continuous=False):

  last_job_id = None
  last_job_log_entries = None
  last_job_status = None
  last_job_begin = None
  last_job_end = None
  last_job_commit_hash_found = False

  for line in read_file(file_path, continuous):

    if not line or line.startswith('#'):
      # skip empty or commented lines
      continue
    parsed_line = deploy_log_line_pattern.match(line)
    if not parsed_line:
      raise ParsingError(line)

    log_time = datetime.strptime(parsed_line.group(1), r'%Y-%m-%d %H:%M:%S')
    job_id = parsed_line.group(2)
    log_level = parsed_line.group(3)
    log_msg = parsed_line.group(4)

    if since and log_time < since:
      # skip entry if it's out of scope
      continue

    if job_id != last_job_id:
      # found a new job
      last_job_id = job_id
      last_job_log_entries = []
      last_job_status = JOB_STATUS_RUNNING
      last_job_begin = log_time

    last_job_log_entries.append({'time': log_time, 'level': log_level, 'msg': log_msg})

    if log_msg == job_fail_msg:
      last_job_status = JOB_STATUS_FAILED
      last_job_end = log_time
      if last_job_commit_hash_found or (not commit_hash and continuous):
        break;
    elif log_msg == job_success_msg:
      last_job_status = JOB_STATUS_SUCCEEDED
      last_job_end = log_time
      if last_job_commit_hash_found or (not commit_hash and continuous):
        break;
    elif commit_hash and log_msg.startswith("Current release is %s" % commit_hash):
      last_job_commit_hash_found = True

  if not last_job_id:
    return None
  elif commit_hash and not last_job_commit_hash_found:
    return None
  else:
    return DeployJob(last_job_id, last_job_status, last_job_begin, last_job_end, last_job_log_entries)


if __name__ == "__main__":

  parser = argparse.ArgumentParser(description='Freistilbox deployment log interpreter.')
  parser.add_argument('--file', '-f', default='~/.deploy/deploy.log', help='Path to log file.')
  parser.add_argument('--quiet', '-q', action='store_true', help='Don\'t write to stdout.')
  parser.add_argument('--verbose', '-v', action='store_true', help='Show job details. (Ignored if --quiet is set)')
  parser.add_argument('--since', '-s', type=int, metavar='TIMESTAMP', help='Don\'t interpret log entries before unix timestamp.')
  parser.add_argument('--commit', '-c', metavar='HASH', help='Look for job triggered by specific git commit.')
  parser.add_argument('--wait', '-w', action='store_true', help='Keep log open until last job completes. This is ignored if last job already completed.')
  # TODO: add wait-timeout argument (and an implementation for it)

  args = parser.parse_args()

  file_path = os.path.expanduser(args.file)
  if not os.path.exists(file_path):
    parser.error("File not found: %s" % file_path)

  if args.quiet:
    def stdout(msg=""):
      pass
  else:
    def stdout(msg=""):
      print(msg)
  def stderr(msg):
    print(msg, file=sys.stderr)

  try:

    since = None
    if args.since:
      since = datetime.fromtimestamp(args.since)

    job = read_last_job(file_path, since, args.commit)

    if args.wait:
      if args.commit and job and job.completed:
        # Already got the completed job triggered by args.commit.
        # No need to wait any longer - continue right away!
        pass
      elif job and job.completed:
        # Got time of last completed job in past.
        # Now wait for next job and continue once it wrote its "job complete" message into the log file.
        job = read_last_job(file_path, job.end + timedelta(microseconds=1), None, True)
      else:
        # Last job in past did not yet complete (or no job found at all since log file was empty).
        # Wait for the next "job complete" message in the log file and then continue.
        job = read_last_job(file_path, since, args.commit, True)

    if not job:
      msg = "No deployment job found"
      if args.commit:
        msg += " for git commit %s" % args.commit
      if since:
        msg += " since %s" % since.strftime("%Y-%m-%d %H:%M:%S")
      stderr("%s." % msg)
      sys.exit(23)

    if args.verbose:
      if since:
        stdout("Checked log entries since %s." % since.strftime("%Y-%m-%d %H:%M:%S"))
      stdout()
      if args.commit:
        stdout("Job triggered by git commit %s" % args.commit)
      else:
        stdout("Most recent job")
      stdout("  id: %s" % job.job_id)
      stdout("  status: %s" % JOB_STATUS_LABEL.get(job.status, 'unknown').upper())
      if job.completed:
        since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
        till = job.end.strftime("%H:%M:%S")
        stdout("  from %s to %s (%s)" % (since, till, job.duration))
      else:
        since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
        duration = (datetime.now() - job.begin)
        stdout("  running since %s (%s)" % (since, duration))
      stdout()

    for log_entry in job.log:
      stdout("%s - %s: %s" % (log_entry['time'].strftime('%c'), log_entry['level'], log_entry['msg']))

    if job.status == JOB_STATUS_SUCCEEDED:
      sys.exit(0)
    elif job.status == JOB_STATUS_RUNNING:
      sys.exit(21)
    elif job.status == JOB_STATUS_FAILED:
      sys.exit(22)
    else:
      sys.exit(24)

  except PermissionError as e:
    stderr(e)
    sys.exit(1)
  except ParsingError as e:
    stderr(e)
    sys.exit(13)
  except KeyboardInterrupt as e:
    stdout()
    sys.exit(1)

## freistilbox-mysql-dump.sh
#!/bin/bash

set -e
set -o pipefail


[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox cluster ID defined."; exit 1; }
CLUSTER=$1
shift

[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox site ID defined."; exit 1; }
SITE=$1
shift

SSH_HOST="${CLUSTER}s.freistilbox.net"

DRUSH_CMD="/usr/local/bin/drush"
FSB_DOCROOT="/srv/www/freistilbox/clients/c11000/${SITE}/current/docroot/"
DUMP_OPTIONS="--extra-dump=--opt --extra-dump=--hex-blob --extra-dump=--skip-comments"

echo "Creating MySQL Dump of site ${SITE} on Freistilbox cluster ${CLUSTER}"

ssh -C ${SITE}@${SSH_HOST} "${DRUSH_CMD} --root=$FSB_DOCROOT sql:dump $DUMP_OPTIONS" | gzip --no-name > ${SITE}_dump.sql.gz
	#!/usr/bin/env python3

	import os
	import sys
	import re
	import argparse
	from datetime import timedelta
	from datetime import datetime


	deploy_log_line_pattern = re.compile(r'^[A-Z], \[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})#(\d+)\] \s*([\w]+) \S+ : (.+)$')

	job_success_msg = 'Deployment finished.'
	job_fail_msg = 'Aborting deployment.'

	JOB_STATUS_RUNNING = 0
	JOB_STATUS_SUCCEEDED = 1
	JOB_STATUS_FAILED = 2
	JOB_STATUS_LABEL = {
	JOB_STATUS_RUNNING: 'running',
	JOB_STATUS_FAILED: 'fail',
	JOB_STATUS_SUCCEEDED: 'success'
	}



	def read_file(file_path, continuous=False):

	if not continuous:
	with open(file_path,'r') as file_handle:
	for line in file_handle:
	line = line.strip()
	if line:
	yield line
	else:
	with open(file_path,'r') as file_handle:
	while not file_handle.closed:
	line = file_handle.readline().strip()
	if line:
	yield line


	class ParsingError(Exception):
	def __init__(self, raw_line):
	super(ParsingError, self).__init__("Parsing error!! Raw input line: %s" % raw_line)


	class DeployJob:

	def __init__(self, job_id, status, begin, end, log_entries):
	"""

	"""
	self.job_id = job_id
	self.status = status
	self.begin = begin
	self.end = end
	self.log = log_entries


	@property
	def completed(self) -> bool:
	return self.status in [JOB_STATUS_FAILED, JOB_STATUS_SUCCEEDED]

	@property
	def duration(self) -> timedelta:
	if self.begin and self.end:
	return self.end - self.begin
	else:
	return None


	def read_last_job(file_path, since=None, commit_hash=None, continuous=False):

	last_job_id = None
	last_job_log_entries = None
	last_job_status = None
	last_job_begin = None
	last_job_end = None
	last_job_commit_hash_found = False

	for line in read_file(file_path, continuous):

	if not line or line.startswith('#'):
	# skip empty or commented lines
	continue
	parsed_line = deploy_log_line_pattern.match(line)
	if not parsed_line:
	raise ParsingError(line)

	log_time = datetime.strptime(parsed_line.group(1), r'%Y-%m-%d %H:%M:%S')
	job_id = parsed_line.group(2)
	log_level = parsed_line.group(3)
	log_msg = parsed_line.group(4)

	if since and log_time < since:
	# skip entry if it's out of scope
	continue

	if job_id != last_job_id:
	# found a new job
	last_job_id = job_id
	last_job_log_entries = []
	last_job_status = JOB_STATUS_RUNNING
	last_job_begin = log_time

	last_job_log_entries.append({'time': log_time, 'level': log_level, 'msg': log_msg})

	if log_msg == job_fail_msg:
	last_job_status = JOB_STATUS_FAILED
	last_job_end = log_time
	if last_job_commit_hash_found or (not commit_hash and continuous):
	break;
	elif log_msg == job_success_msg:
	last_job_status = JOB_STATUS_SUCCEEDED
	last_job_end = log_time
	if last_job_commit_hash_found or (not commit_hash and continuous):
	break;
	elif commit_hash and log_msg.startswith("Current release is %s" % commit_hash):
	last_job_commit_hash_found = True

	if not last_job_id:
	return None
	elif commit_hash and not last_job_commit_hash_found:
	return None
	else:
	return DeployJob(last_job_id, last_job_status, last_job_begin, last_job_end, last_job_log_entries)



	if __name__ == "__main__":

	parser = argparse.ArgumentParser(description='Freistilbox deployment log interpreter.')
	parser.add_argument('--file', '-f', default='~/.deploy/deploy.log', help='Path to log file.')
	parser.add_argument('--quiet', '-q', action='store_true', help='Don\'t write to stdout.')
	parser.add_argument('--verbose', '-v', action='store_true', help='Show job details. (Ignored if --quiet is set)')
	parser.add_argument('--since', '-s', type=int, metavar='TIMESTAMP', help='Don\'t interpret log entries before unix timestamp.')
	parser.add_argument('--commit', '-c', metavar='HASH', help='Look for job triggered by specific git commit.')
	parser.add_argument('--wait', '-w', action='store_true', help='Keep log open until last job completes. This is ignored if last job already completed.')
	# TODO: add wait-timeout argument (and an implementation for it)

	args = parser.parse_args()

	file_path = os.path.expanduser(args.file)
	if not os.path.exists(file_path):
	parser.error("File not found: %s" % file_path)

	if args.quiet:
	def stdout(msg=""):
	pass
	else:
	def stdout(msg=""):
	print(msg)
	def stderr(msg):
	print(msg, file=sys.stderr)

	try:

	since = None
	if args.since:
	since = datetime.fromtimestamp(args.since)

	job = read_last_job(file_path, since, args.commit)

	if args.wait:
	if args.commit and job and job.completed:
	# Already got the completed job triggered by args.commit.
	# No need to wait any longer - continue right away!
	pass
	elif job and job.completed:
	# Got time of last completed job in past.
	# Now wait for next job and continue once it wrote its "job complete" message into the log file.
	job = read_last_job(file_path, job.end + timedelta(microseconds=1), None, True)
	else:
	# Last job in past did not yet complete (or no job found at all since log file was empty).
	# Wait for the next "job complete" message in the log file and then continue.
	job = read_last_job(file_path, since, args.commit, True)

	if not job:
	msg = "No deployment job found"
	if args.commit:
	msg += " for git commit %s" % args.commit
	if since:
	msg += " since %s" % since.strftime("%Y-%m-%d %H:%M:%S")
	stderr("%s." % msg)
	sys.exit(23)

	if args.verbose:
	if since:
	stdout("Checked log entries since %s." % since.strftime("%Y-%m-%d %H:%M:%S"))
	stdout()
	if args.commit:
	stdout("Job triggered by git commit %s" % args.commit)
	else:
	stdout("Most recent job")
	stdout(" id: %s" % job.job_id)
	stdout(" status: %s" % JOB_STATUS_LABEL.get(job.status, 'unknown').upper())
	if job.completed:
	since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
	till = job.end.strftime("%H:%M:%S")
	stdout(" from %s to %s (%s)" % (since, till, job.duration))
	else:
	since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
	duration = (datetime.now() - job.begin)
	stdout(" running since %s (%s)" % (since, duration))
	stdout()

	for log_entry in job.log:
	stdout("%s - %s: %s" % (log_entry['time'].strftime('%c'), log_entry['level'], log_entry['msg']))

	if job.status == JOB_STATUS_SUCCEEDED:
	sys.exit(0)
	elif job.status == JOB_STATUS_RUNNING:
	sys.exit(21)
	elif job.status == JOB_STATUS_FAILED:
	sys.exit(22)
	else:
	sys.exit(24)

	except PermissionError as e:
	stderr(e)
	sys.exit(1)
	except ParsingError as e:
	stderr(e)
	sys.exit(13)
	except KeyboardInterrupt as e:
	stdout()
	sys.exit(1)
	#!/bin/bash

	set -e
	set -o pipefail


	[ "$#" -gt 0 ] \|\| { >&2 echo "no Freistilbox cluster ID defined."; exit 1; }
	CLUSTER=$1
	shift

	[ "$#" -gt 0 ] \|\| { >&2 echo "no Freistilbox site ID defined."; exit 1; }
	SITE=$1
	shift

	SSH_HOST="${CLUSTER}s.freistilbox.net"

	DRUSH_CMD="/usr/local/bin/drush"
	FSB_DOCROOT="/srv/www/freistilbox/clients/c11000/${SITE}/current/docroot/"
	DUMP_OPTIONS="--extra-dump=--opt --extra-dump=--hex-blob --extra-dump=--skip-comments"

	echo "Creating MySQL Dump of site ${SITE} on Freistilbox cluster ${CLUSTER}"

	ssh -C ${SITE}@${SSH_HOST} "${DRUSH_CMD} --root=$FSB_DOCROOT sql:dump $DUMP_OPTIONS" \| gzip --no-name > ${SITE}_dump.sql.gz