Skip to content

Instantly share code, notes, and snippets.

@guedressel
Last active March 27, 2018 17:09
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save guedressel/df33a09fc36f0cfddb713d14c2b289be to your computer and use it in GitHub Desktop.
Freistilbox utils
#!/usr/bin/env python3
import os
import sys
import re
import argparse
from datetime import timedelta
from datetime import datetime
deploy_log_line_pattern = re.compile(r'^[A-Z], \[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})#(\d+)\] \s*([\w]+) \S+ : (.+)$')
job_success_msg = 'Deployment finished.'
job_fail_msg = 'Aborting deployment.'
JOB_STATUS_RUNNING = 0
JOB_STATUS_SUCCEEDED = 1
JOB_STATUS_FAILED = 2
JOB_STATUS_LABEL = {
JOB_STATUS_RUNNING: 'running',
JOB_STATUS_FAILED: 'fail',
JOB_STATUS_SUCCEEDED: 'success'
}
def read_file(file_path, continuous=False):
if not continuous:
with open(file_path,'r') as file_handle:
for line in file_handle:
line = line.strip()
if line:
yield line
else:
with open(file_path,'r') as file_handle:
while not file_handle.closed:
line = file_handle.readline().strip()
if line:
yield line
class ParsingError(Exception):
def __init__(self, raw_line):
super(ParsingError, self).__init__("Parsing error!! Raw input line: %s" % raw_line)
class DeployJob:
def __init__(self, job_id, status, begin, end, log_entries):
"""
"""
self.job_id = job_id
self.status = status
self.begin = begin
self.end = end
self.log = log_entries
@property
def completed(self) -> bool:
return self.status in [JOB_STATUS_FAILED, JOB_STATUS_SUCCEEDED]
@property
def duration(self) -> timedelta:
if self.begin and self.end:
return self.end - self.begin
else:
return None
def read_last_job(file_path, since=None, commit_hash=None, continuous=False):
last_job_id = None
last_job_log_entries = None
last_job_status = None
last_job_begin = None
last_job_end = None
last_job_commit_hash_found = False
for line in read_file(file_path, continuous):
if not line or line.startswith('#'):
# skip empty or commented lines
continue
parsed_line = deploy_log_line_pattern.match(line)
if not parsed_line:
raise ParsingError(line)
log_time = datetime.strptime(parsed_line.group(1), r'%Y-%m-%d %H:%M:%S')
job_id = parsed_line.group(2)
log_level = parsed_line.group(3)
log_msg = parsed_line.group(4)
if since and log_time < since:
# skip entry if it's out of scope
continue
if job_id != last_job_id:
# found a new job
last_job_id = job_id
last_job_log_entries = []
last_job_status = JOB_STATUS_RUNNING
last_job_begin = log_time
last_job_log_entries.append({'time': log_time, 'level': log_level, 'msg': log_msg})
if log_msg == job_fail_msg:
last_job_status = JOB_STATUS_FAILED
last_job_end = log_time
if last_job_commit_hash_found or (not commit_hash and continuous):
break;
elif log_msg == job_success_msg:
last_job_status = JOB_STATUS_SUCCEEDED
last_job_end = log_time
if last_job_commit_hash_found or (not commit_hash and continuous):
break;
elif commit_hash and log_msg.startswith("Current release is %s" % commit_hash):
last_job_commit_hash_found = True
if not last_job_id:
return None
elif commit_hash and not last_job_commit_hash_found:
return None
else:
return DeployJob(last_job_id, last_job_status, last_job_begin, last_job_end, last_job_log_entries)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Freistilbox deployment log interpreter.')
parser.add_argument('--file', '-f', default='~/.deploy/deploy.log', help='Path to log file.')
parser.add_argument('--quiet', '-q', action='store_true', help='Don\'t write to stdout.')
parser.add_argument('--verbose', '-v', action='store_true', help='Show job details. (Ignored if --quiet is set)')
parser.add_argument('--since', '-s', type=int, metavar='TIMESTAMP', help='Don\'t interpret log entries before unix timestamp.')
parser.add_argument('--commit', '-c', metavar='HASH', help='Look for job triggered by specific git commit.')
parser.add_argument('--wait', '-w', action='store_true', help='Keep log open until last job completes. This is ignored if last job already completed.')
# TODO: add wait-timeout argument (and an implementation for it)
args = parser.parse_args()
file_path = os.path.expanduser(args.file)
if not os.path.exists(file_path):
parser.error("File not found: %s" % file_path)
if args.quiet:
def stdout(msg=""):
pass
else:
def stdout(msg=""):
print(msg)
def stderr(msg):
print(msg, file=sys.stderr)
try:
since = None
if args.since:
since = datetime.fromtimestamp(args.since)
job = read_last_job(file_path, since, args.commit)
if args.wait:
if args.commit and job and job.completed:
# Already got the completed job triggered by args.commit.
# No need to wait any longer - continue right away!
pass
elif job and job.completed:
# Got time of last completed job in past.
# Now wait for next job and continue once it wrote its "job complete" message into the log file.
job = read_last_job(file_path, job.end + timedelta(microseconds=1), None, True)
else:
# Last job in past did not yet complete (or no job found at all since log file was empty).
# Wait for the next "job complete" message in the log file and then continue.
job = read_last_job(file_path, since, args.commit, True)
if not job:
msg = "No deployment job found"
if args.commit:
msg += " for git commit %s" % args.commit
if since:
msg += " since %s" % since.strftime("%Y-%m-%d %H:%M:%S")
stderr("%s." % msg)
sys.exit(23)
if args.verbose:
if since:
stdout("Checked log entries since %s." % since.strftime("%Y-%m-%d %H:%M:%S"))
stdout()
if args.commit:
stdout("Job triggered by git commit %s" % args.commit)
else:
stdout("Most recent job")
stdout(" id: %s" % job.job_id)
stdout(" status: %s" % JOB_STATUS_LABEL.get(job.status, 'unknown').upper())
if job.completed:
since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
till = job.end.strftime("%H:%M:%S")
stdout(" from %s to %s (%s)" % (since, till, job.duration))
else:
since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
duration = (datetime.now() - job.begin)
stdout(" running since %s (%s)" % (since, duration))
stdout()
for log_entry in job.log:
stdout("%s - %s: %s" % (log_entry['time'].strftime('%c'), log_entry['level'], log_entry['msg']))
if job.status == JOB_STATUS_SUCCEEDED:
sys.exit(0)
elif job.status == JOB_STATUS_RUNNING:
sys.exit(21)
elif job.status == JOB_STATUS_FAILED:
sys.exit(22)
else:
sys.exit(24)
except PermissionError as e:
stderr(e)
sys.exit(1)
except ParsingError as e:
stderr(e)
sys.exit(13)
except KeyboardInterrupt as e:
stdout()
sys.exit(1)
#!/bin/bash
set -e
set -o pipefail
[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox cluster ID defined."; exit 1; }
CLUSTER=$1
shift
[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox site ID defined."; exit 1; }
SITE=$1
shift
SSH_HOST="${CLUSTER}s.freistilbox.net"
DRUSH_CMD="/usr/local/bin/drush"
FSB_DOCROOT="/srv/www/freistilbox/clients/c11000/${SITE}/current/docroot/"
DUMP_OPTIONS="--extra-dump=--opt --extra-dump=--hex-blob --extra-dump=--skip-comments"
echo "Creating MySQL Dump of site ${SITE} on Freistilbox cluster ${CLUSTER}"
ssh -C ${SITE}@${SSH_HOST} "${DRUSH_CMD} --root=$FSB_DOCROOT sql:dump $DUMP_OPTIONS" | gzip --no-name > ${SITE}_dump.sql.gz
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment