Last active
December 25, 2015 21:08
-
-
Save gforcada/7040082 to your computer and use it in GitHub Desktop.
Script that (re)starts a Plone instance, loads its main page and all links within the Plone instance that are on the main page (some filters provided)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| # ---------------------------------------------------------------------------- | |
| # Script to warm up the instance. | |
| # | |
| # The script starts/restarts the instance and measures the time it takes the | |
| # instance to become responsive. Afterwards it crawls the front-page for links | |
| # and warms up these resources. | |
| # ---------------------------------------------------------------------------- | |
| from datetime import datetime | |
| from email.mime.text import MIMEText | |
| from urllib2 import URLError | |
| import lxml.html | |
| import os | |
| import smtplib | |
| import subprocess | |
| import sys | |
| import time | |
| import urllib2 | |
| # Constants | |
| EMAIL_HOST = 'mailhost' | |
| EMAIL_FROM = 'server@freitag.de' | |
| EMAIL_TO = 'dev@freitag.de' | |
| EMAIL_SUBJECT = '[Freitag Server restart][{0}]' | |
| PLONE_LOCAL_PORT = '8080' | |
| PLONE_SUPERVISOR_PORT = '8081' | |
| PLONE_INSTANCE = 'website' | |
| SLEEP_SHORT = 10 | |
| SLEEP_LONG = 60 | |
| FAILED = '\033[91mFAILED\033[0m' | |
| OK = '\033[92mOK\033[0m' | |
| CHECK_EXISTS = [ | |
| 'der Freitag', | |
| 'Salon', | |
| 'Buch der Woche', | |
| 'Filmothek', | |
| 'Kollektion', | |
| 'der Freitag Mediengesellschaft mbH & Co. KG', | |
| ] | |
| CHECK_NOT_EXISTS = ['Fehler 404', ] | |
| IGNORE_MIDDLE = [ | |
| '@@', | |
| '++theme++', | |
| '#', | |
| '/front-page/', | |
| '?', | |
| ] | |
| IGNORE_END = [ | |
| '.css', | |
| '.js', | |
| '.png', | |
| '.jpg', | |
| '.jpeg', | |
| '.gif', | |
| '#', | |
| '/kiosk-finder', | |
| ] | |
| class Startup(): | |
| start_instance_time = None | |
| warmup_front_page_time = None | |
| warmup_time = None | |
| def __init__(self, base_url, has_supervisor=False): | |
| self.email_text = '' | |
| self.base_url = base_url | |
| self.has_supervisor = has_supervisor | |
| start = datetime.now() | |
| self.start_instance() | |
| self.start_instance_time = datetime.now() - start | |
| self.log('Start took: {0}.'.format(self.start_instance_time)) | |
| start = datetime.now() | |
| links = self.warmup_front_page() | |
| self.warmup_front_page_time = datetime.now() - start | |
| self.log( | |
| 'Front-page warmup took: {0}.'.format(self.warmup_front_page_time) | |
| ) | |
| #self.ab(self.base_url) | |
| start = datetime.now() | |
| self.warmup(links) | |
| self.warmup_time = datetime.now() - start | |
| self.log('Warmup took: {0}'.format(self.warmup_time)) | |
| self.elapsed_time() | |
| self.send_email() | |
| def _headline(self, headline): | |
| self.log('----------------------------------------\n') | |
| self.log(headline) | |
| self.log('----------------------------------------\n') | |
| def log(self, text): | |
| print text | |
| # remove the colorings, on a plain text mail are not seen at all | |
| # and just clutter the text | |
| for markup in ((OK, 'OK'), (FAILED, 'FAILED')): | |
| if markup[0] in text: | |
| text = text.replace(markup[0], markup[1]) | |
| self.email_text += '\n{0}'.format(text) | |
| def _probing(self, url, max_attempts=10, | |
| check_exists=CHECK_EXISTS, | |
| check_not_exists=CHECK_NOT_EXISTS): | |
| i = 0 | |
| start = datetime.now() | |
| while True: | |
| try: | |
| output = urllib2.urlopen(url).read() | |
| if [x for x in check_exists if x in output] and \ | |
| [x for x in check_not_exists if x not in output]: | |
| elapsed = datetime.now() - start | |
| self.log('Warmup: {0} [ {1} sec. ] [ {2} ]'.format( | |
| url, | |
| elapsed.seconds, | |
| OK)) | |
| return output | |
| except URLError: | |
| self.log('.') | |
| time.sleep(SLEEP_SHORT) | |
| if i >= max_attempts: | |
| elapsed = datetime.now() - start | |
| self.log('Warmup: {0} [ {1} sec. ] [ {2} ]'.format( | |
| url, | |
| elapsed.seconds, | |
| FAILED)) | |
| break | |
| i += 1 | |
| def start_instance(self): | |
| if self.has_supervisor: | |
| self._headline('Restart Supervisor') | |
| p = subprocess.Popen(['./bin/supervisorctl', 'status'], | |
| stdout=subprocess.PIPE) | |
| out, err = p.communicate() | |
| if 'refused connection' in out: | |
| self.log( | |
| 'Supervisor daemon not running, starting the daemon...' | |
| ) | |
| time.sleep(SLEEP_LONG) | |
| subprocess.call(['bin/supervisord', ]) | |
| elif 'RUNNING' in out: | |
| self.log('Supervisor: instance is running, restarting...') | |
| subprocess.call(['bin/supervisorctl', 'shutdown']) | |
| time.sleep(SLEEP_LONG) | |
| subprocess.call(['bin/supervisord', ]) | |
| elif 'STOPPED' in out: | |
| self.log('Supervisor: instance not running, starting...') | |
| time.sleep(SLEEP_LONG) | |
| subprocess.call(['bin/supervisorctl', 'start', 'all']) | |
| else: | |
| self.log( | |
| "Don't know what state supervisor is in, exiting " | |
| 'start_instance!' | |
| ) | |
| return | |
| elif os.path.exists('var/instance.pid'): | |
| self._headline('Restart Zope Instance') | |
| self.restart = True | |
| subprocess.call(['bin/instance', 'restart']) | |
| else: | |
| self._headline('Start Zope Instance') | |
| self.restart = False | |
| subprocess.call(['bin/instance', 'start']) | |
| self._probing( | |
| self.base_url + '/kontakt', | |
| check_exists=['Kontakt', 'Empfänger']) | |
| def warmup_front_page(self): | |
| self._headline('Warmup Front-Page') | |
| output = self._probing( | |
| self.base_url, | |
| check_exists=['Das Meinungsmedium', 'section-front-page']) | |
| tree = lxml.html.fromstring(output) | |
| links = [] | |
| for link in tree.iterlinks(): | |
| link = link[2] | |
| ignore = False | |
| if not link.startswith(self.base_url): | |
| continue | |
| if link == self.base_url: | |
| continue | |
| for middle in IGNORE_MIDDLE: | |
| if not ignore and middle in link: | |
| ignore = True | |
| if ignore: | |
| continue | |
| for end in IGNORE_END: | |
| if not ignore and link.endswith(end): | |
| ignore = True | |
| if ignore: | |
| continue | |
| if link not in links: | |
| links.append(link) | |
| self.log('{0} links found on the front-page.'.format(len(links))) | |
| return links | |
| def warmup(self, urls): | |
| self._headline('Warmup') | |
| for url in urls: | |
| self._probing(url) | |
| def elapsed_time(self): | |
| # Compute elapsed time | |
| self._headline('Elapsed Time') | |
| self.log('Start took: {0}.'.format(self.start_instance_time)) | |
| self.log( | |
| 'Front-page warmup took: {0}.'.format(self.warmup_front_page_time) | |
| ) | |
| self.log('Warmup took: {0}'.format(self.warmup_time)) | |
| self.log( | |
| 'Full script took: {0}'.format( | |
| self.start_instance_time + | |
| self.warmup_front_page_time + | |
| self.warmup_time) | |
| ) | |
| def send_email(self): | |
| msg = MIMEText(self.email_text) | |
| msg['Subject'] = EMAIL_SUBJECT.format(os.uname()[1]) | |
| msg['From'] = EMAIL_FROM | |
| msg['To'] = EMAIL_TO | |
| smtp = smtplib.SMTP(EMAIL_HOST) | |
| smtp.sendmail(EMAIL_FROM, [EMAIL_TO, ], msg.as_string()) | |
| smtp.quit() | |
| if __name__ == '__main__': | |
| arg = True | |
| try: | |
| instance = sys.argv[1] | |
| except IndexError: | |
| arg = False | |
| pass | |
| if arg and instance == 'local': | |
| url = 'http://localhost:{0}/{1}'.format(PLONE_LOCAL_PORT, | |
| PLONE_INSTANCE) | |
| Startup(url, has_supervisor=False) | |
| else: | |
| url = 'http://localhost:{0}/{1}'.format(PLONE_SUPERVISOR_PORT, | |
| PLONE_INSTANCE) | |
| Startup(url, has_supervisor=True) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment