retry.py

# Patches on Boto Retry Strategy

"""
Handles the check whether it need to retry or not
"""

from __future__ import with_statement
import json
import boto
from xml.dom.minidom import parseString


class InvalidDocumentException(Exception):
    """
    Invalid Document Exception.
    """
    pass

class ErrorXMLParser(object):
    """
    Simple Parser for Boto XML Response (HTTP STATUS: 4XX)
    ** Format:
       <?xml version="1.0" encoding="UTF-8"?>
       <Response>
        <Errors>
         <Error>
          <Code>InvalidInstanceID.NotFound</Code>
          <Message>The instance ID 'i-ac851fae' does not exist</Message>
         </Error>
        </Errors>
        <RequestID>1e6c308c-0674-418c-b7b3-92968073b5b4</RequestID>
       </Response>

    """

    def __init__(self, document):
        if not isinstance(document, str) or not document:
            raise InvalidDocumentException("invalid document found: %r" % document)
        else:
            self.dom = parseString(document)

    def get_error_code(self):
        return self.get_element("Code")

    def get_message(self):
        return self.get_element("Message")

    def get_element(self, tag_name):
        return self.handle_token(self.dom.getElementsByTagName(tag_name))

    def get_text(self, nodelist):
        rc = []
        for node in nodelist:
            if node.nodeType == node.TEXT_NODE:
                rc.append(node.data)
        return ''.join(rc)

    def handle_token(self, tokenlist):
        for token in tokenlist:
            texts = self.get_text(token.childNodes)
        return texts


def retry_check(request, response, config_file="/etc/retry_enhanced.json"):
    """
    Check whether this request need to retry or not
    """
    try:
        retry_cfg = json.load(open(config_file, "r"))
        status_codes = [item['response']['http_status_code'] for item in retry_cfg.values()]
    except Exception, e:
        boto.log.exception(e)
        return False 

    try:
        endpoint = request.host
        status = response.status
        if status in status_codes:
            return False
        response_msg = response.msg
        reason = response.reason
    except Exception, e:
        boto.log.exception(e)
        return False

    try:
        xml_obj = ErrorXMLParser(response.read())
        error_code = xml_obj.get_error_code().lower()
    except Exception, e:
        boto.log.error("response reason: %r" % reason)
        boto.log.error("response message: \n%s" % response_msg)
        boto.log.error(e)
        return False

    for item in retry_cfg.values():
        try:
            if status == item['response']['http_status_code']\
               and item['response']['service_error_code'].lower() in error_code:
                msg = "Received the response with (%s: %s) " % (status, error_code)
                msg += "will trigger the request again."
                boto.log.info(msg)
                return True
        except Exception, e:
            boto.log.exception(e)
            return False

    else:
        msg = "Received the response with (%s: %s) " % (status, error_code)
        msg += "not meet the requirement of retrying."
        boto.log.info(msg)
        return False