zzzeek/test.py

## test.py
"""oslo.db enginefacade with intelligent retries demonstration.

This script illustrates three API functions:

* run_api_call_always_outermost - this API call is always outside of any
  transactional context.   A decorator asserts that this is the case.
  Database errors that occur within it will trigger
  the whole function to be retried.

* run_internal_api_call - this API call is always called inside of a
  transactional context.  A decorator asserts that this is the case.

* run_api_call_that_also_might_be_internal - this API call may be called
  inside, or outside, of a transactional context.  A decorator checks to see
  if one is in progress or not, and applies "retry" only if there is no
  external context.

The current oslo_db.enginefacade API is used fully and also includes a new
function that can be added to oslo.db which returns true/false if there
is already an enginefacade block in progress.

"""

import logging
import random

from oslo_context import context as oslo_context
from oslo_db import api as oslo_db_api
from oslo_db.sqlalchemy import enginefacade as _enginefacade

oslo_db_api.LOG.setLevel(logging.DEBUG)

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
logging.basicConfig()

# oslo.db's retry decorator, which seems to be popular.
_retry_db_errors = oslo_db_api.wrap_db_retry(
    max_retries=10,
    retry_interval=0.5,
    inc_retry_interval=False,  # we're in a hurry
    exception_checker=lambda e: True,
    jitter=True,
)

# set up enginefacade using current API patterns
_enginefacade.transaction_context_provider(oslo_context.RequestContext)

enginefacade = _enginefacade.transaction_context()
enginefacade.configure(
    connection="mysql+pymysql://scott:tiger@localhost/test",
    # connection_debug=50,
)


def enginefacade_context_in_transaction(context):
    """Given a context used by an enginefacade, determine if the enginefacade
    has already been invoked to begin a transaction block.

    This function can be added to oslo.db.

    """
    return hasattr(context, "_enginefacade_context") and hasattr(
        context._enginefacade_context, "current"
    )


def retry_db_errors_external(fn):
    """External API retry decorator.

    Apply this decorator to API methods that are guaranteed to be called
    before any database context has been started.

    """

    decorated = _retry_db_errors(fn)

    def go(context, *arg, **kw):
        assert not enginefacade_context_in_transaction(context)
        log.info(
            "running API method %s **ALWAYS OUTSIDE** an existing "
            "transaction.",
            fn.__name__,
        )
        return decorated(context, *arg, **kw)

    return go


def retry_db_errors_maybe_external(fn):
    """External / internal API retry decorator.

    Apply this decorator to API methods that may be called outside of a
    database context, or might be called inside one.   The "retry" feature
    only takes place if it were called outside.


    """
    outer_version = _retry_db_errors(fn)
    inner_version = fn

    def go(context, *arg, **kw):
        if enginefacade_context_in_transaction(context):
            log.info(
                "running API method %s **INSIDE** an existing transaction.  "
                "If it fails, you will see the outer name, not this one, "
                "retried.",
                fn.__name__,
            )
            return inner_version(context, *arg, **kw)
        else:
            log.info(
                "running API method %s **OUTSIDE** of an existing "
                "transaction.  If it fails, you will see this name retried.",
                fn.__name__,
            )
            return outer_version(context, *arg, **kw)

    return go


def assert_internal_only(fn):
    """Internal API non retry decorator.

    This decorator does nothing except assert that the function is
    definitely called within an existing enginefacade block.


    """

    def go(context, *arg, **kw):
        assert enginefacade_context_in_transaction(context)
        log.info(
            "running API method %s **ALWAYS INSIDE** an existing "
            "transaction.  It should never be retried directly",
            fn.__name__,
        )
        return fn(context, *arg, **kw)

    return go


@retry_db_errors_external
def run_api_call_always_outermost(context, seed2, seed3):
    """internal API funtion that is always invoked outside of an enginefacade
    block.

    """
    with enginefacade.reader.using(context):
        if seed2 == 1:
            return run_internal_api_call(context)
        else:
            return run_api_call_that_also_might_be_internal(
                context, seed2, seed3
            )


@assert_internal_only
def run_internal_api_call(context):
    """internal API funtion that is always invoked inside of an enginefacade
    block.

    This call intentionally fails half the time.

    """
    with enginefacade.reader.using(context):
        if random.randint(1, 3) == 1:
            return context.session.execute(
                "select 'internal only API call'"
            ).scalar()
        else:
            # intentionally generate a database error 2/3 times
            return context.session.execute(
                "select * from no_such_table"
            ).scalar()


@retry_db_errors_maybe_external
def run_api_call_that_also_might_be_internal(context, seed2, seed3):
    """API function that may ber called from an internal or external context."""

    with enginefacade.reader.using(context):

        if seed3 == 1:
            # call the internal only call that might fail
            return run_internal_api_call(context)
        else:
            return context.session.execute(
                "select 'internal/external API call'"
            ).scalar()


def outermost_request_method(seed1, seed2, seed3):
    """The application's entrypoint that creates the context."""

    context = oslo_context.RequestContext()

    if seed1 == 1:
        result = run_api_call_always_outermost(context, seed2, seed3)
    else:
        result = run_api_call_that_also_might_be_internal(
            context, seed2, seed3
        )

    print("got result: %s" % result)


for x in range(50):
    print("\n\n============================================")
    print("Request: %s" % x)

    seed1, seed2, seed3 = (
        random.randint(1, 2),
        random.randint(1, 2),
        random.randint(1, 2),
    )
    outermost_request_method(seed1, seed2, seed3)
	"""oslo.db enginefacade with intelligent retries demonstration.

	This script illustrates three API functions:

	* run_api_call_always_outermost - this API call is always outside of any
	transactional context. A decorator asserts that this is the case.
	Database errors that occur within it will trigger
	the whole function to be retried.

	* run_internal_api_call - this API call is always called inside of a
	transactional context. A decorator asserts that this is the case.

	* run_api_call_that_also_might_be_internal - this API call may be called
	inside, or outside, of a transactional context. A decorator checks to see
	if one is in progress or not, and applies "retry" only if there is no
	external context.

	The current oslo_db.enginefacade API is used fully and also includes a new
	function that can be added to oslo.db which returns true/false if there
	is already an enginefacade block in progress.

	"""

	import logging
	import random

	from oslo_context import context as oslo_context
	from oslo_db import api as oslo_db_api
	from oslo_db.sqlalchemy import enginefacade as _enginefacade

	oslo_db_api.LOG.setLevel(logging.DEBUG)

	log = logging.getLogger(__name__)
	log.setLevel(logging.DEBUG)
	logging.basicConfig()

	# oslo.db's retry decorator, which seems to be popular.
	_retry_db_errors = oslo_db_api.wrap_db_retry(
	max_retries=10,
	retry_interval=0.5,
	inc_retry_interval=False, # we're in a hurry
	exception_checker=lambda e: True,
	jitter=True,
	)

	# set up enginefacade using current API patterns
	_enginefacade.transaction_context_provider(oslo_context.RequestContext)

	enginefacade = _enginefacade.transaction_context()
	enginefacade.configure(
	connection="mysql+pymysql://scott:tiger@localhost/test",
	# connection_debug=50,
	)


	def enginefacade_context_in_transaction(context):
	"""Given a context used by an enginefacade, determine if the enginefacade
	has already been invoked to begin a transaction block.

	This function can be added to oslo.db.

	"""
	return hasattr(context, "_enginefacade_context") and hasattr(
	context._enginefacade_context, "current"
	)


	def retry_db_errors_external(fn):
	"""External API retry decorator.

	Apply this decorator to API methods that are guaranteed to be called
	before any database context has been started.

	"""

	decorated = _retry_db_errors(fn)

	def go(context, arg, *kw):
	assert not enginefacade_context_in_transaction(context)
	log.info(
	"running API method %s ALWAYS OUTSIDE an existing "
	"transaction.",
	fn.__name__,
	)
	return decorated(context, arg, *kw)

	return go


	def retry_db_errors_maybe_external(fn):
	"""External / internal API retry decorator.

	Apply this decorator to API methods that may be called outside of a
	database context, or might be called inside one. The "retry" feature
	only takes place if it were called outside.


	"""
	outer_version = _retry_db_errors(fn)
	inner_version = fn

	def go(context, arg, *kw):
	if enginefacade_context_in_transaction(context):
	log.info(
	"running API method %s INSIDE an existing transaction. "
	"If it fails, you will see the outer name, not this one, "
	"retried.",
	fn.__name__,
	)
	return inner_version(context, arg, *kw)
	else:
	log.info(
	"running API method %s OUTSIDE of an existing "
	"transaction. If it fails, you will see this name retried.",
	fn.__name__,
	)
	return outer_version(context, arg, *kw)

	return go


	def assert_internal_only(fn):
	"""Internal API non retry decorator.

	This decorator does nothing except assert that the function is
	definitely called within an existing enginefacade block.


	"""

	def go(context, arg, *kw):
	assert enginefacade_context_in_transaction(context)
	log.info(
	"running API method %s ALWAYS INSIDE an existing "
	"transaction. It should never be retried directly",
	fn.__name__,
	)
	return fn(context, arg, *kw)

	return go


	@retry_db_errors_external
	def run_api_call_always_outermost(context, seed2, seed3):
	"""internal API funtion that is always invoked outside of an enginefacade
	block.

	"""
	with enginefacade.reader.using(context):
	if seed2 == 1:
	return run_internal_api_call(context)
	else:
	return run_api_call_that_also_might_be_internal(
	context, seed2, seed3
	)


	@assert_internal_only
	def run_internal_api_call(context):
	"""internal API funtion that is always invoked inside of an enginefacade
	block.

	This call intentionally fails half the time.

	"""
	with enginefacade.reader.using(context):
	if random.randint(1, 3) == 1:
	return context.session.execute(
	"select 'internal only API call'"
	).scalar()
	else:
	# intentionally generate a database error 2/3 times
	return context.session.execute(
	"select * from no_such_table"
	).scalar()


	@retry_db_errors_maybe_external
	def run_api_call_that_also_might_be_internal(context, seed2, seed3):
	"""API function that may ber called from an internal or external context."""

	with enginefacade.reader.using(context):

	if seed3 == 1:
	# call the internal only call that might fail
	return run_internal_api_call(context)
	else:
	return context.session.execute(
	"select 'internal/external API call'"
	).scalar()


	def outermost_request_method(seed1, seed2, seed3):
	"""The application's entrypoint that creates the context."""

	context = oslo_context.RequestContext()

	if seed1 == 1:
	result = run_api_call_always_outermost(context, seed2, seed3)
	else:
	result = run_api_call_that_also_might_be_internal(
	context, seed2, seed3
	)

	print("got result: %s" % result)


	for x in range(50):
	print("\n\n============================================")
	print("Request: %s" % x)

	seed1, seed2, seed3 = (
	random.randint(1, 2),
	random.randint(1, 2),
	random.randint(1, 2),
	)
	outermost_request_method(seed1, seed2, seed3)