Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Discovering Descriptors
"""Code for the talk "Discovering Descriptors"
> PyCon CZ 2017 - 8 to 10 June - Prage, Czech Republic
June 9th, 2017 11:00 CET
> EuroPython 2017 - 9 to 16 July - Rimini, Italy
July 11th, 2017 15:45 CET
https://ep2017.europython.eu/conference/talks/discovering-descriptors
Python 3.6
More references about descriptors at:
https://rmariano.github.io/itarch/categories/descriptors.html
Contents:
1. A first look at descriptors
1.1 __get__
1.2 __set_name__
1.3 __set__
1.4 __delete__
1.5 Data descriptor with __get__ & __set__
2. Descriptors as used in CPython
2.1 Methods are functions
2.2 Example of binding a function as a method
3. Extended Uses
3.1 Decorators & Descriptors
3.2 Object oriented design with descriptors
3.3 More Examples
"""
import types
from functools import wraps
# 1. A first look at Descriptors
# [1.1,1.2] - Example 1: __get__
# pylint: disable=too-few-public-methods
class DateFormatter:
"""Format date attributes as strings, in a particular format, providing an
attribute named ``<str>_<attr>``, which is the str representation of
<attr>.
>>> df = DateFormatter()
>>> DateFormatter.__set_name__(df, DateFormatter, 'wrong')
Traceback (most recent call last):
...
AttributeError: 'DateFormatter.wrong' should have a prefix with '_'
A custom formatter can be provided::
>>> from datetime import datetime
>>> class Test:
... str_x = DateFormatter(fmt='%Y')
>>> test = Test()
>>> test.x = datetime(2017, 6, 9)
>>> test.str_x
'2017'
"""
FORMAT = "%Y-%m-%d %H:%M"
def __init__(self, name=None, *, fmt=FORMAT):
self.name = name
self.fmt = fmt
def __get__(self, instance, owner):
"""Get the datetime from the object, and format it as a string,
returning the result.
Since name(descriptor) != name(attribute in obj), then we can use
``getattr`` and it won't recursive infinitely.
"""
if instance is None:
return self
date_value = getattr(instance, self.name)
if date_value is None:
return ''
return date_value.strftime(self.fmt)
def __set_name__(self, owner, name):
"""Unless another name was explicitly provided when it was created,
assume the default name for the property is the same as the descriptor
without the ``str_`` prefix.
If the name is not set and does not comply with the default, raise an
exception in order to avoid infinite recursion with the ``getattr``.
"""
if self.name is None:
_, _, self.name = name.partition('_')
if not self.name:
raise AttributeError(
f"'{owner.__name__}.{name}' should have a prefix with '_'")
class FileStat:
"""Stats of a file in a virtual file system.
>>> from datetime import datetime
>>> created = updated = datetime(2017, 6, 9, 11, 15, 19)
>>> f1 = FileStat('/home/mariano/file1', created, updated)
The default values are instances of ``datetime``::
>>> f1.created_at
datetime.datetime(2017, 6, 9, 11, 15, 19)
But the same with the ``str_`` prefix is a descriptor that formats this
value:
>>> f1.str_created_at
'2017-06-09 11:15'
>>> f1.str_updated_at
'2017-06-09 11:15'
If the value is None, default to empty string::
>>> f1.str_removed_at
''
>>> removed = datetime(2017, 5, 27, 17, 40, 59)
>>> deck2 = FileStat('/tmp/test', created, removed=removed)
>>> deck2.str_removed_at
'2017-05-27 17:40'
"""
str_created_at = DateFormatter('created_at')
str_updated_at = DateFormatter('updated_at')
str_removed_at = DateFormatter()
def __init__(self, filename, created, updated=None, removed=None):
self.filename = filename
self.created_at = created
self.updated_at = updated
self.removed_at = removed
# [1.3] - Example 2: __set__
class TracedProperty:
"""Keep count of how many times an attribute changed its value"""
# pylint: disable=unused-argument, attribute-defined-outside-init
def __set_name__(self, owner, name):
self.name = name
self.count_name = f'count_{name}'
def __set__(self, instance, value):
try:
current_value = instance.__dict__[self.name]
except KeyError:
instance.__dict__[self.count_name] = 0
else:
if current_value != value:
instance.__dict__[self.count_name] += 1
instance.__dict__[self.name] = value
class Traveller:
"""
>>> tourist = Traveller('John Smith')
>>> tourist.city = 'Barcelona'
>>> tourist.country = 'Spain'
>>> tourist.count_city
0
>>> tourist.count_country
0
>>> tourist.city = 'Stockholm'
>>> tourist.country = 'Sweden'
>>> tourist.count_city
1
>>> tourist.count_country
1
>>> tourist.city = 'Gothenburg'
>>> tourist.count_city
2
>>> tourist.count_country
1
>>> tourist.country = 'Sweden'
>>> tourist.count_country
1
The name is not under the logic of the descriptor, it's just a regular
instance attribute.
>>> tourist.name = 'John'
>>> hasattr(tourist, 'count_name')
False
>>> tourist.name == 'John'
True
"""
city = TracedProperty()
country = TracedProperty()
def __init__(self, name):
self.name = name
# [1.4] Example 3: __delete__
class ProtectedAttribute:
"""A class attribute that can be protected against deletion"""
# pylint: disable=unused-argument, attribute-defined-outside-init
def __set_name__(self, owner, name):
self.name = name
def __set__(self, instance, value):
instance.__dict__[self.name] = value
def __delete__(self, instance):
raise AttributeError(f"Can't delete {self.name} for {instance!s}")
class ProtectedUser:
"""
>>> usr = ProtectedUser('jsmith', '127.0.0.1')
>>> usr.username
'jsmith'
>>> del usr.username
Traceback (most recent call last):
...
AttributeError: Can't delete username for ProtectedUser[jsmith]
>>> usr.location
'127.0.0.1'
>>> del usr.location
>>> usr.location
Traceback (most recent call last):
...
AttributeError: 'ProtectedUser' object has no attribute 'location'
"""
username = ProtectedAttribute()
def __init__(self, username, location):
self.username = username
self.location = location
def __str__(self):
return f"{self.__class__.__name__}[{self.username}]"
# [1.4] - __get__ & __set__
class DataDescriptor:
"""Example to show that instance.__dict__ does not get modified with a data
descriptor, unless explicitly done by the descriptor itself.
This descriptor holds the same values for all instances.
"""
def __get__(self, instance, owner):
return self.value
# pylint: disable=unused-argument, attribute-defined-outside-init
def __set__(self, instance, value):
self.value = value
class Managed:
"""In this class the data descriptor for the attribute named 'descriptor'
takes precedence over the dictionary of the instance.
>>> managed = Managed()
>>> vars(managed)
{}
>>> managed.descriptor = 'foo'
>>> managed.descriptor
'foo'
>>> vars(managed)
{}
>>> managed_2 = Managed()
>>> vars(managed_2)
{}
>>> managed_2.descriptor
'foo'
Changing the value on the dictionary, doesn't affect the behaviour of the
descriptor, when requesting for the attribute:
>>> managed_2.__dict__['descriptor'] = 'something different'
>>> vars(managed_2)
{'descriptor': 'something different'}
>>> managed_2.descriptor
'foo'
"""
descriptor = DataDescriptor()
# [2] - Descriptors as used in CPython
# [2.1] - Methods are functions
class Class:
"""Class dictionaries store methods as functions.
>>> isinstance(Class.__dict__['method'], types.FunctionType)
True
To support method calls, functions have a ``__get__`` which does the
binding, when accessed from an instance.
Functions, in turn, are non-data descriptors::
>>> hasattr(Class.__dict__['method'], '__get__')
True
"""
def method(self, *args):
"""Instance method (function, that has 'self' as a first parameter).
The method is not actually called directly::
>>> instance = Class()
>>> instance.method('arg1', 'arg2')
"instance got ('arg1', 'arg2')"
Instead, it passes through the descriptor first (the ``__get__`` on the
function definition), which binds the parameters (self being the first
fixed one), creates the method, and then continues with the resulting
callable.
The previous call is actually::
>>> Class.method.__get__(instance, Class)('arg1', 'arg2')
"instance got ('arg1', 'arg2')"
"""
return f'{self!s} got {args}'
def __str__(self):
return 'instance'
# [2.2] - Example of binding a function as a method
def mtbf(system_monitor):
"""Mean Time Between Failures
https://en.wikipedia.org/wiki/Mean_time_between_failures
"""
operational_intervals = zip(
system_monitor.downtimes,
system_monitor.uptimes)
operational_time = sum(
(start_downtime - start_uptime)
for start_downtime, start_uptime in operational_intervals)
try:
return operational_time / len(system_monitor.downtimes)
except ZeroDivisionError:
return 0
class MTBF:
"""Compute Mean Time Between Failures"""
def __call__(self, instance):
"""``self`` is now a descriptor, so we apply the logic of the function
to a parameter that will be provided when invoked as a method.
"""
return mtbf(instance)
def __get__(self, instance, owner=None):
"""Return the self callable bound to ``instance``"""
if instance is None:
return self
return types.MethodType(self, instance)
class SystemMonitor:
"""Collect metrics on software & hardware components
>>> monitor = SystemMonitor('production', (0, 7, 12), (5, 12))
>>> mtbf(monitor)
5.0
>>> monitor.mtbf()
5.0
>>> SystemMonitor.mtbf.__get__(monitor, SystemMonitor)()
5.0
"""
def __init__(self, name, uptimes=None, downtimes=None):
self.name = name
self.uptimes = uptimes or []
self.downtimes = downtimes or []
mtbf = MTBF()
# [3] - Extended Uses
# [3.1] - Decorators & Descriptors
class DomainObject:
"""Dummy object that requires the common parameters for processing"""
def __init__(self, *args):
self.args = args
def process(self):
"""Show the arguments of the call"""
return ', '.join(self.args)
task1 = task2 = process
def task(self, taskno):
"""Example main task"""
result = self.process()
return f"Task {taskno}: {result}"
def __str__(self):
args = ','.join(self.args)
return f'{self.__class__.__name__}[{args}]'
def resolver_function(root, args, context, info):
"""A function that always requires these parameters for constructing an
object and operating with it.
>>> resolver_function('root', 'args', 'context', 'info')
'root, args, context, info'
"""
helper = DomainObject(root, args, context, info)
helper.process()
helper.task1()
helper.task2()
return helper.task1()
class DomainArgs:
"""The first attempt of a decorator will work for regular functions, but
doesn't handle the case for methods.
"""
def __init__(self, func):
self.func = func
wraps(func)(self)
def __call__(self, root, args, context, info):
"""Changes the signature of the wrapped function. Exposes the same old
4 parameters, but passes the required object to the internal function,
that can assume that's already been processed by this decorator.
"""
helper = DomainObject(root, args, context, info)
return self.func(helper)
class DomainArgsInjector(DomainArgs):
"""By implementing ``__get__``, along with the above logic, this can now
handle the case of being called from a class/instance.
"""
def __get__(self, instance, owner):
mapped = self.func.__get__(instance, owner)
return self.__class__(mapped)
@DomainArgs
def resolver_function2(helper):
"""The first version of the decorator, works for regular functions.
>>> resolver_function2('root', 'args', 'context', 'info')
'root, args, context, info'
"""
helper.task1()
helper.task2()
return helper.process()
class ViewResolver:
"""An object that contains method, that also require the logic of using a
helper with the parameters they receive.
"""
@DomainArgs
def resolve_method(self, helper):
"""With the first decorator, this method fails.
>>> vr = ViewResolver()
>>> vr.resolve_method('root', 'args', 'context', 'info')
Traceback (most recent call last):
...
TypeError: resolve_method() missing 1 required positional argument: 'helper' # noqa
"""
response = self.run(helper)
return f"Method: {response}"
@DomainArgsInjector
def method_resolver(self, helper):
"""The enhanced decorator can work for methods like this one.
>>> vr = ViewResolver()
>>> vr.method_resolver('root2', 'args2', 'context2', 'info2')
'Method resolver: root2, args2, context2, info2'
"""
response = self.run(helper)
return f"Method resolver: {response}"
@DomainArgsInjector
@classmethod
def cls_resolver(cls, helper):
"""
>>> ViewResolver.cls_resolver('r', 'a', 'c', 'i')
'Class method for ViewResolver with: DomainObject[r,a,c,i]'
"""
return f"Class method for {cls.__name__} with: {helper!s}"
@DomainArgsInjector
@staticmethod
def static_resolver(helper):
"""
>>> ViewResolver.static_resolver('root', 'args', 'ctx', 'info')
'Static method with DomainObject[root,args,ctx,info]'
>>> ViewResolver().static_resolver('root', 'args', 'ctx', 'info')
'Static method with DomainObject[root,args,ctx,info]'
"""
return f"Static method with {helper!s}"
@staticmethod
def run(helper):
"""Main task"""
return helper.process()
# [3.2] - Object-oriented Design with Descriptors
class ProtectedFomatter(DateFormatter, ProtectedAttribute):
"""Format date & prevent deletion.
From the two extended classes, this one inherits:
* __get__ from DateFormatter
* __set__ & __set__ from ProtectedAttribute
"""
class FileHandler:
"""
>>> from datetime import datetime
>>> mdate = datetime(2017, 6, 9, 11, 15, 59)
>>> fh = FileHandler('/dev/null', mdate)
Get the datetime attribute formatted as string [example 1.1]:
>>> fh.str_mdate
'2017-06-09'
Try to delete the protected attribute [example 1.4]:
>>> del fh.str_mdate
Traceback (most recent call last):
...
AttributeError: Can't delete mdate for file:///dev/null
"""
str_mdate = ProtectedFomatter(fmt='%Y-%m-%d')
def __init__(self, path, mdate):
self.abspath = path
self.mdate = mdate
def __str__(self):
return f"file://{self.abspath}"
# [3.3] - More Examples
class LazyProperty:
"""From Python Cookbook 3rd edition, recipe 8.10
at
https://github.com/dabeaz/python-cookbook/blob/master/src/8/lazily_computed_attributes/example1.py
"""
def __init__(self, function):
self.function = function
def __get__(self, instance, owner):
if instance is None:
return self
result = self.function(instance)
setattr(instance, self.function.__name__, result)
return result
# Run
if __name__ == '__main__':
import doctest
FAIL_COUNT, _ = doctest.testmod(verbose=False)
raise SystemExit(FAIL_COUNT)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.