Skip to content

Instantly share code, notes, and snippets.

@mylokin
Created November 17, 2014 14:44
Show Gist options
  • Save mylokin/7c03d807fdb5588a4404 to your computer and use it in GitHub Desktop.
Save mylokin/7c03d807fdb5588a4404 to your computer and use it in GitHub Desktop.
Backport of https://code.djangoproject.com/ticket/21952 for Django 1.6.x
diff --git a/django/dispatch/dispatcher.py b/django/dispatch/dispatcher.py
index d26433a..ba97077 100644
--- a/django/dispatch/dispatcher.py
+++ b/django/dispatch/dispatcher.py
@@ -1,11 +1,13 @@
-import weakref
+import sys
import threading
+import weakref
-from django.dispatch import saferef
from django.utils.six.moves import xrange
-
-WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref)
+if sys.version_info < (3, 4):
+ from .weakref_backports import WeakMethod
+else:
+ from weakref import WeakMethod
def _make_id(target):
@@ -25,7 +27,7 @@ class Signal(object):
Internal attributes:
receivers
- { receriverkey (id) : weakref(receiver) }
+ { receiverkey (id) : weakref(receiver) }
"""
def __init__(self, providing_args=None, use_caching=False):
"""
@@ -46,6 +48,7 @@ class Signal(object):
# 'sender_receivers_cache'. The cache is cleaned when .connect() or
# .disconnect() is called and populated on send().
self.sender_receivers_cache = weakref.WeakKeyDictionary() if use_caching else {}
+ self._dead_receivers = False
def connect(self, receiver, sender=None, weak=True, dispatch_uid=None):
"""
@@ -57,9 +60,7 @@ class Signal(object):
A function or an instance method which is to receive signals.
Receivers must be hashable objects.
- If weak is True, then receiver must be weak-referencable (more
- precisely saferef.safeRef() must be able to create a reference
- to the receiver).
+ If weak is True, then receiver must be weak-referencable.
Receivers must be able to accept keyword arguments.
@@ -111,9 +112,20 @@ class Signal(object):
lookup_key = (_make_id(receiver), _make_id(sender))
if weak:
- receiver = saferef.safeRef(receiver, onDelete=self._remove_receiver)
+ ref = weakref.ref
+ receiver_object = receiver
+ # Check for bound methods
+ if hasattr(receiver, '__self__') and hasattr(receiver, '__func__'):
+ ref = WeakMethod
+ receiver_object = receiver.__self__
+ if sys.version_info >= (3, 4):
+ receiver = ref(receiver)
+ weakref.finalize(receiver_object, self._remove_receiver)
+ else:
+ receiver = ref(receiver, self._remove_receiver)
with self.lock:
+ self._clear_dead_receivers()
for r_key, _ in self.receivers:
if r_key == lookup_key:
break
@@ -149,6 +161,7 @@ class Signal(object):
lookup_key = (_make_id(receiver), _make_id(sender))
with self.lock:
+ self._clear_dead_receivers()
for index in xrange(len(self.receivers)):
(r_key, _) = self.receivers[index]
if r_key == lookup_key:
@@ -224,6 +237,17 @@ class Signal(object):
responses.append((receiver, response))
return responses
+ def _clear_dead_receivers(self):
+ # Note: caller is assumed to hold self.lock.
+ if self._dead_receivers:
+ self._dead_receivers = False
+ new_receivers = []
+ for r in self.receivers:
+ if isinstance(r[1], weakref.ReferenceType) and r[1]() is None:
+ continue
+ new_receivers.append(r)
+ self.receivers = new_receivers
+
def _live_receivers(self, sender):
"""
Filter sequence of receivers to get resolved, live receivers.
@@ -232,7 +256,7 @@ class Signal(object):
live receivers.
"""
receivers = None
- if self.use_caching:
+ if self.use_caching and not self._dead_receivers:
receivers = self.sender_receivers_cache.get(sender)
# We could end up here with NO_RECEIVERS even if we do check this case in
# .send() prior to calling _live_receivers() due to concurrent .send() call.
@@ -240,6 +264,7 @@ class Signal(object):
return []
if receivers is None:
with self.lock:
+ self._clear_dead_receivers()
senderkey = _make_id(sender)
receivers = []
for (receiverkey, r_senderkey), receiver in self.receivers:
@@ -253,7 +278,7 @@ class Signal(object):
self.sender_receivers_cache[sender] = receivers
non_weak_receivers = []
for receiver in receivers:
- if isinstance(receiver, WEAKREF_TYPES):
+ if isinstance(receiver, weakref.ReferenceType):
# Dereference the weak reference.
receiver = receiver()
if receiver is not None:
@@ -262,24 +287,14 @@ class Signal(object):
non_weak_receivers.append(receiver)
return non_weak_receivers
- def _remove_receiver(self, receiver):
- """
- Remove dead receivers from connections.
- """
-
- with self.lock:
- to_remove = []
- for key, connected_receiver in self.receivers:
- if connected_receiver == receiver:
- to_remove.append(key)
- for key in to_remove:
- last_idx = len(self.receivers) - 1
- # enumerate in reverse order so that indexes are valid even
- # after we delete some items
- for idx, (r_key, _) in enumerate(reversed(self.receivers)):
- if r_key == key:
- del self.receivers[last_idx - idx]
- self.sender_receivers_cache.clear()
+ def _remove_receiver(self, receiver=None):
+ # Mark that the self.receivers list has dead weakrefs. If so, we will
+ # clean those up in connect, disconnect and _live_receivers while
+ # holding self.lock. Note that doing the cleanup here isn't a good
+ # idea, _remove_receiver() will be called as side effect of garbage
+ # collection, and so the call can happen while we are already holding
+ # self.lock.
+ self._dead_receivers = True
def receiver(signal, **kwargs):
diff --git a/django/dispatch/saferef.py b/django/dispatch/saferef.py
deleted file mode 100644
index c7731d4..0000000
--- a/django/dispatch/saferef.py
+++ /dev/null
@@ -1,257 +0,0 @@
-"""
-"Safe weakrefs", originally from pyDispatcher.
-
-Provides a way to safely weakref any function, including bound methods (which
-aren't handled by the core weakref module).
-"""
-
-import traceback
-import weakref
-
-def safeRef(target, onDelete = None):
- """Return a *safe* weak reference to a callable target
-
- target -- the object to be weakly referenced, if it's a
- bound method reference, will create a BoundMethodWeakref,
- otherwise creates a simple weakref.
- onDelete -- if provided, will have a hard reference stored
- to the callable to be called after the safe reference
- goes out of scope with the reference object, (either a
- weakref or a BoundMethodWeakref) as argument.
- """
- if hasattr(target, '__self__'):
- if target.__self__ is not None:
- # Turn a bound method into a BoundMethodWeakref instance.
- # Keep track of these instances for lookup by disconnect().
- assert hasattr(target, '__func__'), """safeRef target %r has __self__, but no __func__, don't know how to create reference"""%( target,)
- reference = get_bound_method_weakref(
- target=target,
- onDelete=onDelete
- )
- return reference
- if callable(onDelete):
- return weakref.ref(target, onDelete)
- else:
- return weakref.ref( target )
-
-class BoundMethodWeakref(object):
- """'Safe' and reusable weak references to instance methods
-
- BoundMethodWeakref objects provide a mechanism for
- referencing a bound method without requiring that the
- method object itself (which is normally a transient
- object) is kept alive. Instead, the BoundMethodWeakref
- object keeps weak references to both the object and the
- function which together define the instance method.
-
- Attributes:
- key -- the identity key for the reference, calculated
- by the class's calculateKey method applied to the
- target instance method
- deletionMethods -- sequence of callable objects taking
- single argument, a reference to this object which
- will be called when *either* the target object or
- target function is garbage collected (i.e. when
- this object becomes invalid). These are specified
- as the onDelete parameters of safeRef calls.
- weakSelf -- weak reference to the target object
- weakFunc -- weak reference to the target function
-
- Class Attributes:
- _allInstances -- class attribute pointing to all live
- BoundMethodWeakref objects indexed by the class's
- calculateKey(target) method applied to the target
- objects. This weak value dictionary is used to
- short-circuit creation so that multiple references
- to the same (object, function) pair produce the
- same BoundMethodWeakref instance.
-
- """
-
- _allInstances = weakref.WeakValueDictionary()
-
- def __new__( cls, target, onDelete=None, *arguments,**named ):
- """Create new instance or return current instance
-
- Basically this method of construction allows us to
- short-circuit creation of references to already-
- referenced instance methods. The key corresponding
- to the target is calculated, and if there is already
- an existing reference, that is returned, with its
- deletionMethods attribute updated. Otherwise the
- new instance is created and registered in the table
- of already-referenced methods.
- """
- key = cls.calculateKey(target)
- current =cls._allInstances.get(key)
- if current is not None:
- current.deletionMethods.append( onDelete)
- return current
- else:
- base = super( BoundMethodWeakref, cls).__new__( cls )
- cls._allInstances[key] = base
- base.__init__( target, onDelete, *arguments,**named)
- return base
-
- def __init__(self, target, onDelete=None):
- """Return a weak-reference-like instance for a bound method
-
- target -- the instance-method target for the weak
- reference, must have __self__ and __func__ attributes
- and be reconstructable via:
- target.__func__.__get__( target.__self__ )
- which is true of built-in instance methods.
- onDelete -- optional callback which will be called
- when this weak reference ceases to be valid
- (i.e. either the object or the function is garbage
- collected). Should take a single argument,
- which will be passed a pointer to this object.
- """
- def remove(weak, self=self):
- """Set self.isDead to true when method or instance is destroyed"""
- methods = self.deletionMethods[:]
- del self.deletionMethods[:]
- try:
- del self.__class__._allInstances[ self.key ]
- except KeyError:
- pass
- for function in methods:
- try:
- if callable( function ):
- function( self )
- except Exception as e:
- try:
- traceback.print_exc()
- except AttributeError:
- print('Exception during saferef %s cleanup function %s: %s' % (
- self, function, e)
- )
- self.deletionMethods = [onDelete]
- self.key = self.calculateKey( target )
- self.weakSelf = weakref.ref(target.__self__, remove)
- self.weakFunc = weakref.ref(target.__func__, remove)
- self.selfName = str(target.__self__)
- self.funcName = str(target.__func__.__name__)
-
- def calculateKey( cls, target ):
- """Calculate the reference key for this reference
-
- Currently this is a two-tuple of the id()'s of the
- target object and the target function respectively.
- """
- return (id(target.__self__),id(target.__func__))
- calculateKey = classmethod( calculateKey )
-
- def __str__(self):
- """Give a friendly representation of the object"""
- return """%s( %s.%s )"""%(
- self.__class__.__name__,
- self.selfName,
- self.funcName,
- )
-
- __repr__ = __str__
-
- def __hash__(self):
- return hash(self.key)
-
- def __bool__( self ):
- """Whether we are still a valid reference"""
- return self() is not None
-
- def __nonzero__(self): # Python 2 compatibility
- return type(self).__bool__(self)
-
- def __eq__(self, other):
- """Compare with another reference"""
- if not isinstance(other, self.__class__):
- return self.__class__ == type(other)
- return self.key == other.key
-
- def __call__(self):
- """Return a strong reference to the bound method
-
- If the target cannot be retrieved, then will
- return None, otherwise returns a bound instance
- method for our object and function.
-
- Note:
- You may call this method any number of times,
- as it does not invalidate the reference.
- """
- target = self.weakSelf()
- if target is not None:
- function = self.weakFunc()
- if function is not None:
- return function.__get__(target)
- return None
-
-class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):
- """A specialized BoundMethodWeakref, for platforms where instance methods
- are not descriptors.
-
- It assumes that the function name and the target attribute name are the
- same, instead of assuming that the function is a descriptor. This approach
- is equally fast, but not 100% reliable because functions can be stored on an
- attribute named differenty than the function's name such as in:
-
- class A: pass
- def foo(self): return "foo"
- A.bar = foo
-
- But this shouldn't be a common use case. So, on platforms where methods
- aren't descriptors (such as Jython) this implementation has the advantage
- of working in the most cases.
- """
- def __init__(self, target, onDelete=None):
- """Return a weak-reference-like instance for a bound method
-
- target -- the instance-method target for the weak
- reference, must have __self__ and __func__ attributes
- and be reconstructable via:
- target.__func__.__get__( target.__self__ )
- which is true of built-in instance methods.
- onDelete -- optional callback which will be called
- when this weak reference ceases to be valid
- (i.e. either the object or the function is garbage
- collected). Should take a single argument,
- which will be passed a pointer to this object.
- """
- assert getattr(target.__self__, target.__name__) == target, \
- ("method %s isn't available as the attribute %s of %s" %
- (target, target.__name__, target.__self__))
- super(BoundNonDescriptorMethodWeakref, self).__init__(target, onDelete)
-
- def __call__(self):
- """Return a strong reference to the bound method
-
- If the target cannot be retrieved, then will
- return None, otherwise returns a bound instance
- method for our object and function.
-
- Note:
- You may call this method any number of times,
- as it does not invalidate the reference.
- """
- target = self.weakSelf()
- if target is not None:
- function = self.weakFunc()
- if function is not None:
- # Using partial() would be another option, but it erases the
- # "signature" of the function. That is, after a function is
- # curried, the inspect module can't be used to determine how
- # many arguments the function expects, nor what keyword
- # arguments it supports, and pydispatcher needs this
- # information.
- return getattr(target, function.__name__)
- return None
-
-def get_bound_method_weakref(target, onDelete):
- """Instantiates the appropiate BoundMethodWeakRef, depending on the details of
- the underlying class method implementation"""
- if hasattr(target, '__get__'):
- # target method is a descriptor, so the default implementation works:
- return BoundMethodWeakref(target=target, onDelete=onDelete)
- else:
- # no luck, use the alternative implementation:
- return BoundNonDescriptorMethodWeakref(target=target, onDelete=onDelete)
diff --git a/django/dispatch/weakref_backports.py b/django/dispatch/weakref_backports.py
new file mode 100644
index 0000000..4647436
--- /dev/null
+++ b/django/dispatch/weakref_backports.py
@@ -0,0 +1,69 @@
+"""
+weakref_backports is a partial backport of the weakref module for python
+versions below 3.4.
+
+Copyright (C) 2013 Python Software Foundation, see license.python.txt for
+details.
+
+The following changes were made to the original sources during backporting:
+
+ * Added `self` to `super` calls.
+ * Removed `from None` when raising exceptions.
+
+"""
+from weakref import ref
+
+
+class WeakMethod(ref):
+ """
+ A custom `weakref.ref` subclass which simulates a weak reference to
+ a bound method, working around the lifetime problem of bound methods.
+ """
+
+ __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__"
+
+ def __new__(cls, meth, callback=None):
+ try:
+ obj = meth.__self__
+ func = meth.__func__
+ except AttributeError:
+ raise TypeError("argument should be a bound method, not {}"
+ .format(type(meth)))
+ def _cb(arg):
+ # The self-weakref trick is needed to avoid creating a reference
+ # cycle.
+ self = self_wr()
+ if self._alive:
+ self._alive = False
+ if callback is not None:
+ callback(self)
+ self = ref.__new__(cls, obj, _cb)
+ self._func_ref = ref(func, _cb)
+ self._meth_type = type(meth)
+ self._alive = True
+ self_wr = ref(self)
+ return self
+
+ def __call__(self):
+ obj = super(WeakMethod, self).__call__()
+ func = self._func_ref()
+ if obj is None or func is None:
+ return None
+ return self._meth_type(func, obj)
+
+ def __eq__(self, other):
+ if isinstance(other, WeakMethod):
+ if not self._alive or not other._alive:
+ return self is other
+ return ref.__eq__(self, other) and self._func_ref == other._func_ref
+ return False
+
+ def __ne__(self, other):
+ if isinstance(other, WeakMethod):
+ if not self._alive or not other._alive:
+ return self is not other
+ return ref.__ne__(self, other) or self._func_ref != other._func_ref
+ return True
+
+ __hash__ = ref.__hash__
+
diff --git a/tests/dispatch/tests/__init__.py b/tests/dispatch/tests/__init__.py
index b6d2621..1d6eb4d 100644
--- a/tests/dispatch/tests/__init__.py
+++ b/tests/dispatch/tests/__init__.py
@@ -5,4 +5,3 @@ Unit-tests for the dispatch project
from __future__ import absolute_import
from .test_dispatcher import DispatcherTests, ReceiverTestCase
-from .test_saferef import SaferefTests
diff --git a/tests/dispatch/tests/test_dispatcher.py b/tests/dispatch/tests/test_dispatcher.py
index b5edf28..b6bb02d 100644
--- a/tests/dispatch/tests/test_dispatcher.py
+++ b/tests/dispatch/tests/test_dispatcher.py
@@ -1,10 +1,10 @@
import gc
import sys
import time
+import unittest
import weakref
from django.dispatch import Signal, receiver
-from django.utils import unittest
if sys.platform.startswith('java'):
@@ -44,11 +44,9 @@ class DispatcherTests(unittest.TestCase):
def _testIsClean(self, signal):
"""Assert that everything has been cleaned up automatically"""
+ self.assertFalse(signal.has_listeners())
self.assertEqual(signal.receivers, [])
- # force cleanup just in case
- signal.receivers = []
-
def testExact(self):
a_signal.connect(receiver_1_arg, sender=self)
expected = [(receiver_1_arg,"test")]
diff --git a/tests/dispatch/tests/test_saferef.py b/tests/dispatch/tests/test_saferef.py
deleted file mode 100644
index 30eaddf..0000000
--- a/tests/dispatch/tests/test_saferef.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from django.dispatch.saferef import safeRef
-from django.utils.six.moves import xrange
-from django.utils import unittest
-
-class Test1(object):
- def x(self):
- pass
-
-def test2(obj):
- pass
-
-class Test2(object):
- def __call__(self, obj):
- pass
-
-class SaferefTests(unittest.TestCase):
- def setUp(self):
- ts = []
- ss = []
- for x in xrange(5000):
- t = Test1()
- ts.append(t)
- s = safeRef(t.x, self._closure)
- ss.append(s)
- ts.append(test2)
- ss.append(safeRef(test2, self._closure))
- for x in xrange(30):
- t = Test2()
- ts.append(t)
- s = safeRef(t, self._closure)
- ss.append(s)
- self.ts = ts
- self.ss = ss
- self.closureCount = 0
-
- def tearDown(self):
- del self.ts
- del self.ss
-
- def testIn(self):
- """Test the "in" operator for safe references (cmp)"""
- for t in self.ts[:50]:
- self.assertTrue(safeRef(t.x) in self.ss)
-
- def testValid(self):
- """Test that the references are valid (return instance methods)"""
- for s in self.ss:
- self.assertTrue(s())
-
- def testShortCircuit(self):
- """Test that creation short-circuits to reuse existing references"""
- sd = {}
- for s in self.ss:
- sd[s] = 1
- for t in self.ts:
- if hasattr(t, 'x'):
- self.assertTrue(safeRef(t.x) in sd)
- else:
- self.assertTrue(safeRef(t) in sd)
-
- def testRepresentation(self):
- """Test that the reference object's representation works
-
- XXX Doesn't currently check the results, just that no error
- is raised
- """
- repr(self.ss[-1])
-
- def _closure(self, ref):
- """Dumb utility mechanism to increment deletion counter"""
- self.closureCount +=1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment