Embed URL

HTTPS clone URL

SSH clone URL

You can clone with HTTPS or SSH.

Download Gist

Rails Lighthouse ticket #6129

View active_record_memory_bloat_patch_3.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
From 318d03d056315256e29ec85928a48425cb6e64f2 Mon Sep 17 00:00:00 2001
From: Brian Durand <brian@embellishedvisions.com>
Date: Tue, 7 Dec 2010 16:11:26 -0600
Subject: [PATCH] Change the ActiveRecord after_commit and after_rollback logic to only keep track of objects that can be affected by these callbacks to reduce memory bloat seen with large transactions involving many records.
 
---
activerecord/activerecord.gemspec | 1 +
.../abstract/database_statements.rb | 18 +++++-
.../test/cases/transaction_callbacks_test.rb | 61 ++++++++++++++++++++
3 files changed, 76 insertions(+), 4 deletions(-)
 
diff --git a/activerecord/activerecord.gemspec b/activerecord/activerecord.gemspec
index b1df248..78ca038 100644
--- a/activerecord/activerecord.gemspec
+++ b/activerecord/activerecord.gemspec
@@ -25,4 +25,5 @@ Gem::Specification.new do |s|
s.add_dependency('activemodel', version)
s.add_dependency('arel', '~> 2.0.2')
s.add_dependency('tzinfo', '~> 0.3.23')
+ s.add_dependency('ref', '~> 1.0.0')
end
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
index 01e53b4..17b2535 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
@@ -1,4 +1,5 @@
require 'active_support/core_ext/module/deprecation'
+require 'ref'
module ActiveRecord
module ConnectionAdapters # :nodoc:
@@ -209,8 +210,11 @@ module ActiveRecord
# Register a record with the current transaction so that its after_commit and after_rollback callbacks
# can be called.
def add_transaction_record(record)
+ # Use a weak reference unless transaction based callbacks are defined to prevent large transactions
+ # from balooning the heap.
+ ref_class = record._commit_callbacks.empty? && record._rollback_callbacks.empty? ? Ref::WeakReference : Ref::StrongReference
last_batch = @_current_transaction_records.last
- last_batch << record if last_batch
+ last_batch << ref_class.new(record) if last_batch
end
# Begins the transaction (and turns off auto-committing).
@@ -318,10 +322,10 @@ module ActiveRecord
# is false, only rollback records since the last save point.
def rollback_transaction_records(rollback) #:nodoc
if rollback
- records = @_current_transaction_records.flatten
+ records = _dereference_transaction_records(@_current_transaction_records)
@_current_transaction_records.clear
else
- records = @_current_transaction_records.pop
+ records = _dereference_transaction_records(@_current_transaction_records.pop)
end
unless records.blank?
@@ -337,7 +341,7 @@ module ActiveRecord
# Send a commit message to all records after they have been committed.
def commit_transaction_records #:nodoc
- records = @_current_transaction_records.flatten
+ records = _dereference_transaction_records(@_current_transaction_records)
@_current_transaction_records.clear
unless records.blank?
records.uniq.each do |record|
@@ -349,6 +353,12 @@ module ActiveRecord
end
end
end
+
+ # Flatten and dereference weak references to transaction records. Any garbage collected
+ # weak references will be removed.
+ def _dereference_transaction_records(references) #:nodoc
+ references.flatten.collect{|ref| ref.object}.compact
+ end
end
end
end
diff --git a/activerecord/test/cases/transaction_callbacks_test.rb b/activerecord/test/cases/transaction_callbacks_test.rb
index 85f222b..afc2478 100644
--- a/activerecord/test/cases/transaction_callbacks_test.rb
+++ b/activerecord/test/cases/transaction_callbacks_test.rb
@@ -244,6 +244,67 @@ class TransactionCallbacksTest < ActiveRecord::TestCase
assert_equal :rollback, @first.last_after_transaction_error
assert_equal [:after_rollback], @second.history
end
+
+ def test_transaction_records_only_held_if_referenced_or_callbacks_defined_on_rollback
+ Ref::Mock.use do
+ topic_1 = TopicWithCallbacks.new
+ topic_2 = Topic.new
+ topic_3 = TopicWithCallbacks.new
+ topic_4 = Topic.new
+ Topic.transaction do
+ [topic_1, topic_2, topic_3, topic_4].each_with_index do |topic, index|
+ topic.id = index + 1
+ def topic.rolledback!(*args)
+ @rolledback_flag = true
+ end
+ def topic.rolledback?
+ @rolledback_flag if instance_variable_defined?(:@rolledback_flag)
+ end
+ topic.add_to_transaction
+ end
+ # Fake garbage collection to release weak references to topic_3 and topic_4
+ Ref::Mock.gc(topic_3, topic_4)
+ raise ActiveRecord::Rollback
+ end
+ # Expected behavior is that the object without a hard reference and that doesn't implement
+ # after transaction callbacks will not be retained by the garbage collector and won't get
+ # the rolledback! message.
+ assert topic_1.rolledback?
+ assert topic_2.rolledback?
+ assert topic_3.rolledback?
+ assert !topic_4.rolledback?
+ end
+ end
+
+ def test_transaction_records_only_held_if_referenced_or_callbacks_defined_on_commit
+ Ref::Mock.use do
+ topic_1 = TopicWithCallbacks.new
+ topic_2 = Topic.new
+ topic_3 = TopicWithCallbacks.new
+ topic_4 = Topic.new
+ Topic.transaction do
+ [topic_1, topic_2, topic_3, topic_4].each_with_index do |topic, index|
+ topic.id = index + 1
+ def topic.committed!(*args)
+ @commit_flag = true
+ end
+ def topic.committed?
+ @commit_flag if instance_variable_defined?(:@commit_flag)
+ end
+ topic.add_to_transaction
+ end
+ # Fake garbage collection to release weak references to topic_3 and topic_4
+ Ref::Mock.gc(topic_3, topic_4)
+ end
+ # Expected behavior is that the object without a hard reference and that doesn't implement
+ # after transaction callbacks will not be retained by the garbage collector and won't get
+ # the rolledback! message.
+ assert topic_1.committed?
+ assert topic_2.committed?
+ assert topic_3.committed?
+ assert !topic_4.committed?
+ end
+ end
end
--
1.7.3.4
 
View active_record_memory_bloat_patch_3.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
From 4405d494e458b4dc6085e2cf9518579531fc7066 Mon Sep 17 00:00:00 2001
From: Brian Durand <brian@embellishedvisions.com>
Date: Tue, 7 Dec 2010 16:11:26 -0600
Subject: [PATCH] Add new ActiveSupport::WeakReference implementation that performs consistently across all runtimes. This is then usd to reduce memory bloat in large transactions by only holding onto strong references to objects that implement the transaction callbacks.
 
---
.../abstract/database_statements.rb | 23 ++-
.../test/cases/transaction_callbacks_test.rb | 69 ++++++
activesupport/lib/active_support.rb | 2 +
activesupport/lib/active_support/weak_hash.rb | 118 ++++++++++
activesupport/lib/active_support/weak_reference.rb | 239 ++++++++++++++++++++
activesupport/test/weak_hash_test.rb | 102 +++++++++
activesupport/test/weak_reference_test.rb | 55 +++++
7 files changed, 605 insertions(+), 3 deletions(-)
create mode 100644 activesupport/lib/active_support/weak_hash.rb
create mode 100644 activesupport/lib/active_support/weak_reference.rb
create mode 100644 activesupport/test/weak_hash_test.rb
create mode 100644 activesupport/test/weak_reference_test.rb
 
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
index ee9a0af..f697cc2 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
@@ -207,6 +207,9 @@ module ActiveRecord
# Register a record with the current transaction so that its after_commit and after_rollback callbacks
# can be called.
def add_transaction_record(record)
+ # Use a weak reference unless transaction based callbacks are defined to prevent large transactions
+ # from balooning the heap.
+ record = ActiveSupport::WeakReference.new(record) if record._commit_callbacks.empty? && record._rollback_callbacks.empty?
last_batch = @_current_transaction_records.last
last_batch << record if last_batch
end
@@ -313,10 +316,10 @@ module ActiveRecord
# is false, only rollback records since the last save point.
def rollback_transaction_records(rollback) #:nodoc
if rollback
- records = @_current_transaction_records.flatten
+ records = _dereference_transaction_records(@_current_transaction_records)
@_current_transaction_records.clear
else
- records = @_current_transaction_records.pop
+ records = _dereference_transaction_records(@_current_transaction_records.pop)
end
unless records.blank?
@@ -332,7 +335,7 @@ module ActiveRecord
# Send a commit message to all records after they have been committed.
def commit_transaction_records #:nodoc
- records = @_current_transaction_records.flatten
+ records = _dereference_transaction_records(@_current_transaction_records)
@_current_transaction_records.clear
unless records.blank?
records.uniq.each do |record|
@@ -344,6 +347,20 @@ module ActiveRecord
end
end
end
+
+ # Flatten and dereference weak references to transaction records. Any garbage collected
+ # weak references will be removed.
+ def _dereference_transaction_records(records) #:nodoc
+ records.collect do |ref|
+ if ref.is_a?(ActiveSupport::WeakReference)
+ ref.object
+ elsif ref.is_a?(Array)
+ _dereference_transaction_records(ref)
+ else
+ ref
+ end
+ end.flatten.compact
+ end
end
end
end
diff --git a/activerecord/test/cases/transaction_callbacks_test.rb b/activerecord/test/cases/transaction_callbacks_test.rb
index 85f222b..941b34f 100644
--- a/activerecord/test/cases/transaction_callbacks_test.rb
+++ b/activerecord/test/cases/transaction_callbacks_test.rb
@@ -244,6 +244,75 @@ class TransactionCallbacksTest < ActiveRecord::TestCase
assert_equal :rollback, @first.last_after_transaction_error
assert_equal [:after_rollback], @second.history
end
+
+ def test_transaction_records_only_held_if_referenced_or_callbacks_defined_on_rollback
+ save_implementation = ActiveSupport::WeakReference.implementation
+ ActiveSupport::WeakReference.implementation = ActiveSupport::WeakReference::TestImpl
+ begin
+ topic_1 = TopicWithCallbacks.new
+ topic_2 = Topic.new
+ topic_3 = TopicWithCallbacks.new
+ topic_4 = Topic.new
+ Topic.transaction do
+ [topic_1, topic_2, topic_3, topic_4].each_with_index do |topic, index|
+ topic.id = index + 1
+ def topic.rolledback!(*args)
+ @rolledback_flag = true
+ end
+ def topic.rolledback?
+ @rolledback_flag if instance_variable_defined?(:@rolledback_flag)
+ end
+ topic.add_to_transaction
+ end
+ # Fake garbage collection to release weak references to topic_3 and topic_4
+ ActiveSupport::WeakReference::TestImpl.gc(topic_3, topic_4)
+ raise ActiveRecord::Rollback
+ end
+ # Expected behavior is that the object without a hard reference and that doesn't implement
+ # after transaction callbacks will not be retained by the garbage collector and won't get
+ # the rolledback! message.
+ assert topic_1.rolledback?
+ assert topic_2.rolledback?
+ assert topic_3.rolledback?
+ assert !topic_4.rolledback?
+ ensure
+ ActiveSupport::WeakReference.implementation = save_implementation
+ end
+ end
+
+ def test_transaction_records_only_held_if_referenced_or_callbacks_defined_on_commit
+ save_implementation = ActiveSupport::WeakReference.implementation
+ ActiveSupport::WeakReference.implementation = ActiveSupport::WeakReference::TestImpl
+ begin
+ topic_1 = TopicWithCallbacks.new
+ topic_2 = Topic.new
+ topic_3 = TopicWithCallbacks.new
+ topic_4 = Topic.new
+ Topic.transaction do
+ [topic_1, topic_2, topic_3, topic_4].each_with_index do |topic, index|
+ topic.id = index + 1
+ def topic.committed!(*args)
+ @commit_flag = true
+ end
+ def topic.committed?
+ @commit_flag if instance_variable_defined?(:@commit_flag)
+ end
+ topic.add_to_transaction
+ end
+ # Fake garbage collection to release weak references to topic_3 and topic_4
+ ActiveSupport::WeakReference::TestImpl.gc(topic_3, topic_4)
+ end
+ # Expected behavior is that the object without a hard reference and that doesn't implement
+ # after transaction callbacks will not be retained by the garbage collector and won't get
+ # the rolledback! message.
+ assert topic_1.committed?
+ assert topic_2.committed?
+ assert topic_3.committed?
+ assert !topic_4.committed?
+ ensure
+ ActiveSupport::WeakReference.implementation = save_implementation
+ end
+ end
end
diff --git a/activesupport/lib/active_support.rb b/activesupport/lib/active_support.rb
index 6b87774..f30f8ad 100644
--- a/activesupport/lib/active_support.rb
+++ b/activesupport/lib/active_support.rb
@@ -75,6 +75,8 @@ module ActiveSupport
autoload :SafeBuffer, "active_support/core_ext/string/output_safety"
autoload :TestCase
+ autoload :WeakHash
+ autoload :WeakReference
end
autoload :I18n, "active_support/i18n"
diff --git a/activesupport/lib/active_support/weak_hash.rb b/activesupport/lib/active_support/weak_hash.rb
new file mode 100644
index 0000000..58f12a4
--- /dev/null
+++ b/activesupport/lib/active_support/weak_hash.rb
@@ -0,0 +1,118 @@
+require 'thread'
+
+module ActiveSupport
+ # Implementation of a map in which only weak references are kept to the map values.
+ # This allows the garbage collector to reclaim these objects if the
+ # only reference to them is in the weak hash map. This is often useful for cache
+ # implementations since the map can be allowed to grow without bound and the
+ # garbage collector can be relied on to clean it up as necessary. One must be careful,
+ # though, when accessing values since they can be collected at any time until their
+ # is a strong reference to them.
+ #
+ # Example usage:
+ #
+ # cache = WeakHash.new
+ # foo = "foo"
+ # cache["strong"] = foo # add a value with a strong reference
+ # cache["weak"] = "bar" # add a value without a strong reference
+ # cache["strong"] # "foo"
+ # cache["weak"] # "bar"
+ # GC.start
+ # cache["strong"] # "foo"
+ # cache["weak"] # nil
+ class WeakHash
+ # Create a new WeakHash. Values added to the hash will be cleaned up by the garbage
+ # collector if there are no other reference except in the WeakHash.
+ def initialize
+ @weak_references = {}
+ @weak_references_to_keys_map = {}
+ @weak_reference_cleanup = lambda{|object_id| remove_weak_reference_to(object_id)}
+ @mutex = Mutex.new
+ end
+
+ # Get a value from the map by key. If the value has been reclaimed by the garbage
+ # collector, this will return nil.
+ def [](key)
+ ref = @weak_references[key]
+ value = ref.object if ref
+ value
+ end
+
+ # Add a key/value to the map.
+ def []=(key, value)
+ ObjectSpace.define_finalizer(value, @weak_reference_cleanup)
+ key = key.dup if key.is_a?(String)
+ @mutex.synchronize do
+ @weak_references[key] = WeakReference.new(value)
+ keys_for_id = @weak_references_to_keys_map[value.__id__]
+ unless keys_for_id
+ keys_for_id = []
+ @weak_references_to_keys_map[value.__id__] = keys_for_id
+ end
+ keys_for_id << key
+ end
+ value
+ end
+
+ # Remove the value associated the the key from the map.
+ def delete(key)
+ ref = @weak_references.delete(key)
+ if ref
+ keys_to_id = @weak_references_to_keys_map[ref.referenced_object_id]
+ if keys_to_id
+ keys_to_id.delete(key)
+ @weak_references_to_keys_map.delete(ref.referenced_object_id) if keys_to_id.empty?
+ end
+ ref.object
+ else
+ nil
+ end
+ end
+
+ # Iterate through all the key/value pairs in the map that have not been reclaimed
+ # by the garbage collector.
+ def each
+ @weak_references.each do |key, ref|
+ value = ref.object
+ yield(key, value) if value
+ end
+ end
+
+ # Clear the map of all key/value pairs.
+ def clear
+ @mutex.synchronize do
+ @weak_references.clear
+ @weak_references_to_keys_map.clear
+ end
+ end
+
+ # Merge the values from another hash into this map.
+ def merge!(other_hash)
+ other_hash.each do |key, value|
+ self[key] = value
+ end
+ end
+
+ def inspect
+ live_entries = {}
+ each do |key, value|
+ live_entries[key] = value
+ end
+ live_entries.inspect
+ end
+
+ private
+
+ def remove_weak_reference_to(object_id)
+ @mutex.synchronize do
+ keys = @weak_references_to_keys_map[object_id]
+ if keys
+ keys.each do |key|
+ @weak_references.delete(key)
+ end
+ @weak_references_to_keys_map.delete(object_id)
+ end
+ end
+ end
+ end
+end
\ No newline at end of file
diff --git a/activesupport/lib/active_support/weak_reference.rb b/activesupport/lib/active_support/weak_reference.rb
new file mode 100644
index 0000000..6610aa9
--- /dev/null
+++ b/activesupport/lib/active_support/weak_reference.rb
@@ -0,0 +1,239 @@
+require 'thread'
+
+module ActiveSupport
+ # WeakReference is a class to represent a reference to an object that is not seen by
+ # the tracing phase of the garbage collector. This allows the referenced
+ # object to be garbage collected as if nothing is referring to it.
+ #
+ # This class provides several compatibility and performance benefits over using
+ # the WeakRef implementation that comes with Ruby and is compatible with Jruby.
+ #
+ # Usage:
+ #
+ # foo = Object.new
+ # ref = ActiveSupport::WeakReference.new(foo)
+ # ref.alive? # should be true
+ # ref.object # should be foo
+ # ObjectSpace.garbage_collect
+ # ref.alive? # should be false
+ # ref.object # should be nil
+ class WeakReference
+ # The object id of the object being referenced.
+ attr_reader :referenced_object_id
+
+ class << self
+ # Create a new WeakReference. The actual implementation class will be
+ # optimized for the runtime is being used. If the default implementation
+ # isn't appropriate, it can be changed by setting a different one.
+ def new(obj)
+ ref = implementation.allocate
+ ref.instance_variable_set(:@referenced_object_id, obj.__id__)
+ ref.send(:initialize, obj)
+ ref
+ end
+
+ # Get the implementation class for weak references.
+ def implementation
+ @implementation
+ end
+
+ # Set the implementation class for weak references.
+ def implementation=(klass)
+ @implementation = klass
+ end
+ end
+
+ # Get the referenced object. If the object has been reclaimed by the
+ # garbage collector, then this will return nil.
+ def object
+ raise NotImplementedError
+ end
+
+ def inspect
+ obj = object
+ "<##{self.class.name}: #{obj ? obj.inspect : "##{referenced_object_id} (not accessible)"}>"
+ end
+
+ # This is a pure ruby implementation of a weak reference. It is much more
+ # efficient than the bundled WeakRef implementation because it does not
+ # subclass Delegator which is very heavy to instantiate and utilizes a
+ # fair amount of memory.
+ #
+ # This implementation cannot be used by Jruby if ObjectSpace has been
+ # disabled.
+ class StandardImpl < WeakReference
+ # Map of references to the object_id's they refer to.
+ @@referenced_object_ids = {}
+
+ # Map of object_ids to references to them.
+ @@object_id_references = {}
+
+ @@mutex = Mutex.new
+
+ # Finalizer that cleans up weak references when an object is destroyed.
+ @@object_finalizer = lambda do |object_id|
+ @@mutex.synchronize do
+ reference_ids = @@object_id_references[object_id]
+ if reference_ids
+ reference_ids.each do |reference_object_id|
+ @@referenced_object_ids.delete(reference_object_id)
+ end
+ @@object_id_references.delete(object_id)
+ end
+ end
+ end
+
+ # Finalizer that cleans up weak references when references are destroyed.
+ @@reference_finalizer = lambda do |object_id|
+ @@mutex.synchronize do
+ referenced_id = @@referenced_object_ids.delete(object_id)
+ if referenced_id
+ obj = ObjectSpace._id2ref(referenced_object_id) rescue nil
+ if obj
+ backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
+ if backreferences
+ backreferences.delete(object_id)
+ obj.remove_instance_variable(:@__weak_backreferences__) if backreferences.empty?
+ end
+ end
+ references = @@object_id_references[referenced_id]
+ if references
+ references.delete(object_id)
+ @@object_id_references.delete(referenced_id) if references.empty?
+ end
+ end
+ end
+ end
+
+ # Create a new weak reference to an object. The existence of the weak reference
+ # will not prevent the garbage collector from reclaiming the referenced object.
+ def initialize(obj)
+ ObjectSpace.define_finalizer(obj, @@object_finalizer)
+ ObjectSpace.define_finalizer(self, @@reference_finalizer)
+ @@mutex.synchronize do
+ @@referenced_object_ids[self.__id__] = obj.__id__
+ add_backreference(obj)
+ references = @@object_id_references[obj.__id__]
+ unless references
+ references = []
+ @@object_id_references[obj.__id__] = references
+ end
+ references.push(self.__id__)
+ end
+ end
+
+ # Get the reference object. If the object has already been garbage collected,
+ # then this method will return nil.
+ def object
+ obj = nil
+ begin
+ if referenced_object_id == @@referenced_object_ids[self.object_id]
+ obj = ObjectSpace._id2ref(referenced_object_id)
+ obj = nil unless verify_backreferences(obj)
+ end
+ rescue RangeError
+ # Object has been garbage collected.
+ end
+ obj
+ end
+
+ private
+
+ def add_backreference(obj)
+ backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
+ unless backreferences
+ backreferences = []
+ obj.instance_variable_set(:@__weak_backreferences__, backreferences)
+ end
+ backreferences << object_id
+ end
+
+ def verify_backreferences(obj)
+ backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
+ backreferences && backreferences.include?(object_id)
+ end
+ end
+
+ # This implementation of a weak reference utilizes the weakling library which will
+ # use native Java weak references when running under Jruby. If the weakling gem
+ # has been required this will automatically be used.
+ class WeaklingImpl < WeakReference
+ def initialize(obj)
+ @ref = ::Weakling::WeakRef.new(obj)
+ end
+
+ def object
+ @ref.get
+ rescue RefError
+ nil
+ end
+ end
+
+ # This implementation of a weak reference simply wraps the standard WeakRef implementation
+ # that comes with Ruby. It is used as a fallback for Jruby if case the weakling gem
+ # is not available.
+ class WeakRefImpl < WeakReference
+ def initialize(obj)
+ @ref = ::WeakRef.new(obj)
+ end
+
+ def object
+ @ref.__getobj__
+ rescue => e
+ # Jruby implementation uses RefError while MRI uses WeakRef::RefError
+ if (defined?(RefError) && e.is_a?(RefError)) || (defined?(::WeakRef::RefError) && e.is_a?(::WeakRef::RefError))
+ nil
+ else
+ raise e
+ end
+ end
+ end
+
+ # This implementation can be used for testing. It implements the proper interface,
+ # but allows for mimicking garbage collection on demand.
+ class TestImpl < WeakReference
+ @@object_list = {}
+
+ def initialize(obj)
+ @object = obj
+ @@object_list[obj.__id__] = true
+ ObjectSpace.define_finalizer(self, lambda{@@object_list.delete(obj.__id__)})
+ end
+
+ def object
+ if @@object_list.include?(@object.__id__)
+ @object
+ else
+ @object = nil
+ end
+ end
+
+ # Simulate garbage collection of the objects passed in as arguments. If no objects
+ # are specified, all objects will be reclaimed.
+ def self.gc(*objects)
+ if objects.empty?
+ @@object_list = {}
+ else
+ objects.each{|obj| @@object_list.delete(obj.__id__)}
+ end
+ end
+ end
+
+ if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+ # If using Jruby set the implementation depending on whether or not the weakling gem is installed.
+ begin
+ require 'weakling'
+ self.implementation = WeaklingImpl
+ rescue LoadError
+ require 'weakref'
+ self.implementation = WeakRefImpl
+ end
+ elsif defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
+ # If using Rubinius set the implementation to use WeakRef since it is very efficient.
+ require 'weakref'
+ self.implementation = WeakRefImpl
+ else
+ self.implementation = StandardImpl
+ end
+ end
+end
diff --git a/activesupport/test/weak_hash_test.rb b/activesupport/test/weak_hash_test.rb
new file mode 100644
index 0000000..ffd3dce
--- /dev/null
+++ b/activesupport/test/weak_hash_test.rb
@@ -0,0 +1,102 @@
+require 'abstract_unit'
+require 'active_support/weak_reference'
+require 'active_support/weak_hash'
+
+class WeakHashTest < ActiveSupport::TestCase
+
+ def setup
+ # Use an implementation of WeakReference that is meant for testing since it allows
+ # for simulating garbage collection.
+ @weak_reference_implementation = ActiveSupport::WeakReference.implementation
+ ActiveSupport::WeakReference.implementation = ActiveSupport::WeakReference::TestImpl
+ end
+
+ def teardown
+ ActiveSupport::WeakReference.implementation = @weak_reference_implementation
+ ActiveSupport::WeakReference::TestImpl.gc
+ end
+
+ def test_keeps_entries_with_strong_references
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ assert_equal value_1, hash["key 1"]
+ assert_equal value_2, hash["key 2"]
+ end
+
+ def test_removes_entries_that_have_been_garbage_collected
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ assert_equal "value 2", hash["key 2"]
+ assert_equal "value 1", hash["key 1"]
+ ActiveSupport::WeakReference::TestImpl.gc(value_2)
+ assert_nil hash["key 2"]
+ assert_equal value_1, hash["key 1"]
+ end
+
+ def test_can_clear_the_map
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ hash.clear
+ assert_nil hash["key 1"]
+ assert_nil hash["key 2"]
+ end
+
+ def test_can_delete_entries
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ ActiveSupport::WeakReference::TestImpl.gc(value_2)
+ assert_nil hash.delete("key 2")
+ assert_equal value_1, hash.delete("key 1")
+ assert_nil hash["key 1"]
+ end
+
+ def test_can_merge_in_another_hash
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ value_3 = "value 3"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ hash.merge!("key 3" => value_3)
+ assert_equal "value 2", hash["key 2"]
+ assert_equal value_1, hash["key 1"]
+ ActiveSupport::WeakReference::TestImpl.gc(value_2)
+ assert_nil hash["key 2"]
+ assert_equal value_1, hash["key 1"]
+ assert_equal value_3, hash["key 3"]
+ end
+
+ def test_can_iterate_over_all_entries
+ hash = ActiveSupport::WeakHash.new
+ value_1 = "value 1"
+ value_2 = "value 2"
+ value_3 = "value 3"
+ hash["key 1"] = value_1
+ hash["key 2"] = value_2
+ hash["key 3"] = value_3
+ ActiveSupport::WeakReference::TestImpl.gc(value_2)
+ keys = []
+ values = []
+ hash.each{|k,v| keys << k; values << v}
+ assert_equal ["key 1", "key 3"], keys.sort
+ assert_equal ["value 1", "value 3"], values.sort
+ end
+
+ def test_inspect
+ hash = ActiveSupport::WeakHash.new
+ hash["key 1"] = "value 1"
+ assert hash.inspect
+ end
+end
diff --git a/activesupport/test/weak_reference_test.rb b/activesupport/test/weak_reference_test.rb
new file mode 100644
index 0000000..5d48f94
--- /dev/null
+++ b/activesupport/test/weak_reference_test.rb
@@ -0,0 +1,55 @@
+require 'abstract_unit'
+require 'active_support/weak_reference'
+
+module WeakReferenceTestBehaviors
+ def test_references_can_get_non_garbage_collected_objects
+ obj = Object.new
+ ref = ActiveSupport::WeakReference.new(obj)
+ assert_equal obj, ref.object
+ assert_equal obj.object_id, ref.referenced_object_id
+ end
+
+ def test_references_get_the_correct_object
+ # Since we can't reliably control the garbage collector, this is a brute force test.
+ id_to_ref = {}
+ 10000.times do |i|
+ obj = Object.new
+ if id_to_ref.key?(obj.object_id)
+ ref = id_to_ref[obj.object_id]
+ if ref.object
+ flunk "weak reference found with a live reference to an object that was not the one it was created with"
+ break
+ end
+ end
+ id_to_ref[obj.object_id] = ActiveSupport::WeakReference.new(obj)
+ GC.start if i % 1000 == 0
+ end
+ end
+
+ def test_inspect
+ ref = ActiveSupport::WeakReference.new(Object.new)
+ assert ref.inspect
+ end
+end
+
+class WeakReferenceTest < ActiveSupport::TestCase
+ def self.weak_reference_implementation
+ ActiveSupport::WeakReference
+ end
+ include WeakReferenceTestBehaviors
+end
+
+# If Jruby is using the weakling backed implementation, test the WeakRef fallback implementation as well.
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+ if defined?(::Weakling::WeakRef)
+ class WeakReferenceWeakRefImplTest < ActiveSupport::TestCase
+ def self.weak_reference_implementation
+ ActiveSupport::WeakReference::WeaklingImpl
+ end
+ include WeakReferenceTestBehaviors
+ end
+ else
+ $stderr.puts "Skipping ActiveSupport::WeakReference::WeakingImpl tests. Install the weakling gem to run them."
+ end
+end
+
--
1.6.4.1
 
View active_record_memory_bloat_patch_3.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
require 'rubygems'
 
require 'thread'
require 'weakref'
 
module ActiveSupport
# WeakReference is a class to represent a reference to an object that is not seen by
# the tracing phase of the garbage collector. This allows the referenced
# object to be garbage collected as if nothing is referring to it.
#
# This class provides several compatibility and performance benefits over using
# the WeakRef implementation that comes with Ruby and is compatible with Jruby.
#
# Usage:
#
# foo = Object.new
# ref = ActiveSupport::WeakReference.new(foo)
# ref.alive? # should be true
# ref.object # should be foo
# ObjectSpace.garbage_collect
# ref.alive? # should be false
# ref.object # should be nil
class WeakReference
# The object id of the object being referenced.
attr_reader :referenced_object_id
class << self
# Create a new WeakReference. The actual implementation class will be
# optimized for the runtime is being used. If the default implementation
# isn't appropriate, it can be changed by setting a different one.
def new(obj)
ref = implementation.allocate
ref.instance_variable_set(:@referenced_object_id, obj.__id__)
ref.send(:initialize, obj)
ref
end
 
# Get the implementation class for weak references.
def implementation
@implementation
end
 
# Set the implementation class for weak references.
def implementation=(klass)
@implementation = klass
end
end
 
# Get the referenced object. If the object has been reclaimed by the
# garbage collector, then this will return nil.
def object
raise NotImplementedError
end
 
def inspect
obj = object
"<##{self.class.name}: #{obj ? obj.inspect : "##{referenced_object_id} (not accessible)"}>"
end
 
# This is a pure ruby implementation of a weak reference. It is much more
# efficient than the bundled WeakRef implementation because it does not
# subclass Delegator which is very heavy to instantiate and utilizes a
# fair amount of memory.
#
# This implementation cannot be used by Jruby if ObjectSpace has been
# disabled.
class StandardImpl < WeakReference
# Map of references to the object_id's they refer to.
@@referenced_object_ids = {}
 
# Map of object_ids to references to them.
@@object_id_references = {}
 
@@mutex = Mutex.new
 
# Finalizer that cleans up weak references when an object is destroyed.
@@object_finalizer = lambda do |object_id|
@@mutex.synchronize do
reference_ids = @@object_id_references[object_id]
if reference_ids
reference_ids.each do |reference_object_id|
@@referenced_object_ids.delete(reference_object_id)
end
@@object_id_references.delete(object_id)
end
end
end
 
# Finalizer that cleans up weak references when references are destroyed.
@@reference_finalizer = lambda do |object_id|
@@mutex.synchronize do
referenced_id = @@referenced_object_ids.delete(object_id)
if referenced_id
obj = ObjectSpace._id2ref(referenced_object_id) rescue nil
if obj
backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
if backreferences
backreferences.delete(object_id)
obj.remove_instance_variable(:@__weak_backreferences__) if backreferences.empty?
end
end
references = @@object_id_references[referenced_id]
if references
references.delete(object_id)
@@object_id_references.delete(referenced_id) if references.empty?
end
end
end
end
 
# Create a new weak reference to an object. The existence of the weak reference
# will not prevent the garbage collector from reclaiming the referenced object.
def initialize(obj)
ObjectSpace.define_finalizer(obj, @@object_finalizer)
ObjectSpace.define_finalizer(self, @@reference_finalizer)
@@mutex.synchronize do
@@referenced_object_ids[self.__id__] = obj.__id__
add_backreference(obj)
references = @@object_id_references[obj.__id__]
unless references
references = []
@@object_id_references[obj.__id__] = references
end
references.push(self.__id__)
end
end
 
# Get the reference object. If the object has already been garbage collected,
# then this method will return nil.
def object
obj = nil
begin
if referenced_object_id == @@referenced_object_ids[self.object_id]
obj = ObjectSpace._id2ref(referenced_object_id)
obj = nil unless verify_backreferences(obj)
end
rescue RangeError
# Object has been garbage collected.
end
obj
end
 
private
 
def add_backreference(obj)
backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
unless backreferences
backreferences = []
obj.instance_variable_set(:@__weak_backreferences__, backreferences)
end
backreferences << object_id
end
 
def verify_backreferences(obj)
backreferences = obj.instance_variable_get(:@__weak_backreferences__) if obj.instance_variable_defined?(:@__weak_backreferences__)
backreferences && backreferences.include?(object_id)
end
end
 
# This implementation of a weak reference utilizes the weakling library which will
# use native Java weak references when running under Jruby. If the weakling gem
# has been required this will automatically be used.
class WeaklingImpl < WeakReference
def initialize(obj)
@ref = ::Weakling::WeakRef.new(obj)
end
 
def object
@ref.get
rescue RefError
nil
end
end
 
# This implementation of a weak reference simply wraps the standard WeakRef implementation
# that comes with Ruby. It is used as a fallback for Jruby if case the weakling gem
# is not available.
class WeakRefImpl < WeakReference
def initialize(obj)
@ref = ::WeakRef.new(obj)
end
 
def object
@ref.__getobj__
rescue => e
# Jruby implementation uses RefError while MRI uses WeakRef::RefError
if (defined?(RefError) && e.is_a?(RefError)) || (defined?(::WeakRef::RefError) && e.is_a?(::WeakRef::RefError))
nil
else
raise e
end
end
end
 
# This implementation can be used for testing. It implements the proper interface,
# but allows for mimicking garbage collection on demand.
class TestImpl < WeakReference
@@object_list = {}
 
def initialize(obj)
@object = obj
@@object_list[obj.__id__] = true
ObjectSpace.define_finalizer(self, lambda{@@object_list.delete(obj.__id__)})
end
 
def object
if @@object_list.include?(@object.__id__)
@object
else
@object = nil
end
end
 
# Simulate garbage collection of the objects passed in as arguments. If no objects
# are specified, all objects will be reclaimed.
def self.gc(*objects)
if objects.empty?
@@object_list = {}
else
objects.each{|obj| @@object_list.delete(obj.__id__)}
end
end
end
 
if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
# If using Jruby set the implementation depending on whether or not the weakling gem is installed.
begin
require 'weakling'
self.implementation = WeaklingImpl
rescue LoadError
self.implementation = WeakRefImpl
end
elsif defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
# If using Rubinius set the implementation to use WeakRef since it is very efficient.
self.implementation = WeakRefImpl
else
self.implementation = StandardImpl
end
end
end
 
class WeakRefTest
def self.test_object_ids
puts "Testing if object id's can be reused..."
i = 0
id_to_class = {}
 
loop do
i += 1
obj = Object.new
 
if id_to_class.key?(obj.object_id)
puts "\nFAIL: found a reused object id after #{i} iterations"
break
end
 
id_to_class[obj.object_id] = obj.class
if i % 50000 == 0
STDOUT.write('.')
STDOUT.flush
end
if i == 100000
puts "\nSUCCESS: no object id's were reused after 100,000 iterations"
break
end
end
end
 
def self.test_weak_ref_object_ids
puts "Testing if object id's can be reused by WeakRef..."
i = 0
n = 0
id_to_ref = {}
 
loop do
i += 1
obj = Object.new
 
if id_to_ref.key?(obj.object_id)
n += 1
ref = id_to_ref[obj.object_id]
if ref.weakref_alive?
puts "\nFAIL: found a reused object id after #{i} iterations"
break
end
end
 
id_to_ref[obj.object_id] = WeakRef.new(obj)
if i % 2500 == 0
STDOUT.write('.')
STDOUT.flush
end
if i == 100000
puts "\nSUCCESS: even with #{n} object id's being reused in 100,000 iterations"
break
end
end
end
def self.test_weak_reference_object_ids
puts "WeakReference implementation is #{ActiveSupport::WeakReference.implementation.name}"
i = 0
n = 0
id_to_ref = {}
 
loop do
i += 1
obj = Object.new
 
if id_to_ref.key?(obj.object_id)
n += 1
ref = id_to_ref[obj.object_id]
if ref.object
puts "\nFAIL: found a reused object id after #{i} iterations"
break
end
end
 
id_to_ref[obj.object_id] = ActiveSupport::WeakReference.new(obj)
if i % 5000 == 0
STDOUT.write('.')
STDOUT.flush
end
if i == 100000
puts "\nSUCCESS: even with #{n} object id's being reused in 100,000 iterations"
break
end
end
end
end
 
if $0 == __FILE__
if ARGV.empty?
STDERR.puts "Usage: ruby weakref_test.rb object|weakref|weakreference"
else
ARGV.each do |arg|
case arg
when "object"
t = Time.now
1000.times{Object.new}
puts "It takes #{Time.now - t} seconds to allocate 1000 Objects"
WeakRefTest.test_object_ids
when "weakref"
t = Time.now
1000.times{WeakRef.new(Object.new)}
puts "It takes #{Time.now - t} seconds to allocate 1000 WeakRefs"
WeakRefTest.test_weak_ref_object_ids
when "weakreference"
t = Time.now
1000.times{ActiveSupport::WeakReference.new(Object.new)}
puts "It takes #{Time.now - t} seconds to allocate 1000 ActiveSupport::WeakReferences"
puts "Testing if object_ids can be reused by ActiveSupport::WeakReference..."
WeakRefTest.test_weak_reference_object_ids
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.