Simple rails-ish app that demonstrates bug in paper_trail gem when using serialized fields that contain unicode characters

README.md

I've found a bug that is specific to the following scenario:

• Using MySQL
• Using a longblob column type
• Using a serialized attribute
• Using paper_trail gem
• Serialized field contains unicode character

Under those conditions, after saving changes to the serialized field, record#changed? still reports true, and record#changes contains an entry for the serialized field where both the before and after elements are identical. Calling record#reload clears the changes and loads the record with the changed value. If paper_trail is removed from the scenario, the ActiveModel attribute mutation tracking works as expected.

Documented in paper-trail-gem/paper_trail#1348

rails-serialize-unicode-test.rb

# frozen_string_literal: true

require "bundler/inline"

gemfile true do
  source "https://rubygems.org"
  gem "activerecord", "~> 6.1.4.1"
  gem "mysql2", "~> 0.5.3"
  gem "paper_trail", "~> 12.1.0"
  gem "minitest-reporters"
  gem "pry-byebug"
end

require "active_record"
require "paper_trail"
require "minitest/autorun"
require "logger"

ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "railstestdb")
ActiveRecord::Base.logger = nil
ActiveRecord::Schema.define do
  create_table :reviews, force: true do |t|
    # blob columns are not a default Rails column type, but Rails will defer to the
    # mysql2 adapter to manage the mapping
    t.tinyblob :tinyblob
    t.blob :blob
    t.mediumblob :mediumblob
    t.longblob :longblob
    t.blob :ignored_blob
    t.blob :skipped_blob

    t.text :tinytext
    t.text :text
    t.text :mediumtext
    t.longtext :longtext
  end

  create_table :versions, force: true do |t|
    t.string   :item_type, null: false
    t.integer  :item_id,   null: false
    t.string   :event,     null: false
    t.string   :whodunnit
    t.text     :object, limit: 1_073_741_823
    t.text     :object_changes, limit: 1_073_741_823
    t.datetime :created_at
  end
  add_index :versions, %i[item_type item_id]
end

# ActiveRecord::Base.logger = Logger.new(STDOUT)

class Review < ActiveRecord::Base; end

class SerializedReview < Review
  # Use default YAML serialization
  serialize :tinyblob
  serialize :blob
  serialize :mediumblob
  serialize :longblob

  serialize :tinytext
  serialize :text
  serialize :mediumtext
  serialize :longtext

  serialize :ignored_blob
  serialize :skipped_blob
end

class AuditedReview < Review
  has_paper_trail ignore: [:ignored_blob], skip: [:skipped_blob]
end

class AuditedSerializedReview < SerializedReview
  has_paper_trail
end

Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new

class ReviewTest < ActiveSupport::TestCase
  setup do
    PaperTrail.enabled = true
  end

  UNICODE_CHAR = "\u2022"

  # Serialized blob columns that use paper trail will not reset changes
  # after saving when the field contains unicode characters
  #
  # THESE TESTS FAIL
  def test_audited_serialized_tinyblob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(tinyblob: { "a" => UNICODE_CHAR }) do |record|
      record.tinyblob["b"] = "c"
    end
  end

  def test_audited_serialized_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
      record.blob["b"] = "c"
    end
  end

  def test_audited_serialized_mediumblob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(mediumblob: { "a" => UNICODE_CHAR }) do |record|
      record.mediumblob["b"] = "c"
    end
  end

  def test_audited_serialized_longblob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(longblob: { "a" => UNICODE_CHAR }) do |record|
      record.longblob["b"] = "c"
    end
  end

  # Serialized blob columns on classes that use paper trail will not reset
  # changes after saving when the field contains unicode characters, even if
  # the attribute is ignored or skipped by paper trail.
  #
  # THESE TESTS FAIL
  def test_audited_serialized_ignored_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(ignored_blob: { "a" => UNICODE_CHAR }) do |record|
      record.ignored_blob["b"] = "c"
    end
  end

  def test_audited_serialized_skipped_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview.create!(skipped_blob: { "a" => UNICODE_CHAR }) do |record|
      record.skipped_blob["b"] = "c"
    end
  end

  # Serialized blob columns on classes that use paper trail will not reset
  # changes after saving when the field contains unicode characters, even if
  # the changed fields did not include the blob column.
  #
  # THIS TEST FAILS
  def test_audited_serialized_blob_column_when_other_column_is_updated_FAILS
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
      record.text = "c"
    end
  end

  # Serialized blob columns that use paper trail WILL reset changes
  # after saving when the field contains unicode characters and PaperTrail
  # is DISABLED
  #
  # THIS TEST PASSES
  def test_audited_serialized_blob_column_paper_trail_disabled_PASSES
    PaperTrail.enabled = false
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
      record.blob["b"] = "c"
    end
  end

  # Serialized blob columns that use paper trail will not reset changes
  # after saving when the field contains unicode characters, BUT reloading
  # the record clears them and loads updated field content.
  #
  # THIS TEST PASSES
  def test_reloaded_audited_serialized_blob_column_PASSES
    record = assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }), reload_after_update: true do |record|
      record.blob["b"] = "c"
    end
    assert_equal record.blob["a"], "\u2022"
    assert_equal record.blob["b"], "c"
  end

  # Serialized _text_ columns that use paper trail WILL reset changes
  # after saving when the field contains unicode characters
  #
  # THESE TESTS PASS
  def test_audited_serialized_tinytext_column_PASSES
    assert_changes_empty AuditedSerializedReview.create!(tinytext: { "a" => UNICODE_CHAR }) do |record|
      record.tinytext["b"] = "c"
    end
  end

  def test_audited_serialized_text_column_PASSES
    assert_changes_empty AuditedSerializedReview.create!(text: { "a" => UNICODE_CHAR }) do |record|
      record.text["b"] = "c"
    end
  end

  def test_audited_serialized_mediumtext_column_PASSES
    assert_changes_empty AuditedSerializedReview.create!(mediumtext: { "a" => UNICODE_CHAR }) do |record|
      record.mediumtext["b"] = "c"
    end
  end

  def test_audited_serialized_longtext_column_PASSES
    assert_changes_empty AuditedSerializedReview.create!(longtext: { "a" => UNICODE_CHAR }) do |record|
      record.longtext["b"] = "c"
    end
  end

  # Serialized blob columns that DO NOT use paper trail WILL reset changes
  # after saving when the field contains unicode characters
  #
  # THIS TEST PASSES
  def test_unaudited_serialized_blob_column_PASSES
    assert_changes_empty SerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
      record.blob["b"] = "c"
    end
  end

  # Serialized blob columns that use paper trail WILL reset changes
  # after saving when the field DOES NOT contain unicode characters
  #
  # THIS TEST PASSES
  def test_audited_serialized_blob_column_without_unicode_PASSES
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => "not unicode" }) do |record|
      record.blob["b"] = "c"
    end
  end

  # UN-serialized blob columns that use paper trail WILL reset changes
  # after saving when the field contains unicode characters
  #
  # THIS TEST PASSES
  def test_audited_unserialized_blob_column_PASSES
    assert_changes_empty AuditedReview.create!(blob: UNICODE_CHAR) do |record|
      record.blob = "c"
    end
  end

  def assert_changes_empty(record, options = {})
    reload_after_update = options.delete(:reload_after_update) || false

    # Changes are empty after creation
    assert_empty record.changes

    # Changes are populated after updating serialized column
    yield(record)
    refute_empty record.changes
    record.save!

    # This should have been reset after `save!` and should be empty
    record.reload if reload_after_update
    assert_empty record.changes

    record
  end
end

rails-serialize-unicode-test.rb.txt

ReviewTest
  test_audited_serialized_blob_column_FAILS                       FAIL (0.09s)
        Expected {"blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:96:in `test_audited_serialized_blob_column_FAILS'

  test_audited_serialized_tinyblob_column_FAILS                   FAIL (0.05s)
        Expected {"tinyblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:90:in `test_audited_serialized_tinyblob_column_FAILS'

  test_audited_serialized_mediumtext_column_PASSES                PASS (0.05s)
  test_audited_serialized_blob_column_without_unicode_PASSES      PASS (0.05s)
  test_audited_serialized_longtext_column_PASSES                  PASS (0.05s)
  test_reloaded_audited_serialized_blob_column_PASSES             PASS (0.06s)
  test_audited_serialized_blob_column_when_other_column_is_updated_FAILS FAIL (0.05s)
        Expected {"blob"=>[{"a"=>"•"}, {"a"=>"•"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:136:in `test_audited_serialized_blob_column_when_other_column_is_updated_FAILS'

  test_audited_serialized_text_column_PASSES                      PASS (0.05s)
  test_audited_serialized_tinytext_column_PASSES                  PASS (0.05s)
  test_audited_unserialized_blob_column_PASSES                    PASS (0.06s)
  test_audited_serialized_blob_column_paper_trail_disabled_PASSES PASS (0.05s)
  test_audited_serialized_mediumblob_column_FAILS                 FAIL (0.05s)
        Expected {"mediumblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:102:in `test_audited_serialized_mediumblob_column_FAILS'

  test_audited_serialized_ignored_blob_column_FAILS               FAIL (0.05s)
        Expected {"ignored_blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:119:in `test_audited_serialized_ignored_blob_column_FAILS'

  test_unaudited_serialized_blob_column_PASSES                    PASS (0.05s)
  test_audited_serialized_skipped_blob_column_FAILS               FAIL (0.05s)
        Expected {"skipped_blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:125:in `test_audited_serialized_skipped_blob_column_FAILS'

  test_audited_serialized_longblob_column_FAILS                   FAIL (0.05s)
        Expected {"longblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
        /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:108:in `test_audited_serialized_longblob_column_FAILS'

structure.sql

-- $ mysqldump --compact=ON --set-gtid-purged=OFF --no-data=ON railstestdb

/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `ar_internal_metadata` (
  `key` varchar(255) NOT NULL,
  `value` varchar(255) DEFAULT NULL,
  `created_at` datetime(6) NOT NULL,
  `updated_at` datetime(6) NOT NULL,
  PRIMARY KEY (`key`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `reviews` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT,
  `longblob` longblob,
  `longtext` longtext,
  `text` text,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `versions` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT,
  `item_type` varchar(255) NOT NULL,
  `item_id` int(11) NOT NULL,
  `event` varchar(255) NOT NULL,
  `whodunnit` varchar(255) DEFAULT NULL,
  `object` longtext,
  `object_changes` longtext,
  `created_at` datetime DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `index_versions_on_item_type_and_item_id` (`item_type`,`item_id`)
) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;