perf shows that even accessing thread's own stack can be painful. Does RocksDB compare keys in the non-temporal way (MOVNT* of x86) to avoid massive cache pollution?

perf-annotate.cache-misses.txt

rocksdb::MemTable::KeyComparator::operator()  /work/ceph-3/build/lib/libfio_ceph_objectstore.so
       │        _ZN7rocksdb14GetVarint32PtrEPKcS1_Pj():
       │                                                  uint32_t* value);
       │        inline const char* GetVarint32Ptr(const char* p,
       │                                          const char* limit,
       │                                          uint32_t* value) {
       │          if (p < limit) {
       │            uint32_t result = *(reinterpret_cast<const unsigned char*>(p));
  0,43 │42b266:   movzbl (%rsi),%edx
       │        _ZN7rocksdb22GetLengthPrefixedSliceEPKc():
       │            return false;
       │          }
       │        }
       │
       │        inline Slice GetLengthPrefixedSlice(const char* data) {
       │          uint32_t len = 0;
 83,15 │42b269:   movl   $0x0,-0x40(%rbp)
       │        _ZNK7rocksdb8MemTable13KeyComparatorclEPKcS3_():
  0,16 │42b270:   mov    %fs:0x28,%rax
  0,17 │42b279:   mov    %rax,-0x28(%rbp)
  0,02 │42b27d:   xor    %eax,%eax
       │        _ZN7rocksdb14GetVarint32PtrEPKcS1_Pj():
       │        inline const char* GetVarint32Ptr(const char* p,
       │                                          const char* limit,
       │                                          uint32_t* value) {
       │          if (p < limit) {
       │            uint32_t result = *(reinterpret_cast<const unsigned char*>(p));
       │            if ((result & 128) == 0) {
  0,47 │42b27f:   test   %dl,%dl
       │42b281: ↓ js     42b2f0 <rocksdb::MemTable::KeyComparator::operator()(char const*, char const*) const+0xa0>
       │              *value = result;
       │              return p + 1;
Press 'h' for help on key bindings

perf-annotate.cycles_pebs.txt

rocksdb::MemTable::KeyComparator::operator()  /work/ceph-3/build/lib/libfio_ceph_objectstore.so
       │    _ZN7rocksdb14GetVarint32PtrEPKcS1_Pj():
       │                                              uint32_t* value);
       │    inline const char* GetVarint32Ptr(const char* p,
       │                                      const char* limit,
       │                                      uint32_t* value) {
       │      if (p < limit) {
       │        uint32_t result = *(reinterpret_cast<const unsigned char*>(p));
  0,19 │      movzbl (%rsi),%edx
       │    _ZN7rocksdb22GetLengthPrefixedSliceEPKc():
       │        return false;
       │      }
       │    }
       │
       │    inline Slice GetLengthPrefixedSlice(const char* data) {
       │      uint32_t len = 0;
 88,78 │      movl   $0x0,-0x40(%rbp)
       │    _ZNK7rocksdb8MemTable13KeyComparatorclEPKcS3_():
  0,52 │      mov    %fs:0x28,%rax
  0,14 │      mov    %rax,-0x28(%rbp)
  0,02 │      xor    %eax,%eax
       │    _ZN7rocksdb14GetVarint32PtrEPKcS1_Pj():
       │    inline const char* GetVarint32Ptr(const char* p,
       │                                      const char* limit,
       │                                      uint32_t* value) {
       │      if (p < limit) {
       │        uint32_t result = *(reinterpret_cast<const unsigned char*>(p));
       │        if ((result & 128) == 0) {
  0,60 │      test   %dl,%dl
       │    ↓ js     a0
       │          *value = result;
       │          return p + 1;
Press 'h' for help on key bindings

perf-histogram.cycles_pebs.txt

+   11,15%     0,18%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] BlueStore::_kv_sync_thread
+   11,12%     0,00%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] BlueStore::KVSyncThread::entry
+   11,12%     0,00%  bstore_kv_sync  libpthread-2.23.so          [.] start_thread
+   11,09%     0,00%  bstore_kv_sync  libc-2.23.so                [.] __clone
-    8,76%     0,00%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] RocksDBStore::submit_transaction_sync
   - 8,75% RocksDBStore::submit_transaction_sync
      - 8,73% RocksDBStore::submit_common
         - 8,72% rocksdb::DBImpl::Write
            - rocksdb::DBImpl::WriteImpl
               - 7,06% rocksdb::WriteBatchInternal::InsertInto
                  - 7,06% rocksdb::WriteBatch::Iterate
                     - 6,81% rocksdb::MemTableInserter::PutCF
                        - 6,73% rocksdb::MemTable::Add
                           - 6,37% rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Insert<false>
                              - 5,61% rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::RecomputeSpliceLevels
                                 - rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::FindSpliceForLevel
                                    - rocksdb::MemTable::KeyComparator::operator()
                                       - rocksdb::InternalKeyComparator::Compare
                                          - rocksdb::(anonymous namespace)::BytewiseComparatorImpl::Compare
                                               1,62% __memcmp_sse4_1
                                0,53% rocksdb::MemTable::KeyComparator::operator()
               + 1,56% rocksdb::DBImpl::WriteToWAL
+    8,73%     0,01%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] RocksDBStore::submit_common
+    8,72%     0,02%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::DBImpl::WriteImpl
+    8,72%     0,00%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::DBImpl::Write
+    8,26%     0,13%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::WriteBatch::Iterate
+    7,06%     0,01%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::WriteBatchInternal::InsertInto
+    6,99%     0,08%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::MemTable::Add
+    6,81%     0,06%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::MemTableInserter::PutCF
+    6,50%     0,25%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Insert<false>
+    5,78%     3,22%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::MemTable::KeyComparator::operator()
+    5,70%     0,06%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::RecomputeSpliceLevels
+    5,65%     0,48%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::FindSpliceForLevel
+    2,51%     0,45%  bstore_kv_sync  libfio_ceph_objectstore.so  [.] rocksdb::InternalKeyComparator::Compare
+    2,13%     2,09%  bstore_kv_sync  libc-2.23.so                [.] __memcmp_sse4_1