Ceph issue #38827
- discovered during QA run of ceph/ceph#27012
../src/vstart.sh -l -n -b --msgr21 -o "ms_mon_cluster_mode=secure" -o "ms_mon_service_mode=secure" -o "ms_mon_client_mode=secure" -o "ms_service_mode=secure" -o "ms_cluster_mode=secure" -o "ms_client_mode=secure"
The crucial thing when spawing valgrind without XML output is to specify --error-limit=no. Otherwise the problem will be hidden.
valgrind --error-limit=no --leak-check=full --trace-children=no --child-silent-after-fork=no -- bin/ceph-mon -i a -c ./ceph.conf -f -d --debug_ms=30
valgrind -v --leak-check=full --trace-children=yes --child-silent-after-fork=no --log-fd=1 --error-limit=no --track-origins=yes --show-mismatched-frees=no -- bin/ceph-mon -i a -c ./ceph.conf -f -d --debug_ms=30 2>&1 | tee val_mon.a.log_dbg
(gdb) bt
#0 0x0000000010826e7c in aes_gcm_cipher () from /lib64/libcrypto.so.10
#1 0x0000000010822dd7 in EVP_DecryptFinal_ex () from /lib64/libcrypto.so.10
#2 0x0000000005386045 in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update_final(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x19946d40,
ciphertext_and_tag=<optimized out>, alignment=8) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:267
#3 0x0000000005375902 in ProtocolV2::handle_read_frame_epilogue_main(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x198e81f0,
buffer=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3a5cdd1, DIE 0x3bbeeb4>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:1264
#4 0x000000000535b2d4 in ProtocolV2::run_continuation (this=this@entry=0x198e81f0, continuation=...) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:45
#5 0x000000000535b6af in ProtocolV2::read_event (this=0x198e81f0) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:451
#6 0x000000000532a244 in AsyncConnection::process (this=0x198e5aa0) at /work/ceph-rzarzynski-3/src/msg/async/AsyncConnection.cc:446
#7 0x000000000537f0a7 in EventCenter::process_events (this=this@entry=0x15d2e080, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x1f5c6720)
at /work/ceph-rzarzynski-3/src/msg/async/Event.cc:441
#8 0x00000000053835b5 in operator() (__closure=0x19609578) at /work/ceph-rzarzynski-3/src/msg/async/Stack.cc:53
#9 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316
#10 0x000000000561467f in execute_native_thread_routine () from /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0
#11 0x0000000010b60e25 in start_thread (arg=0x1f5c9700) at pthread_create.c:308
#12 0x0000000011b3334d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Dump of assembler code for function aes_gcm_cipher:
...
0x0000000010826e75 <+581>: callq 0x107bcb10 <CRYPTO_gcm128_finish>
0x0000000010826e7a <+586>: test %eax,%eax
=> 0x0000000010826e7c <+588>: jne 0x10826ce0 <aes_gcm_cipher+176>
0x0000000010826e82 <+594>: movl $0x0,0xfc(%rbx)
static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
const unsigned char *in, size_t len)
{
EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
/* If not set up, return error */
if (!gctx->key_set)
return -1;
if (gctx->tls_aad_len >= 0)
return aes_gcm_tls_cipher(ctx, out, in, len);
if (!gctx->iv_set)
return -1;
if (in) {
// ...
} else {
if (!ctx->encrypt) {
if (gctx->taglen < 0)
return -1;
if (CRYPTO_gcm128_finish(&gctx->gcm, ctx->buf, gctx->taglen) != 0)
return -1;
gctx->iv_set = 0;
return 0;
}
CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16);
gctx->taglen = 16;
/* Don't reuse the IV */
gctx->iv_set = 0;
return 0;
}
}static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
{
EVP_AES_GCM_CTX *gctx = c->cipher_data;
switch (type) {
// ...
case EVP_CTRL_GCM_SET_TAG:
if (arg <= 0 || arg > 16 || c->encrypt)
return 0;
memcpy(c->buf, ptr, arg);
gctx->taglen = arg;
return 1;(gdb) frame 0
#0 0x0000000010827e7c in aes_gcm_cipher () from /lib64/libcrypto.so.10
(gdb) info registers rdi rsi rdx
rdi 0x21e65310 568742672
rsi 0x21e65118 568742168
rdx 0x10 16
(gdb) monitor xb 0x21e65118 0x10
00 00 00 00 00 00 00 00
0x21E65118: 0xef 0xbd 0x35 0x20 0x95 0x57 0x5d 0xee
00 00 00 00 00 00 00 00
0x21E65120: 0x64 0x43 0xb3 0x08 0x95 0x51 0x1b 0xfa
(gdb) frame 2
#2 0x0000000005386a46 in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update_final(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x21e65070,
ciphertext_and_tag=<optimized out>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:326
326 if (0 >= EVP_DecryptFinal_ex(ectx.get(), nullptr, &final_len)) {
(gdb) print auth_tag
$1 = {_buffers = {_root = {next = 0x21e6a380}, _tail = 0x21e6a380, _size = 1}, _carriage = 0xbc8e00 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 16, _memcopy_count = 0,
last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0x205c8de0, ls = 0x205c8de0, p = {cur = 0x21e6a380}, off = 0, p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0,
_off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}
...
(gdb) print (ceph::buffer::v14_2_0::ptr_node *)0x21e6a380
$2 = (ceph::buffer::v14_2_0::ptr_node *) 0x21e6a380
(gdb) print *(ceph::buffer::v14_2_0::ptr_node *)0x21e6a380
$3 = {<ceph::buffer::v14_2_0::ptr_hook> = {next = 0x205c8de0}, <ceph::buffer::v14_2_0::ptr> = {_raw = 0x21e6a2e0, _off = 16, _len = 16}, <No data fields>}
(gdb) print *(ceph::buffer::raw *)0x21e6a2e0
$4 = {_vptr.raw = 0x59a1db0 <vtable for ceph::buffer::raw_combined+16>, bptr_storage = {__data = '\000' <repeats 23 times>, __align = {<No data fields>}},
data = 0x21e6a2c0 "C\003\245\017\377;<\211\211\247\263\240\366\a\360\367\357\275\065 \225W]\356dC\263\b\225Q\033\372\260\035\232\005", len = 32, nref = {<std::__atomic_base<unsigned int>> = {
static _S_alignment = 4, _M_i = 1}, static is_always_lock_free = true}, mempool = 10, last_crc_offset = {first = 18446744073709551615, second = 18446744073709551615}, last_crc_val = {first = 0,
second = 0}, crc_spinlock = {af = {<std::__atomic_flag_base> = {_M_i = false}, <No data fields>}}}
(gdb) mo xb 0x21e6a2d0 16
00 00 00 00 00 00 00 00
0x21E6A2D0: 0xef 0xbd 0x35 0x20 0x95 0x57 0x5d 0xee
00 00 00 00 00 00 00 00
0x21E6A2D8: 0x64 0x43 0xb3 0x08 0x95 0x51 0x1b 0xfa
Oops, auth_tag in authenticated_decrypt_update_final is demaged as well. It bases on data that came from the network.
These definedness bits have been interpreted wrongly. According to the manual they operate in inversed logic:
These hexadecimal digits encode the validity of each bit of the corresponding byte, using 0 if the bit is defined and 1 if the bit is undefined.
(gdb) break ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update_final
...
(gdb) break CRYPTO_gcm128_finish
...
(gdb) next
1707 return CRYPTO_memcmp(ctx->Xi.c, tag, len);
(gdb) print &ctx->Xi.c
$8 = (u8 (*)[16]) 0x1ea551e0
(gdb) mo xb 0x1ea551e0 16
ff ff ff ff ff ff ff ff
0x1EA551E0: 0xe2 0xb1 0x69 0xd0 0x02 0x41 0x1c 0xcb
ff ff ff ff ff ff ff ff
0x1EA551E8: 0xa6 0xc7 0xb4 0x50 0x3b 0xb6 0x6f 0xd5
struct gcm128_context {
/* Following 6 names follow names in GCM specification */
union {
u64 u[2];
u32 d[4];
u8 c[16];
size_t t[16 / sizeof(size_t)];
} Yi, EKi, EK0, len, Xi, H;
Ciphertext inputs for EVP_DecryptFinal have been validated and they look fine. Also nonce and key used for cipher initialization appear defined.
OPENSSL_ia32cap="~0x200000200000000" valgrind -v --leak-check=full --trace-children=yes --child-silent-after-fork=no --log-fd=1 --error-limit=no --track-origins=yes --show-mismatched-frees=no --free-fill=0x88 -- bin/ceph-mon -i a -c ./ceph.conf -f -d --debug_ms=30 2>&1 | tee val_mon.a.log_dbg
$ valgrind -v --leak-check=full --trace-children=yes --child-silent-after-fork=no --log-fd=1 --error-limit=no --show-mismatched-frees=no --vgdb=full --vgdb-error=0 -- bin/ceph-mon -i a -c ./ceph.conf -f -d --debug_ms=30 2>&1 | tee val_mon.a.log_dbg
(gdb) break ghash-x86_64.s:1734
No source file named ghash-x86_64.s.
Make breakpoint pending on future shared library load? (y or [n]) y
Breakpoint 2 (ghash-x86_64.s:1734) pending.
(gdb) cont
Continuing.
warning: File "/usr/local/lib64/libstdc++.so.6.0.24-gdb.py" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load:/usr/bin/mono-gdb.py".
To enable execution of this file add
add-auto-load-safe-path /usr/local/lib64/libstdc++.so.6.0.24-gdb.py
line to your configuration file "/home/rzarzynski/.gdbinit".
To completely disable this security protection add
set auto-load safe-path /
line to your configuration file "/home/rzarzynski/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
info "(gdb)Auto-loading safe path"
[New Thread 29685]
[Switching to Thread 29685]
(gdb) cont
Continuing.
Breakpoint 1, gcm_ghash_avx () at ghash-x86_64.s:1365
1365 vzeroupper
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1365
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199361e0, in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005",
out=0x1993adb0 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x1993adb0 "", in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005", len=32)
at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x19935ff0, out=out@entry=0x1993adb0 "", outl=outl@entry=0x1f5c5ee0,
in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x19935f80,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000535f951 in ProtocolV2::handle_read_frame_preamble_main(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x198d5f10,
buffer=<optimized out>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:985
#6 0x000000000535b2d4 in ProtocolV2::run_continuation (this=this@entry=0x198d5f10, continuation=...) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:45
#7 0x000000000535b6af in ProtocolV2::read_event (this=0x198d5f10) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:451
#8 0x000000000532a244 in AsyncConnection::process (this=0x198d37c0) at /work/ceph-rzarzynski-3/src/msg/async/AsyncConnection.cc:446
#9 0x000000000537f0a7 in EventCenter::process_events (this=this@entry=0x15d22b30, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x1f5c6720)
at /work/ceph-rzarzynski-3/src/msg/async/Event.cc:441
#10 0x00000000053835b5 in operator() (__closure=0x15d26eb8) at /work/ceph-rzarzynski-3/src/msg/async/Stack.cc:53
#11 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316
#12 0x000000000561467f in execute_native_thread_routine () from /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0
#13 0x0000000010b60e25 in start_thread (arg=0x1f5c9700) at pthread_create.c:308
#14 0x0000000011b3334d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) frame 1
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199361e0, in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005",
out=0x1993adb0 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
1617 GHASH(ctx, in, i);
(gdb) print &ctx->Xi
$1 = (union {...} *) 0x19936220
(gdb) mo xb 0x19936220 16
00 00 00 00 00 00 00 00
0x19936220: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936228: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
(gdb) info registers rdi rsi rdx rcx
rdi 0x19936220 429089312
rsi 0x19936240 429089344
rdx 0x19937e50 429096528
rcx 0x20 32
(gdb) # input validation
(gdb) mo xb 0x19937e50 32
00 00 00 00 00 00 00 00
0x19937E50: 0x34 0x3b 0xc4 0x15 0x90 0xa4 0xe3 0x69
00 00 00 00 00 00 00 00
0x19937E58: 0xf0 0x09 0x66 0x8c 0x3a 0x7c 0x22 0x32
00 00 00 00 00 00 00 00
0x19937E60: 0xcc 0x95 0x1e 0x98 0x56 0x0a 0x78 0xb4
00 00 00 00 00 00 00 00
0x19937E68: 0x94 0xe1 0x39 0x7b 0xf4 0xde 0x87 0xe9
(gdb) mo xb 0x19936240 256
00 00 00 00 00 00 00 00
0x19936240: 0x83 0xaa 0x4a 0x8e 0x5d 0x68 0xc8 0x63
00 00 00 00 00 00 00 00
0x19936248: 0xe7 0x9c 0xd6 0x1f 0x30 0x67 0xc9 0x4d
00 00 00 00 00 00 00 00
0x19936250: 0x93 0xac 0xaf 0xbe 0x43 0xe5 0xbe 0x13
00 00 00 00 00 00 00 00
0x19936258: 0x8f 0x98 0x4d 0x8f 0x91 0xf6 0x6b 0x97
00 00 00 00 00 00 00 00
0x19936260: 0x64 0x36 0x9c 0x91 0x6d 0x0f 0x01 0x2e
00 00 00 00 00 00 00 00
0x19936268: 0x1c 0x34 0xe2 0x31 0xd2 0x13 0xd5 0x84
00 00 00 00 00 00 00 00
0x19936270: 0x8a 0x41 0xa0 0xac 0x64 0x85 0x54 0x90
00 00 00 00 00 00 00 00
0x19936278: 0xee 0x21 0xc9 0xaa 0xea 0x37 0x18 0xb9
00 00 00 00 00 00 00 00
0x19936280: 0x11 0x91 0xa4 0x5c 0xb3 0xcf 0x74 0xfc
00 00 00 00 00 00 00 00
0x19936288: 0x27 0xbf 0xa1 0x96 0x64 0xa8 0xc5 0xdc
00 00 00 00 00 00 00 00
0x19936290: 0x64 0x60 0x69 0x06 0x8e 0xb2 0x4c 0x29
00 00 00 00 00 00 00 00
0x19936298: 0x36 0x2e 0x05 0xca 0xd7 0x67 0xb1 0x20
00 00 00 00 00 00 00 00
0x199362A0: 0x68 0x80 0xfb 0x7f 0x96 0xf4 0xee 0x42
00 00 00 00 00 00 00 00
0x199362A8: 0x2a 0x0f 0x79 0x62 0x7d 0xd9 0x9f 0x89
00 00 00 00 00 00 00 00
0x199362B0: 0xa3 0xe6 0x20 0x18 0xc9 0x87 0xa0 0x13
00 00 00 00 00 00 00 00
0x199362B8: 0x70 0xad 0x88 0x55 0xbe 0x1c 0x13 0xdb
00 00 00 00 00 00 00 00
0x199362C0: 0x42 0x8f 0x82 0x1d 0xeb 0x2d 0x71 0xcb
00 00 00 00 00 00 00 00
0x199362C8: 0xd3 0x4b 0xa8 0x4d 0x77 0x9b 0xb3 0xc8
00 00 00 00 00 00 00 00
0x199362D0: 0x40 0xfb 0x89 0xb2 0xa9 0x1e 0x54 0x04
00 00 00 00 00 00 00 00
0x199362D8: 0xdd 0x62 0x8c 0xf1 0xe6 0xd5 0xc2 0x54
00 00 00 00 00 00 00 00
0x199362E0: 0xd6 0xc7 0x16 0x14 0x95 0x93 0x9a 0x96
00 00 00 00 00 00 00 00
0x199362E8: 0x5c 0x08 0x8a 0x08 0xe7 0xea 0x52 0xa9
00 00 00 00 00 00 00 00
0x199362F0: 0x8a 0xcf 0x9c 0x1c 0x72 0x79 0xc8 0x3f
00 00 00 00 00 00 00 00
0x199362F8: 0x9d 0x99 0x05 0x43 0x4f 0xcb 0x96 0x50
00 00 00 00 00 00 00 00
0x19936300: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936308: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936310: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936318: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936320: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936328: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936330: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x19936338: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
Breakpoint 2, gcm_ghash_avx () at ghash-x86_64.s:1734
1734 vzeroupper
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1734
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199361e0, in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005",
out=0x1993adb0 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x1993adb0 "", in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005", len=32)
at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x19935ff0, out=out@entry=0x1993adb0 "", outl=outl@entry=0x1f5c5ee0,
in=0x19937e50 "4;\304\025\220\244\343i\360\tf\214:|\"2̕\036\230V\nx\264\224\341\071{\364އ\351\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x19935f80,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000535f951 in ProtocolV2::handle_read_frame_preamble_main(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x198d5f10,
buffer=<optimized out>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:985
#6 0x000000000535b2d4 in ProtocolV2::run_continuation (this=this@entry=0x198d5f10, continuation=...) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:45
#7 0x000000000535b6af in ProtocolV2::read_event (this=0x198d5f10) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:451
#8 0x000000000532a244 in AsyncConnection::process (this=0x198d37c0) at /work/ceph-rzarzynski-3/src/msg/async/AsyncConnection.cc:446
#9 0x000000000537f0a7 in EventCenter::process_events (this=this@entry=0x15d22b30, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x1f5c6720)
at /work/ceph-rzarzynski-3/src/msg/async/Event.cc:441
#10 0x00000000053835b5 in operator() (__closure=0x15d26eb8) at /work/ceph-rzarzynski-3/src/msg/async/Stack.cc:53
#11 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316
#12 0x000000000561467f in execute_native_thread_routine () from /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0
#13 0x0000000010b60e25 in start_thread (arg=0x1f5c9700) at pthread_create.c:308
#14 0x0000000011b3334d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) mo xb 0x19936220 16
00 00 00 00 00 00 00 00
0x19936220: 0xd8 0x98 0xc8 0xa5 0x0a 0x02 0xc9 0x47
00 00 00 00 00 00 00 00
0x19936228: 0x95 0xd0 0x00 0x31 0xb6 0xfc 0x74 0x23
(gdb) cont
Continuing.
Breakpoint 1, gcm_ghash_avx () at ghash-x86_64.s:1365
1365 vzeroupper
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1365
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199361e0, in=0x1993ae70 "ϩl\375?\257\236\223a\364\f]B\006\276n\021\300̉h\245\324d\306z$\237\252ط\273\260\r\232\005",
out=0x1993af90 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x1993af90 "", in=0x1993ae70 "ϩl\375?\257\236\223a\364\f]B\006\276n\021\300̉h\245\324d\306z$\237\252ط\273\260\r\232\005", len=32)
at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x19935ff0, out=out@entry=0x1993af90 "", outl=outl@entry=0x1f5c5f10,
in=0x1993ae70 "ϩl\375?\257\236\223a\364\f]B\006\276n\021\300̉h\245\324d\306z$\237\252ط\273\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x19935f80,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000536078e in ProtocolV2::handle_read_frame_segment(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x198d5f10,
rx_buffer=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3a5cdd1, DIE 0x3bc2653>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:1135
#6 0x000000000535b2d4 in ProtocolV2::run_continuation (this=this@entry=0x198d5f10, continuation=...) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:45
#7 0x000000000535b6af in ProtocolV2::read_event (this=0x198d5f10) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:451
#8 0x000000000532a244 in AsyncConnection::process (this=0x198d37c0) at /work/ceph-rzarzynski-3/src/msg/async/AsyncConnection.cc:446
#9 0x000000000537f0a7 in EventCenter::process_events (this=this@entry=0x15d22b30, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x1f5c6720)
at /work/ceph-rzarzynski-3/src/msg/async/Event.cc:441
#10 0x00000000053835b5 in operator() (__closure=0x15d26eb8) at /work/ceph-rzarzynski-3/src/msg/async/Stack.cc:53
#11 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_function.h:316
#12 0x000000000561467f in execute_native_thread_routine () from /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0
#13 0x0000000010b60e25 in start_thread (arg=0x1f5c9700) at pthread_create.c:308
#14 0x0000000011b3334d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) mo xb 0x19936220 16
00 00 00 00 00 00 00 00
0x19936220: 0xd8 0x98 0xc8 0xa5 0x0a 0x02 0xc9 0x47
00 00 00 00 00 00 00 00
0x19936228: 0x95 0xd0 0x00 0x31 0xb6 0xfc 0x74 0x23
(gdb) mo xb 0x1993ae70 32
00 00 00 00 00 00 00 00
0x1993AE70: 0xcf 0xa9 0x6c 0xfd 0x3f 0xaf 0x9e 0x93
00 00 00 00 00 00 00 00
0x1993AE78: 0x61 0xf4 0x0c 0x5d 0x42 0x06 0xbe 0x6e
00 00 00 00 00 00 00 00
0x1993AE80: 0x11 0xc0 0xcc 0x89 0x68 0xa5 0xd4 0x64
00 00 00 00 00 00 00 00
0x1993AE88: 0xc6 0x7a 0x24 0x9f 0xaa 0xd8 0xb7 0xbb
(gdb) cont
Continuing.
Breakpoint 2, gcm_ghash_avx () at ghash-x86_64.s:1734
1734 vzeroupper
(gdb) mo xb 0x19936220 16
ff ff ff ff ff ff ff ff
0x19936220: 0xbc 0x57 0x2a 0x2c 0x65 0x8a 0x8c 0xdb
ff ff ff ff ff ff ff ff
0x19936228: 0xea 0x58 0x4d 0xb9 0xed 0x11 0xa4 0x68
# define GHASH(ctx,in,len) (*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len)The idea is to use only 2 (1 for entry, 1 for ret) breakpoints and manual input validation to prove the problem is contained within gcm_ghash_avx procedure of OpenSSL.
(gdb) target remote | /opt/rh/devtoolset-7/root/usr/lib64/valgrind/../../bin/vgdb --pid=29794
...
(gdb) break ghash-x86_64.s:1734
...
(gdb) break ghash-x86_64.s:1365
...
(gdb) cont
Breakpoint 2, gcm_ghash_avx () at ghash-x86_64.s:1365
1365 vzeroupper
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1365
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199bd0b0, in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005",
out=0x199bdd90 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x199bdd90 "", in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005",
len=32) at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x199bcec0, out=out@entry=0x199bdd90 "", outl=outl@entry=0x1f5c5ee0,
in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x199bce50,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000535f951 in ProtocolV2::handle_read_frame_preamble_main(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x19919880,
buffer=<optimized out>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:985
...
(gdb) frame 1
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199bd0b0, in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005",
out=0x199bdd90 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
1617 GHASH(ctx, in, i);
(gdb) print &ctx->Xi
$1 = (union {...} *) 0x199bd0f0
(gdb) print &ctx->Htable
$2 = (u128 (*)[16]) 0x199bd110
(gdb) mo xb 0x199bd0f0 16
00 00 00 00 00 00 00 00
0x199BD0F0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD0F8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
(gdb) mo xb 0x199bd110 256
00 00 00 00 00 00 00 00
0x199BD110: 0x91 0x09 0x86 0x5b 0x8e 0x76 0x75 0x3e
00 00 00 00 00 00 00 00
0x199BD118: 0x6d 0xe2 0xbc 0xde 0x45 0x17 0xae 0xef
00 00 00 00 00 00 00 00
0x199BD120: 0x12 0x27 0x75 0x7c 0xcb 0x7f 0xc0 0xe4
00 00 00 00 00 00 00 00
0x199BD128: 0xfa 0x48 0xe4 0xf9 0xa3 0xf6 0xf0 0xc0
00 00 00 00 00 00 00 00
0x199BD130: 0xfc 0xeb 0x3a 0x85 0xcb 0x61 0xdb 0xd1
00 00 00 00 00 00 00 00
0x199BD138: 0xe8 0x6f 0x91 0x85 0x68 0x89 0x30 0x24
00 00 00 00 00 00 00 00
0x199BD140: 0xa3 0x98 0x19 0xb5 0x13 0x55 0x99 0xd8
00 00 00 00 00 00 00 00
0x199BD148: 0x78 0xb2 0x1e 0xa8 0x1a 0x38 0xb8 0xa7
00 00 00 00 00 00 00 00
0x199BD150: 0x81 0xbe 0x52 0xf5 0xe7 0xee 0xc7 0x05
00 00 00 00 00 00 00 00
0x199BD158: 0xd3 0x42 0x94 0x4f 0xa0 0x19 0xa4 0x35
00 00 00 00 00 00 00 00
0x199BD160: 0xdb 0x2a 0x07 0x1d 0x09 0x6d 0x21 0x7f
00 00 00 00 00 00 00 00
0x199BD168: 0x52 0xfc 0xc6 0xba 0x47 0xf7 0x63 0x30
00 00 00 00 00 00 00 00
0x199BD170: 0x0d 0xb2 0xca 0xdc 0x5c 0x97 0x87 0x78
00 00 00 00 00 00 00 00
0x199BD178: 0x4d 0x22 0x13 0xce 0xbb 0xf8 0x2b 0x8b
00 00 00 00 00 00 00 00
0x199BD180: 0xce 0xe2 0x36 0x76 0xe7 0xe2 0x17 0xf8
00 00 00 00 00 00 00 00
0x199BD188: 0x80 0xe2 0xbf 0x5a 0x73 0xb4 0x13 0xb5
00 00 00 00 00 00 00 00
0x199BD190: 0x40 0x90 0xd9 0x12 0xe7 0x6f 0xac 0xf3
00 00 00 00 00 00 00 00
0x199BD198: 0x4e 0x00 0x89 0x2c 0x94 0x56 0x04 0x4d
00 00 00 00 00 00 00 00
0x199BD1A0: 0xe4 0x51 0x7f 0xeb 0x9c 0xff 0xaa 0x02
00 00 00 00 00 00 00 00
0x199BD1A8: 0x26 0x0b 0xec 0x6c 0x0e 0x6f 0xe7 0xf9
00 00 00 00 00 00 00 00
0x199BD1B0: 0x84 0x89 0x25 0x6e 0xf5 0xbe 0x22 0x90
00 00 00 00 00 00 00 00
0x199BD1B8: 0xb2 0x87 0x82 0x80 0xaa 0xca 0x08 0x97
00 00 00 00 00 00 00 00
0x199BD1C0: 0x36 0x0e 0xa7 0xee 0x5f 0x74 0x2a 0x07
00 00 00 00 00 00 00 00
0x199BD1C8: 0xc2 0x5a 0x93 0x87 0x92 0x90 0x4d 0xfb
00 00 00 00 00 00 00 00
0x199BD1D0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1D8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1E0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1E8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1F0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1F8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD200: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD208: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
(gdb) info registers rdi rsi rdx rcx
rdi 0x199bd0f0 429641968
rsi 0x199bd110 429642000
rdx 0x199bdc70 429644912
rcx 0x20 32
(gdb) mo xb 0x199bdc70 32
00 00 00 00 00 00 00 00
0x199BDC70: 0x02 0x2d 0xe6 0x42 0xdb 0x1a 0x11 0xb2
00 00 00 00 00 00 00 00
0x199BDC78: 0xde 0x67 0xbc 0xf5 0x1e 0x63 0x7c 0xd9
00 00 00 00 00 00 00 00
0x199BDC80: 0x8d 0xbd 0xe6 0x5d 0x49 0xb2 0x10 0x4e
00 00 00 00 00 00 00 00
0x199BDC88: 0xaa 0xc8 0x1f 0x72 0x1c 0xdd 0x5b 0x76
(gdb) cont
Continuing.
Breakpoint 1, gcm_ghash_avx () at ghash-x86_64.s:1734
1734 vzeroupper
(gdb) mo xb 0x199bd0f0 16
00 00 00 00 00 00 00 00
0x199BD0F0: 0x3b 0xc9 0x18 0x01 0x32 0xfa 0x25 0xe5
00 00 00 00 00 00 00 00
0x199BD0F8: 0xb6 0xc2 0xca 0x30 0x37 0x88 0x45 0xd8
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1734
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199bd0b0, in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005",
out=0x199bdd90 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x199bdd90 "", in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005",
len=32) at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x199bcec0, out=out@entry=0x199bdd90 "", outl=outl@entry=0x1f5c5ee0,
in=0x199bdc70 "\002-\346B\333\032\021\262\336g\274\365\036c|ٍ\275\346]I\262\020N\252\310\037r\034\335[v\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x199bce50,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000535f951 in ProtocolV2::handle_read_frame_preamble_main(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x19919880,
buffer=<optimized out>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:985
...
(gdb) cont
Continuing.
Breakpoint 2, gcm_ghash_avx () at ghash-x86_64.s:1365
1365 vzeroupper
(gdb) bt
#0 gcm_ghash_avx () at ghash-x86_64.s:1365
#1 0x00000000107bcaba in CRYPTO_gcm128_decrypt_ctr32 (ctx=ctx@entry=0x199bd0b0, in=0x1e9cec40 "i+\210\002F\214+\355\024ߏil\027zm\036\203K\221\275\265*\330\353\370\350\\1\253\235v\260\r\232\005",
out=0x1e9ced60 "", len=32, stream=0x107a21d0 <aesni_ctr32_encrypt_blocks>) at gcm128.c:1617
#2 0x0000000010826db5 in aes_gcm_cipher (ctx=<optimized out>, out=0x1e9ced60 "", in=0x1e9cec40 "i+\210\002F\214+\355\024ߏil\027zm\036\203K\221\275\265*\330\353\370\350\\1\253\235v\260\r\232\005", len=32)
at e_aes.c:2283
#3 0x0000000010822ba9 in EVP_DecryptUpdate (ctx=0x199bcec0, out=out@entry=0x1e9ced60 "", outl=outl@entry=0x1f5c5f10,
in=0x1e9cec40 "i+\210\002F\214+\355\024ߏil\027zm\036\203K\221\275\265*\330\353\370\350\\1\253\235v\260\r\232\005", inl=inl@entry=32) at evp_enc.c:500
#4 0x00000000053850ca in ceph::crypto::onwire::AES128GCM_OnWireRxHandler::authenticated_decrypt_update(ceph::buffer::v14_2_0::list&&, unsigned int) (this=0x199bce50,
ciphertext=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3e80543, DIE 0x3f0fb25>, alignment=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/crypto_onwire.cc:213
#5 0x000000000536078e in ProtocolV2::handle_read_frame_segment(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int) (this=0x19919880,
rx_buffer=<unknown type in /work/ceph-rzarzynski-3/build/lib/libceph-common.so.0, CU 0x3a5cdd1, DIE 0x3bc2653>, r=<optimized out>) at /work/ceph-rzarzynski-3/src/msg/async/ProtocolV2.cc:1135
...
(gdb) info registers rdi rsi rdx rcx
rdi 0x199bd0f0 429641968
rsi 0x199bd110 429642000
rdx 0x1e9cec40 513600576
rcx 0x20 32
(gdb) mo xb 0x199bd0f0 16
00 00 00 00 00 00 00 00
0x199BD0F0: 0x3b 0xc9 0x18 0x01 0x32 0xfa 0x25 0xe5
00 00 00 00 00 00 00 00
0x199BD0F8: 0xb6 0xc2 0xca 0x30 0x37 0x88 0x45 0xd8
(gdb) mo xb 0x199bd110 256
00 00 00 00 00 00 00 00
0x199BD110: 0x91 0x09 0x86 0x5b 0x8e 0x76 0x75 0x3e
00 00 00 00 00 00 00 00
0x199BD118: 0x6d 0xe2 0xbc 0xde 0x45 0x17 0xae 0xef
00 00 00 00 00 00 00 00
0x199BD120: 0x12 0x27 0x75 0x7c 0xcb 0x7f 0xc0 0xe4
00 00 00 00 00 00 00 00
0x199BD128: 0xfa 0x48 0xe4 0xf9 0xa3 0xf6 0xf0 0xc0
00 00 00 00 00 00 00 00
0x199BD130: 0xfc 0xeb 0x3a 0x85 0xcb 0x61 0xdb 0xd1
00 00 00 00 00 00 00 00
0x199BD138: 0xe8 0x6f 0x91 0x85 0x68 0x89 0x30 0x24
00 00 00 00 00 00 00 00
0x199BD140: 0xa3 0x98 0x19 0xb5 0x13 0x55 0x99 0xd8
00 00 00 00 00 00 00 00
0x199BD148: 0x78 0xb2 0x1e 0xa8 0x1a 0x38 0xb8 0xa7
00 00 00 00 00 00 00 00
0x199BD150: 0x81 0xbe 0x52 0xf5 0xe7 0xee 0xc7 0x05
00 00 00 00 00 00 00 00
0x199BD158: 0xd3 0x42 0x94 0x4f 0xa0 0x19 0xa4 0x35
00 00 00 00 00 00 00 00
0x199BD160: 0xdb 0x2a 0x07 0x1d 0x09 0x6d 0x21 0x7f
00 00 00 00 00 00 00 00
0x199BD168: 0x52 0xfc 0xc6 0xba 0x47 0xf7 0x63 0x30
00 00 00 00 00 00 00 00
0x199BD170: 0x0d 0xb2 0xca 0xdc 0x5c 0x97 0x87 0x78
00 00 00 00 00 00 00 00
0x199BD178: 0x4d 0x22 0x13 0xce 0xbb 0xf8 0x2b 0x8b
00 00 00 00 00 00 00 00
0x199BD180: 0xce 0xe2 0x36 0x76 0xe7 0xe2 0x17 0xf8
00 00 00 00 00 00 00 00
0x199BD188: 0x80 0xe2 0xbf 0x5a 0x73 0xb4 0x13 0xb5
00 00 00 00 00 00 00 00
0x199BD190: 0x40 0x90 0xd9 0x12 0xe7 0x6f 0xac 0xf3
00 00 00 00 00 00 00 00
0x199BD198: 0x4e 0x00 0x89 0x2c 0x94 0x56 0x04 0x4d
00 00 00 00 00 00 00 00
0x199BD1A0: 0xe4 0x51 0x7f 0xeb 0x9c 0xff 0xaa 0x02
00 00 00 00 00 00 00 00
0x199BD1A8: 0x26 0x0b 0xec 0x6c 0x0e 0x6f 0xe7 0xf9
00 00 00 00 00 00 00 00
0x199BD1B0: 0x84 0x89 0x25 0x6e 0xf5 0xbe 0x22 0x90
00 00 00 00 00 00 00 00
0x199BD1B8: 0xb2 0x87 0x82 0x80 0xaa 0xca 0x08 0x97
00 00 00 00 00 00 00 00
0x199BD1C0: 0x36 0x0e 0xa7 0xee 0x5f 0x74 0x2a 0x07
00 00 00 00 00 00 00 00
0x199BD1C8: 0xc2 0x5a 0x93 0x87 0x92 0x90 0x4d 0xfb
00 00 00 00 00 00 00 00
0x199BD1D0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1D8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1E0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1E8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1F0: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD1F8: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD200: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
00 00 00 00 00 00 00 00
0x199BD208: 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00
(gdb) info registers rdi rsi rdx rcx
rdi 0x199bd0f0 429641968
rsi 0x199bd110 429642000
rdx 0x1e9cec40 513600576
rcx 0x20 32
(gdb) mo xb 0x1e9cec40 32
00 00 00 00 00 00 00 00
0x1E9CEC40: 0x69 0x2b 0x88 0x02 0x46 0x8c 0x2b 0xed
00 00 00 00 00 00 00 00
0x1E9CEC48: 0x14 0xdf 0x8f 0x69 0x6c 0x17 0x7a 0x6d
00 00 00 00 00 00 00 00
0x1E9CEC50: 0x1e 0x83 0x4b 0x91 0xbd 0xb5 0x2a 0xd8
00 00 00 00 00 00 00 00
0x1E9CEC58: 0xeb 0xf8 0xe8 0x5c 0x31 0xab 0x9d 0x76
(gdb) cont
Continuing.
Breakpoint 1, gcm_ghash_avx () at ghash-x86_64.s:1734
1734 vzeroupper
(gdb) mo xb 0x199bd0f0 16
ff ff ff ff ff ff ff ff
0x199BD0F0: 0x4c 0xd7 0xfd 0x49 0xa1 0xfe 0xa4 0xea
ff ff ff ff ff ff ff ff
0x199BD0F8: 0xfa 0x88 0x28 0x95 0x40 0x22 0xbc 0xbb