LibAV crash, OSX + 10bit h264: put_hv4_10_mmxext.v_loop

zzz_backtrace

VO: [gl] 1920x1080 => 1920x1080 Planar 420P 10-bit little-endian 
Reading symbols for shared libraries . done
A:  31.3 V:  31.2 A-V:  0.014 ct:  0.000   0/  0 14%  2%  0.8% 36 0 
Program received signal EXC_BAD_ACCESS, Could not access memory.
Reason: KERN_INVALID_ADDRESS at address: 0x00000001052f5ffc
[Switching to process 33890 thread 0x6107]
0x000000010079a1df in put_hv4_10_mmxext.v_loop ()
(gdb) bt
#0  0x000000010079a1df in put_hv4_10_mmxext.v_loop ()
#1  0x000000010079ac22 in stub_put_h264_qpel4_mc21_10_mmxext.body ()
#2  0x0000000100560429 in mc_part ()
#3  0x000000010055fb3a in hl_motion ()
#4  0x000000010055d05e in hl_decode_mb_internal ()
#5  0x000000010055af08 in decode_slice ()
#6  0x000000010055adfe in execute_decode_slices ()
#7  0x00000001005565f3 in decode_nal_units ()
#8  0x0000000100557666 in decode_frame ()
#9  0x000000010064aa88 in frame_worker_thread ()
#10 0x00007fff910bd8bf in _pthread_start ()
#11 0x00007fff910c0b75 in thread_start ()
(gdb) disass $pc-32 $pc+32    
Dump of assembler code from 0x10079a1bf to 0x10079a1ff:
0x000000010079a1bf <ff_avg_h264_qpel16_mc33_10_sse2+47>:	nop    
0x000000010079a1c0 <stub_avg_h264_qpel8_mc33_10_sse2+0>:	lea    (%rsi,%rdx,1),%r8
0x000000010079a1c4 <stub_avg_h264_qpel8_mc33_10_sse2+4>:	add    $0x2,%rsi
0x000000010079a1c8 <stub_avg_h264_qpel8_mc33_10_sse2+8>:	jmpq   0x100799f13 <stub_avg_h264_qpel8_mc11_10_sse2.body>
0x000000010079a1cd <put_hv4_10_mmxext+0>:	neg    %rdx
0x000000010079a1d0 <put_hv4_10_mmxext+3>:	lea    -0x8(%rsi,%rdx,2),%rsi
0x000000010079a1d5 <put_hv4_10_mmxext+8>:	lea    0x8(%rsp),%r8
0x000000010079a1da <put_hv4_10_mmxext+13>:	mov    $0x3,%ecx
0x000000010079a1df <put_hv4_10_mmxext.v_loop+0>:	movq   (%rsi),%mm0
0x000000010079a1e2 <put_hv4_10_mmxext.v_loop+3>:	sub    %rdx,%rsi
0x000000010079a1e5 <put_hv4_10_mmxext.v_loop+6>:	movq   (%rsi),%mm1
0x000000010079a1e8 <put_hv4_10_mmxext.v_loop+9>:	sub    %rdx,%rsi
0x000000010079a1eb <put_hv4_10_mmxext.v_loop+12>:	movq   (%rsi),%mm2
0x000000010079a1ee <put_hv4_10_mmxext.v_loop+15>:	sub    %rdx,%rsi
0x000000010079a1f1 <put_hv4_10_mmxext.v_loop+18>:	movq   (%rsi),%mm3
0x000000010079a1f4 <put_hv4_10_mmxext.v_loop+21>:	sub    %rdx,%rsi
0x000000010079a1f7 <put_hv4_10_mmxext.v_loop+24>:	movq   (%rsi),%mm4
0x000000010079a1fa <put_hv4_10_mmxext.v_loop+27>:	sub    %rdx,%rsi
0x000000010079a1fd <put_hv4_10_mmxext.v_loop+30>:	movq   (%rsi),%mm5
End of assembler dump.
(gdb) info all-registers
rax            0x104d57798	4376065944
rbx            0x108d60818	4443211800
rcx            0x3	3
rdx            0xfffffffffffff0c0	-3904
rsi            0x1052f5ffc	4381958140
rdi            0x108d60818	4443211800
rbp            0x104d57830	0x104d57830
rsp            0x104d57490	0x104d57490
r8             0x104d57498	4376065176
r9             0x1052fbb84	4381981572
r10            0x15	21
r11            0x14040	81984
r12            0x1	1
r13            0x3793fe	3642366
r14            0x1052f7e84	4381965956
r15            0x104c87000	4375212032
rip            0x10079a1df	0x10079a1df <put_hv4_10_mmxext.v_loop>
eflags         0x10287	66183
cs             0x2b	43
ss             0x0	0
ds             0x0	0
es             0x0	0
fs             0x0	0
gs             0x0	0
st0            -nan(0xf7000000f70)	(raw 0xffff00000f7000000f70)
st1            -nan(0x001ef01ef)	(raw 0xffff0000000001ef01ef)
st2            -nan(0xf094f094f096f080)	(raw 0xfffff094f094f096f080)
st3            -nan(0x000000008)	(raw 0xffff0000000000000008)
st4            -nan(0x4080408040804080)	(raw 0xffff4080408040804080)
st5            -nan(0x3800000038)	(raw 0xffff0000003800000038)
st6            -nan(0x4fec4fec4fec4fec)	(raw 0xffff4fec4fec4fec4fec)
st7            <invalid float value>	(raw 0xffff0000000000000000)
fctrl          0x37f	895
fstat          0x0	0
ftag           0xaaaa	43690
fiseg          0x0	0
fioff          0x0	0
foseg          0x0	0
fooff          0x0	0
fop            0x0	0
xmm0           {
  v4_float = {9.18368975e-41, 9.18368975e-41, 9.18368975e-41, 9.18368975e-41}, 
  v2_double = {1.3906923818487063e-309, 1.3906923818487063e-309}, 
  v16_int8 = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}, 
  v8_int16 = {1, 1, 1, 1, 1, 1, 1, 1}, 
  v4_int32 = {65537, 65537, 65537, 65537}, 
  v2_int64 = {281479271743489, 281479271743489}, 
  uint128 = 0x00010001000100010001000100010001
}	(raw 0x01000100010001000100010001000100)
xmm1           {
  v4_float = {8.81648565e-38, 8.74301613e-38, 8.77975257e-38, 8.88995292e-38}, 
  v2_double = {2.3902850044563225e-299, 2.3150611229771886e-299}, 
  v16_int8 = {1, -16, 1, -16, 1, -18, 1, -18, 1, -17, 1, -14, 1, -14, 1, -18}, 
  v8_int16 = {496, 496, 494, 494, 495, 498, 498, 494}, 
  v4_int32 = {32506352, 32375278, 32440818, 32637422}, 
  v2_int64 = {139613718784639470, 139332252398125550}, 
  uint128 = 0x01f001f001ee01ee01ef01f201f201ee
}	(raw 0xee01f201f201ef01ee01ee01f001f001)
xmm2           {
  v4_float = {0, 0, 0, 1.40129846e-45}, 
  v2_double = {0, 4.9406564584124654e-324}, 
  v16_int8 = {0 <repeats 15 times>, 1}, 
  v8_int16 = {0, 0, 0, 0, 0, 0, 0, 1}, 
  v4_int32 = {0, 0, 0, 1}, 
  v2_int64 = {0, 1}, 
  uint128 = 72057594037927936
}	(raw 0x01000000000000000000000000000000)
xmm3           {
  v4_float = {9.18382988e-41, 9.18382988e-41, 9.18382988e-41, 9.18382988e-41}, 
  v2_double = {1.3907136018066209e-309, 1.3907136018066209e-309}, 
  v16_int8 = {0, 1, 0, 2, 0, 1, 0, 2, 0, 1, 0, 2, 0, 1, 0, 2}, 
  v8_int16 = {1, 2, 1, 2, 1, 2, 1, 2}, 
  v4_int32 = {65538, 65538, 65538, 65538}, 
  v2_int64 = {281483566710786, 281483566710786}, 
  uint128 = 0x00010002000100020001000200010002
}	(raw 0x02000100020001000200010002000100)
xmm4           {
  v4_float = {1.49884704e-36, 1.49884704e-36, 1.49884704e-36, 1.49884704e-36}, 
  v2_double = {1.9891361339869654e-289, 1.9891361339869654e-289}, 
  v16_int8 = {3, -1, 3, -1, 3, -1, 3, -1, 3, -1, 3, -1, 3, -1, 3, -1}, 
  v8_int16 = {1023, 1023, 1023, 1023, 1023, 1023, 1023, 1023}, 
  v4_int32 = {67044351, 67044351, 67044351, 67044351}, 
  v2_int64 = {287953294993589247, 287953294993589247}, 
  uint128 = 0x03ff03ff03ff03ff03ff03ff03ff03ff
}	(raw 0xff03ff03ff03ff03ff03ff03ff03ff03)
xmm5           {
  v4_float = {9.69840797e-38, 9.62493285e-38, 9.29404088e-38, 9.55146781e-38}, 
  v2_double = {5.9752431380334946e-299, 4.3315166226753551e-299}, 
  v16_int8 = {2, 4, 2, 5, 2, 3, 1, -1, 1, -3, 2, 3, 2, 2, 2, 2}, 
  v8_int16 = {516, 517, 515, 511, 509, 515, 514, 514}, 
  v4_int32 = {33817093, 33751551, 33358339, 33686018}, 
  v2_int64 = {145243308514542079, 143272975087567362}, 
  uint128 = 0x02040205020301ff01fd020302020202
}	(raw 0x020202020302fd01ff01030205020402)
xmm6           {
  v4_float = {7.23070008e-43, 7.24471306e-43, 7.21668709e-43, 7.16063515e-43}, 
  v2_double = {1.0949498283935125e-311, 1.0928278325995828e-311}, 
  v16_int8 = {0, 0, 2, 4, 0, 0, 2, 5, 0, 0, 2, 3, 0, 0, 1, -1}, 
  v8_int16 = {0, 516, 0, 517, 0, 515, 0, 511}, 
  v4_int32 = {516, 517, 515, 511}, 
  v2_int64 = {2216203125253, 2211908157951}, 
  uint128 = 0x000002040000020500000203000001ff
}	(raw 0xff010000030200000502000004020000)
xmm7           {
  v4_float = {0, 0, 0, 0}, 
  v2_double = {0, 0}, 
  v16_int8 = {0 <repeats 16 times>}, 
  v8_int16 = {0, 0, 0, 0, 0, 0, 0, 0}, 
  v4_int32 = {0, 0, 0, 0}, 
  v2_int64 = {0, 0}, 
  uint128 = 0
}	(raw 0x00000000000000000000000000000000)
xmm8           {
  v4_float = {9.47799381e-38, 9.36750815e-38, 9.40453326e-38, 9.99227707e-38}, 
  v2_double = {5.0792736083821746e-299, 4.7806611497457821e-299}, 
  v16_int8 = {2, 1, 1, -3, 1, -1, 2, 1, 2, 0, 2, 4, 2, 8, 2, 1}, 
  v8_int16 = {513, 509, 511, 513, 512, 516, 520, 513}, 
  v4_int32 = {33620477, 33489409, 33554948, 34079233}, 
  v2_int64 = {144398849224409601, 144117404313059841}, 
  uint128 = 0x020101fd01ff02010200020402080201
}	(raw 0x01020802040200020102ff01fd010102)
xmm9           {
  v4_float = {-nan(0x7ffffb), -nan(0x7ffffd), 4.20389539e-45, 1.40129846e-45}, 
  v2_double = {-nan(0xffffbfffffffd), 6.3659873733898826e-314}, 
  v16_int8 = {-1, -1, -1, -5, -1, -1, -1, -3, 0, 0, 0, 3, 0, 0, 0, 1}, 
  v8_int16 = {-1, -5, -1, -3, 0, 3, 0, 1}, 
  v4_int32 = {-5, -3, 3, 1}, 
  v2_int64 = {-17179869187, 12884901889}, 
  uint128 = 0xfffffffbfffffffd0000000300000001
}	(raw 0x0100000003000000fdfffffffbffffff)
xmm10          {
  v4_float = {0, 0, 0, 0}, 
  v2_double = {0, 0}, 
  v16_int8 = {0 <repeats 16 times>}, 
  v8_int16 = {0, 0, 0, 0, 0, 0, 0, 0}, 
  v4_int32 = {0, 0, 0, 0}, 
  v2_int64 = {0, 0}, 
  uint128 = 0
}	(raw 0x00000000000000000000000000000000)
xmm11          {
  v4_float = {9.3307762e-38, 1.00657522e-37, 9.77188085e-38, 9.91881765e-38}, 
  v2_double = {4.4808433588305024e-299, 6.2739057240377183e-299}, 
  v16_int8 = {1, -2, 2, 5, 2, 9, 2, 7, 2, 5, 2, 9, 2, 7, 2, 9}, 
  v8_int16 = {510, 517, 521, 519, 517, 521, 519, 521}, 
  v4_int32 = {33423877, 34144775, 33882633, 34013705}, 
  v2_int64 = {143554458654671367, 145524800671384073}, 
  uint128 = 0x01fe0205020902070205020902070209
}	(raw 0x0902070209020502070209020502fe01)
xmm12          {
  v4_float = {0, -nan(0x7ffffc), -nan(0x7ffffc), -nan(0x7ffffa)}, 
  v2_double = {2.1219957889890097e-314, -nan(0xffffcfffffffa)}, 
  v16_int8 = {0, 0, 0, 0, -1, -1, -1, -4, -1, -1, -1, -4, -1, -1, -1, -6}, 
  v8_int16 = {0, 0, -1, -4, -1, -4, -1, -6}, 
  v4_int32 = {0, -4, -4, -6}, 
  v2_int64 = {4294967292, -12884901894}, 
  uint128 = 0x00000000fffffffcfffffffcfffffffa
}	(raw 0xfafffffffcfffffffcffffff00000000)
xmm13          {
  v4_float = {1.40129846e-45, -nan(0x7ffffe), -nan(0x7fffff), -nan(0x7ffffb)}, 
  v2_double = {4.2439915809424133e-314, -nan(0xffffffffffffb)}, 
  v16_int8 = {0, 0, 0, 1, -1, -1, -1, -2, -1, -1, -1, -1, -1, -1, -1, -5}, 
  v8_int16 = {0, 1, -1, -2, -1, -1, -1, -5}, 
  v4_int32 = {1, -2, -1, -5}, 
  v2_int64 = {8589934590, -5}, 
  uint128 = 0x00000001fffffffefffffffffffffffb
}	(raw 0xfbfffffffffffffffeffffff01000000)
xmm14          {
  v4_float = {9.77187076e-38, 9.36750815e-38, 9.69840797e-38, 9.84534477e-38}, 
  v2_double = {6.2738647109200814e-299, 5.9752431382421016e-299}, 
  v16_int8 = {2, 5, 2, 0, 1, -1, 2, 1, 2, 4, 2, 5, 2, 6, 2, 5}, 
  v8_int16 = {517, 512, 511, 513, 516, 517, 518, 517}, 
  v4_int32 = {33882624, 33489409, 33817093, 33948165}, 
  v2_int64 = {145524762016154113, 145243308514738693}, 
  uint128 = 0x0205020001ff02010204020502060205
}	(raw 0x05020602050204020102ff0100020502)
xmm15          {
  v4_float = {9.84534252e-38, 9.62492949e-38, 9.40453663e-38, 9.6249407e-38}, 
  v2_double = {6.572522739973027e-299, 4.7806748202519063e-299}, 
  v16_int8 = {2, 6, 2, 3, 2, 3, 1, -4, 2, 0, 2, 7, 2, 3, 2, 6}, 
  v8_int16 = {518, 515, 515, 508, 512, 519, 515, 518}, 
  v4_int32 = {33948163, 33751548, 33554951, 33751558}, 
  v2_int64 = {145806249878028796, 144117417197634054}, 
  uint128 = 0x02060203020301fc0200020702030206
}	(raw 0x0602030207020002fc01030203020602)
mxcsr          0x1f80	8064
(gdb) 

zzz_valgrind

==3634== Thread 6:A-V: 13.196 ct:  0.000   0/  0 123% 461% 328.5% 79 0 
==3634== Invalid read of size 8
==3634==    at 0x62A1DF: put_hv4_10_mmxext.v_loop (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3F05D4: mc_part (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EFB39: hl_motion (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3ED05D: hl_decode_mb_internal (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EAF07: decode_slice (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EADFD: execute_decode_slices (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E65F2: decode_nal_units (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E7665: decode_frame (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4DAA87: frame_worker_thread (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x32488BE: _pthread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x324BB74: thread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==  Address 0x101603dfc is 4 bytes before a block of size 166,656 alloc'd
==3634==    at 0xC160: malloc_zone_memalign (vg_replace_malloc.c:698)
==3634==    by 0x329AED9: posix_memalign (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x2CDDE1: av_mallocz (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavutil.51.23.1.dylib)
==3634==    by 0x4ABC59: init_duplicate_context (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4ABAB3: ff_MPV_common_init (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E8A27: decode_slice_header (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E619B: decode_nal_units (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E7665: decode_frame (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4DAA87: frame_worker_thread (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x32488BE: _pthread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x324BB74: thread_start (in /usr/lib/system/libsystem_c.dylib)
==3634== 
==3634== Invalid read of size 8t:  0.000   0/  0 157% 465% 328.2% 89 0 
==3634==    at 0x62A1DF: put_hv4_10_mmxext.v_loop (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3F05D4: mc_part (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EF5F4: hl_motion (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3ED05D: hl_decode_mb_internal (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EAF07: decode_slice (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3EADFD: execute_decode_slices (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E65F2: decode_nal_units (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E7665: decode_frame (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4DAA87: frame_worker_thread (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x32488BE: _pthread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x324BB74: thread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==  Address 0x101603dfc is 4 bytes before a block of size 166,656 alloc'd
==3634==    at 0xC160: malloc_zone_memalign (vg_replace_malloc.c:698)
==3634==    by 0x329AED9: posix_memalign (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x2CDDE1: av_mallocz (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavutil.51.23.1.dylib)
==3634==    by 0x4ABC59: init_duplicate_context (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4ABAB3: ff_MPV_common_init (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E8A27: decode_slice_header (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E619B: decode_nal_units (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x3E7665: decode_frame (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x4DAA87: frame_worker_thread (in /Users/pigoz/.homebrew/Cellar/libav/HEAD/lib/libavcodec.54.2.0.dylib)
==3634==    by 0x32488BE: _pthread_start (in /usr/lib/system/libsystem_c.dylib)
==3634==    by 0x324BB74: thread_start (in /usr/lib/system/libsystem_c.dylib)