crash.txt

    891 BUG: unable to handle kernel NULL pointer dereference at (null)
    892 IP: [<ffffffffa024c966>] _stp_umodule_relocate.clone.7+0x86/0x130 [orxray_lj_lua_fgraph]
    893 Kernel PGD 17154c067 PUD 175280067 PMD 0
    894 User   PGD 17154c067 PUD 175280067 PMD 0
    895 Oops: 0000 [#1] SMP
    896 last sysfs file: /sys/module/xt_state/sections/__mcount_loc
    897 CPU 6
    898 Modules linked in: orxray_lj_lua_fgraph(U) uprobes(U) ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 iptable_filter ip_tables ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables ipv6 joydev virtio_balloon i2c_piix4 i2c_core virtio_console sg  ]
    899
    900 Pid: 3025, comm: nginx Not tainted 2.6.32-754.24.3.el6.x86_64 #1 QEMU Standard PC (i440FX + PIIX, 1996)
    901 RIP: 0010:[<ffffffffa024c966>]  [<ffffffffa024c966>] _stp_umodule_relocate.clone.7+0x86/0x130 [orxray_lj_lua_fgraph]
    902 RSP: 0018:ffff88002cd83c08  EFLAGS: 00010083
    903 RAX: ffffffffa02d9300 RBX: ffffffffa02d8b58 RCX: 0000000000000036
    904 RDX: 0000000000000000 RSI: ffffffffa02b2e78 RDI: 0000000000000000
    905 RBP: ffff88002cd83c78 R08: ffff880173c89520 R09: ffffffffa027626d
    906 hrtimer: interrupt took 3031592 ns
    907 R10: 000000000000000a R11: ffffffffa0276277 R12: 000000000000000f
    908 R13: 0000000000000036 R14: ffffffffa02b2e78 R15: 0000000000000011
    909 FS:  00007fa8e4ee8720(0000) GS:ffff88002cd80000(0000) knlGS:0000000000000000
    910 CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
    911 CR2: 0000000000000000 CR3: 000000017210a000 CR4: 00000000001406e0
    912 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
    913 DR3: 0000000000000000 DR6: 00000000ffff4ff0 DR7: 0000000000000400
    914 Process nginx (pid: 3025, threadinfo ffff880173fe0000, task ffff880173c89520)
    915 Stack:
    916  ffff88002cd83c90 ffff880173c89520 ffffffffa0276277 000000000000000a
    917 <d> ffffffffa027626d 0000000000081649 0000000000000001 0000000000000000
    918 <d> ffff88002cd83c78 ffffc9000e324000 ffffc9000e342180 0000000000000000
    919 Call Trace:
    920  <IRQ>
    921  [<ffffffffa026c978>] function___global_func_lj_debug_dumpstack__overload_0+0x12d8/0x4580 [orxray_lj_lua_fgraph]
    922  [<ffffffffa0270735>] function___global_func_lj_dump_bt__overload_0+0xb15/0x1d40 [orxray_lj_lua_fgraph]
    923  [<ffffffffa0271e1d>] function___global_func_probe_handler__overload_0+0x4bd/0xcb0 [orxray_lj_lua_fgraph]
    924  [<ffffffff810bd3d0>] ? tick_sched_timer+0x0/0xc0
    925  [<ffffffffa0272938>] probe_64+0x328/0x8f0 [orxray_lj_lua_fgraph]
    926  [<ffffffffa0262efe>] enter_profile_probes+0x3ae/0x490 [orxray_lj_lua_fgraph]
    927  [<ffffffff810b563c>] profile_tick+0x9c/0xa0
    928  [<ffffffff810bd440>] tick_sched_timer+0x70/0xc0
    929  [<ffffffff810af9d0>] __run_hrtimer+0x90/0x1e0
    930  [<ffffffff810afd7e>] hrtimer_interrupt+0xee/0x270
    931  [<ffffffff81039d43>] local_apic_timer_interrupt+0x43/0x70
    932  [<ffffffff81566859>] smp_apic_timer_interrupt+0x49/0x60
    933  [<ffffffff81565193>] apic_timer_interrupt+0x13/0x20
    934  <EOI>
    935  [<ffffffff810e5444>] ? start_callback+0x34/0xe0
    936  [<ffffffff810e6e07>] utrace_report_syscall_exit+0x87/0xf0
    937  [<ffffffff810b048d>] ? hrtimer_try_to_cancel+0x3d/0xd0
    938  [<ffffffff81019303>] tracehook_report_syscall_exit+0xc3/0x110
    939  [<ffffffff810b0542>] ? hrtimer_cancel+0x22/0x30
    940  [<ffffffff8155bde3>] ? do_nanosleep+0x93/0xc0
    941  [<ffffffff810f2ffe>] ? __audit_syscall_exit+0x25e/0x290
    942  [<ffffffff810af400>] ? hrtimer_wakeup+0x0/0x30
    943  [<ffffffff810193c1>] syscall_trace_leave+0x71/0xe0
    944  [<ffffffff81564714>] int_check_syscall_exit_work+0x13/0x1c
    945 Code: c3 77 62 27 a0 eb 17 66 0f 1f 44 00 00 41 83 c4 01 48 83 c3 08 45 39 fc 0f 83 9f 00 00 00 48 8b 03 4c 89 f6 4c 89 e9 48 8b 78 08 <f3> a6 75 de 83 78 18 01 75 d8 48 8b 50 10 4c 89 cf 4c 89 d1 48
    946 RIP  [<ffffffffa024c966>] _stp_umodule_relocate.clone.7+0x86/0x130 [orxray_lj_lua_fgraph]
    947  RSP <ffff88002cd83c08>
    948 CR2: 0000000000000000
    949 ---[ end trace cf258728ab4551e4 ]---
    950 Kernel panic - not syncing: Fatal exception in interrupt
    951 Pid: 3025, comm: nginx Tainted: G      D    -- ------------    2.6.32-754.24.3.el6.x86_64 #1
    952 Call Trace:
    953  <IRQ>  [<ffffffff81558eba>] ? panic+0xa7/0x18b
    954  [<ffffffff8155e342>] ? oops_end+0xf2/0x100
    955  [<ffffffff8105471b>] ? no_context+0xfb/0x260
    956  [<ffffffff810b2555>] ? sched_clock_local+0x25/0x90
    957  [<ffffffff810549a5>] ? __bad_area_nosemaphore+0x125/0x1e0
    958  [<ffffffff81054a73>] ? bad_area_nosemaphore+0x13/0x20
    959  [<ffffffff8105518c>] ? __do_page_fault+0x30c/0x4d0
    960  [<ffffffff810b7382>] ? ktime_get+0x72/0x100
    961  [<ffffffff8103fd32>] ? native_apic_msr_write+0x32/0x40
    962  [<ffffffff810395b3>] ? lapic_next_event+0x23/0x30
    963  [<ffffffff810bbbe2>] ? clockevents_program_event+0x72/0x100
    964  [<ffffffff810bcf9f>] ? tick_program_event+0x2f/0x40
    965  [<ffffffff810afde3>] ? hrtimer_interrupt+0x153/0x270
    966  [<ffffffff815602ce>] ? do_page_fault+0x3e/0xa0
    967  [<ffffffff8155d265>] ? page_fault+0x25/0x30
    968  [<ffffffffa024c966>] ? _stp_umodule_relocate.clone.7+0x86/0x130 [orxray_lj_lua_fgraph]
    969  [<ffffffffa024c9b8>] ? _stp_umodule_relocate.clone.7+0xd8/0x130 [orxray_lj_lua_fgraph]
    970  [<ffffffffa026c978>] ? function___global_func_lj_debug_dumpstack__overload_0+0x12d8/0x4580 [orxray_lj_lua_fgraph]
    971  [<ffffffffa0270735>] ? function___global_func_lj_dump_bt__overload_0+0xb15/0x1d40 [orxray_lj_lua_fgraph]
    972  [<ffffffffa0271e1d>] ? function___global_func_probe_handler__overload_0+0x4bd/0xcb0 [orxray_lj_lua_fgraph]
    973  [<ffffffff810bd3d0>] ? tick_sched_timer+0x0/0xc0
    974  [<ffffffffa0272938>] ? probe_64+0x328/0x8f0 [orxray_lj_lua_fgraph]
    975  [<ffffffffa0262efe>] ? enter_profile_probes+0x3ae/0x490 [orxray_lj_lua_fgraph]
    976  [<ffffffff810b563c>] ? profile_tick+0x9c/0xa0
    977  [<ffffffff810bd440>] ? tick_sched_timer+0x70/0xc0
    978  [<ffffffff810af9d0>] ? __run_hrtimer+0x90/0x1e0
    979  [<ffffffff810afd7e>] ? hrtimer_interrupt+0xee/0x270
    980  [<ffffffff81039d43>] ? local_apic_timer_interrupt+0x43/0x70
    981  [<ffffffff81566859>] ? smp_apic_timer_interrupt+0x49/0x60
    982  [<ffffffff81565193>] ? apic_timer_interrupt+0x13/0x20
    983  <EOI>  [<ffffffff810e5444>] ? start_callback+0x34/0xe0
    984  [<ffffffff810e6e07>] ? utrace_report_syscall_exit+0x87/0xf0
    985  [<ffffffff810b048d>] ? hrtimer_try_to_cancel+0x3d/0xd0
    986  [<ffffffff81019303>] ? tracehook_report_syscall_exit+0xc3/0x110
    987  [<ffffffff810b0542>] ? hrtimer_cancel+0x22/0x30
    988  [<ffffffff8155bde3>] ? do_nanosleep+0x93/0xc0
    989  [<ffffffff810f2ffe>] ? __audit_syscall_exit+0x25e/0x290
    990  [<ffffffff810af400>] ? hrtimer_wakeup+0x0/0x30
    991  [<ffffffff810193c1>] ? syscall_trace_leave+0x71/0xe0
    992  [<ffffffff81564714>] ? int_check_syscall_exit_work+0x13/0x1c

crash> info line *(_stp_umodule_relocate+0x86)
      1 Line 83 of "/usr/local/openresty-stap/share/systemtap/runtime/sym.c" starts at address 0xffffffffa024c966 <_stp_umodule_relocate+134> and ends at 0xffffffffa024c970 <_stp_umodule_relocate+144>.