Skip to content

Instantly share code, notes, and snippets.

@rosstimson
Last active August 29, 2015 13:57
Show Gist options
  • Save rosstimson/9618197 to your computer and use it in GitHub Desktop.
Save rosstimson/9618197 to your computer and use it in GitHub Desktop.
Nginx / OpenResty segfault - ngx_http_lua_socket_tcp_finalize
[root@my-server ~]# gdb /usr/local/nginx/sbin/nginx /tmp/core_dumps/99_27290_nginx
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6_4.1)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/nginx/sbin/nginx...done.
[New Thread 27290]
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /usr/local/luajit/lib/libluajit-5.1.so.2...done.
Loaded symbols for /usr/local/luajit/lib/libluajit-5.1.so.2
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /usr/lib64/libssl.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.10
Reading symbols from /usr/lib64/libcrypto.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.10
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libfreebl3.so
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /usr/local/lualib/cjson.so...done.
Loaded symbols for /usr/local/lualib/cjson.so
Core was generated by `nginx: worker process '.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000515878 in ngx_http_lua_socket_tcp_finalize (r=0xddba80, u=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2461
2461 ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c: No such file or directory.
in ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c
Missing separate debuginfos, use: debuginfo-install nginx-1.5.8.1-1.x86_64
(gdb) backtrace full
#0 0x0000000000515878 in ngx_http_lua_socket_tcp_finalize (r=0xddba80, u=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2461
spool = 0x7fff7a50fd20
cl = 0xe08f58
ll = 0x41222008
ctx = 0xddcbb0
#1 0x0000000000515794 in ngx_http_lua_socket_tcp_cleanup (data=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2431
u = 0x41221f40
r = 0xddba80
#2 0x000000000046c4f1 in ngx_http_free_request (r=0xddba80, rc=0) at src/http/ngx_http_request.c:3395
log = 0xdfabc0
pool = 0x7fff7a50fcd0
linger = {l_onoff = 2052123900, l_linger = 32767}
cln = 0xe00068
ctx = 0x0
clcf = 0x7fff7a50ed13
#3 0x000000000046b208 in ngx_http_set_keepalive (r=0xddba80) at src/http/ngx_http_request.c:2857
tcp_nodelay = 32767
i = 4789529
b = 0xe62ae8
f = 0x7fff7a50fd78
rev = 0xe21b90
wev = 0xddba80
c = 0x7fcedc45e4c0
hc = 0xdfac08
cscf = 0x0
clcf = 0xe14840
#4 0x000000000046a69b in ngx_http_finalize_connection (r=0xddba80) at src/http/ngx_http_request.c:2500
clcf = 0xe14840
#5 0x000000000046a303 in ngx_http_finalize_request (r=0xddba80, rc=0) at src/http/ngx_http_request.c:2401
c = 0x7fcedc45e4c0
pr = 0x1
clcf = 0x0
#6 0x0000000000488421 in ngx_http_upstream_finalize_request (r=0xddba80, u=0xdddaf0, rc=0) at src/http/ngx_http_upstream.c:3563
flush = 0
tp = 0x7b5300
#7 0x000000000048753c in ngx_http_upstream_process_request (r=0xddba80) at src/http/ngx_http_upstream.c:3164
tf = 0xdddfa0
p = 0xdddfa0
u = 0xdddaf0
#8 0x000000000048725a in ngx_http_upstream_process_upstream (r=0xddba80, u=0xdddaf0) at src/http/ngx_http_upstream.c:3095
c = 0x7fcedc45e7e0
#9 0x000000000048257f in ngx_http_upstream_handler (ev=0xe21d30) at src/http/ngx_http_upstream.c:977
c = 0x7fcedc45e4c0
r = 0xddba80
ctx = 0xe62ad0
u = 0xdddaf0
#10 0x000000000044bd07 in ngx_epoll_process_events (cycle=0xddfa90, timer=4660, flags=1) at src/event/modules/ngx_epoll_module.c:691
events = 1
revents = 5
instance = 0
i = 0
level = 14580648
err = 0
rev = 0xe21d30
wev = 0xe3bd40
queue = 0x7b70e0
c = 0x7fcedc45e7e0
#11 0x000000000043c448 in ngx_process_events_and_timers (cycle=0xddfa90) at src/event/ngx_event.c:248
flags = 1
timer = 4660
delta = 1395137731116
#12 0x0000000000449c3a in ngx_worker_process_cycle (cycle=0xddfa90, data=0x0) at src/os/unix/ngx_process_cycle.c:816
worker = 0
i = 0
c = 0x417300
#13 0x0000000000446380 in ngx_spawn_process (cycle=0xddfa90, proc=0x449a34 <ngx_worker_process_cycle>, data=0x0, name=0x563d33 "worker process", respawn=1) at src/os/unix/ngx_process.c:198
on = 1
pid = 0
s = 1
#14 0x0000000000449620 in ngx_reap_children (cycle=0xddfa90) at src/os/unix/ngx_process_cycle.c:627
i = 1
n = 2
live = 0
ch = {command = 2, pid = 27276, slot = 1, fd = -1}
ccf = 0x7b688f
#15 0x0000000000448230 in ngx_master_process_cycle (cycle=0xddfa90) at src/os/unix/ngx_process_cycle.c:180
title = 0xdfc353 ""
p = 0xdfc396 ""
size = 68
i = 3
n = 0
sigio = 0
set = {__val = {0 <repeats 16 times>}}
itv = {it_interval = {tv_sec = 0, tv_usec = 1}, it_value = {tv_sec = 18, tv_usec = 18}}
live = 1
delay = 0
ls = 0x0
ccf = 0xde0050
#16 0x000000000041797b in main (argc=3, argv=0x7fff7a510588) at src/core/nginx.c:407
i = 65
log = 0x7b4ee0
cycle = 0xddba80
init_cycle = {conf_ctx = 0x0, pool = 0xddb530, log = 0x7b4ee0, new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data = 0x0, action = 0x0, next = 0x0}, log_use_stderr = 0, files = 0x0,
free_connections = 0x0, free_connection_n = 0, reusable_connections_queue = {prev = 0x0, next = 0x0}, listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, paths = {elts = 0x0, nelts = 0, size = 0,
nalloc = 0, pool = 0x0}, open_files = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0,
nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0, old_cycle = 0x0, conf_file = {len = 21, data = 0x7fff7a511f5c ""}, conf_param = {len = 0, data = 0x0},
conf_prefix = {len = 11, data = 0x7fff7a511f5c ""}, prefix = {len = 17, data = 0x560006 "/usr/local/nginx/"}, lock_file = {len = 0, data = 0x0}, hostname = {len = 0, data = 0x0}}
ccf = 0xddca88
local memcached = require "resty.memcached"
local config = require 'config'
local cache = {}
local memc, err = memcached:new()
if not memc then
ngx.log(ngx.ERR, "failed to instantiate memc: ", err)
return ngx.exit(500)
end
memc:set_timeout(1000)
local function memcache_connect()
local ok, err = memc:connect("127.0.0.1", 11211)
if not ok then
ngx.log(ngx.ERR, "failed to connect to memcached: ", err)
return ngx.exit(500)
end
end
function cache.get_user_details(token)
memcache_connect()
value, flags, err = memc:get(token)
if value then
local m, err = ngx.re.match(value, '^(\\S+) (\\S+) (\\S+)$')
return m[1], m[2], m[3]
else
return false
end
end
function cache.add_user_details(token, user, business, app)
memcache_connect()
memc:set(token, user..' '..business..' '..app, config.cache_timeout)
end
function cache.delete(token)
memcache_connect()
memc:delete(token)
end
return cache
get_user_details(): RETRIEVED FROM APP: f227be2fc5ff50d0f152cb353b015c69, bb27f4a451b9bd216b38c7fba64c5ce7
2014/03/11 11:32:05 [notice] 5045#0: signal 17 (SIGCHLD) received
2014/03/11 11:32:05 [notice] 5045#0: signal 17 (SIGCHLD) received
2014/03/11 11:32:05 [alert] 5045#0: worker process 26693 exited on signal 11
2014/03/11 11:32:05 [debug] 5045#0: shmtx forced unlock
2014/03/11 11:32:05 [debug] 5045#0: channel 3:7
2014/03/11 11:32:05 [notice] 5045#0: start worker process 27401
2014/03/11 11:32:05 [debug] 27401#0: setproctitle: "nginx: worker process"
2014/03/11 11:32:05 [debug] 27401#0: *37617 reusable connection: 1
2014/03/11 11:32:05 [debug] 27401#0: *37617 reusable connection: 0
2014/03/11 11:32:05 [debug] 27401#0: resolve: "app.example.com"
2014/03/11 11:32:05 [debug] 27401#0: resolve: "app.example.com" 61608
2014/03/11 11:32:05 [debug] 27401#0: resolver DNS response 61608 fl:8180 1/3/0/0
2014/03/11 11:32:05 [debug] 27401#0: resolver DNS response qt:1 cl:1
2014/03/11 11:32:05 [debug] 27401#0: resolver qs:app.example.com
2014/03/11 11:32:05 [debug] 27401#0: resolver naddrs:2 cname:00007FFFFA73849D ttl:59
2014/03/11 11:32:05 [debug] 27401#0: resolve name done: 0
2014/03/11 11:32:05 [debug] 27401#0: resolver expire
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer buf fl:1 s:222
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer in: 0000000000CDECE8
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer out: 0000000000000000
2014/03/11 11:32:06 [debug] 27401#0: *37617 reusable connection: 0
2014/03/11 11:32:06 [debug] 27401#0: *37617 [lua] auth.lua:82:
worker_processes 1;
error_log /var/log/nginx-error.log debug_core;
pid /var/run/nginx.pid;
worker_rlimit_core 500M;
working_directory /tmp/core_dumps/;
events {
worker_connections 1024;
}
http {
lua_code_cache on;
lua_package_path '/etc/nginx/lualib/?.lua;;';
server {
listen 80;
server_name app.example.com
pre-app.example.biz;
access_log /var/log/nginx-acces.log combined;
location /_auth_check {
resolver 8.8.8.8;
access_by_lua_file "/etc/nginx/lualib/main.lua";
proxy_pass_request_headers off;
proxy_set_header Accept "application/json";
proxy_set_header User-Agent "API Proxy";
proxy_set_header X-API-PROXY-AUTH-DIGEST $secret_header_value;
proxy_pass $app_location;
}
location /api/ {
resolver 8.8.8.8;
set $proxy_destination '';
set $app_location '';
set $secret_header_value '';
set $user_guid_header_value '';
set $business_guid_header_value '';
access_by_lua_file "/etc/nginx/lualib/main.lua";
proxy_set_header X-API-PROXY-AUTH-DIGEST $secret_header_value;
proxy_set_header X-API-PROXY-USER-GUID $user_guid_header_value;
proxy_set_header X-API-PROXY-BUSINESS-GUID $business_guid_header_value;
proxy_pass $proxy_destination;
}
location / {
deny all;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment