RedisGearsDump

gistfile1.txt

10:M 30 Apr 2021 21:27:20.002 * <rg> GEARS: QA cached called {'event': 'keymiss', 'key': 'cache_{06S}_{PMC261870.xml:{06S}:26}_Laser Correction', 'type': 'empty', 'value': None}


=== REDIS BUG REPORT START: Cut & paste starting from here ===
10:M 30 Apr 2021 21:27:20.002 # === ASSERTION FAILED ===
10:M 30 Apr 2021 21:27:20.002 # ==> module.c:4085 '(c->flags & CLIENT_BLOCKED) == 0' is not true

------ STACK TRACE ------

Backtrace:
/usr/local/bin/redis-server *:30001 [cluster](+0xd3a54)[0x5598e37e9a54]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x88acd)[0x7efcaeb04acd]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyMethodDef_RawFastCallKeywords+0x2d1)[0x7efcaeb37531]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyCFunction_FastCallKeywords+0x25)[0x7efcaeb375f5]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x909a)[0x7efcaeafba4a]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0xcb5a6)[0x7efcaeb475a6]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x699f)[0x7efcaeaf934f]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0xcb5a6)[0x7efcaeb475a6]
/var/opt/redislabs/modules/rg/python3_99.99.99/lib/python3.7/lib-dynload/_asyncio.cpython-37m-x86_64-linux-gnu.so(+0x820f)[0x7efcb2e2e20f]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyObject_FastCallKeywords+0xd3)[0x7efcaeb376e3]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x1bb249)[0x7efcaec37249]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyMethodDef_RawFastCallDict+0x24b)[0x7efcaeb37bbb]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyCFunction_FastCallDict+0x25)[0x7efcaeb37cc5]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x6fd0)[0x7efcaeaf9980]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x77f1)[0x7efcaeafa1a1]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x77f1)[0x7efcaeafa1a1]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x77f1)[0x7efcaeafa1a1]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyFunction_FastCallDict+0x2d2)[0x7efcaeb36f62]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x364d)[0x7efcaeaf5ffd]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x77f1)[0x7efcaeafa1a1]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyEval_EvalFrameDefault+0x77f1)[0x7efcaeafa1a1]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x7581b)[0x7efcaeaf181b]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyFunction_FastCallDict+0x2d2)[0x7efcaeb36f62]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(_PyObject_Call_Prepend+0xcd)[0x7efcaeb3800d]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(PyObject_Call+0x75)[0x7efcaeb39215]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x2161a3)[0x7efcaec921a3]
/var/opt/redislabs/modules/rg/plugin/gears_python.so(+0x1cb0c7)[0x7efcaec470c7]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x7fa3)[0x7efcb30adfa3]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x3f)[0x7efcb2fdc4cf]

------ INFO OUTPUT ------
# Server
redis_version:6.2.1
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:638282da034c5dbe
redis_mode:cluster
os:Linux 5.4.0-7642-generic x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:8.3.0
process_id:10
process_supervised:no
run_id:e392b4577c52671bdc6c7854372c71872c678950
tcp_port:30001
server_time_usec:1619818040002316
uptime_in_seconds:36367
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:9205303
executable:/usr/local/bin/redis-server
config_file:
io_threads_active:0

# Clients
connected_clients:12
cluster_connections:18
maxclients:10000
client_recent_max_input_buffer:56
client_recent_max_output_buffer:0
blocked_clients:2
tracking_clients:0
clients_in_timeout_table:0

# Memory
used_memory:1588444824
used_memory_human:1.48G
used_memory_rss:3262795776
used_memory_rss_human:3.04G
used_memory_peak:5201974144
used_memory_peak_human:4.84G
used_memory_peak_perc:30.54%
used_memory_overhead:10333808
used_memory_startup:8981432
used_memory_dataset:1578111016
used_memory_dataset_perc:99.91%
allocator_allocated:1591094344
allocator_active:1596485632
allocator_resident:1626009600
total_system_memory:67126095872
total_system_memory_human:62.52G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.00
allocator_frag_bytes:5391288
allocator_rss_ratio:1.02
allocator_rss_bytes:29523968
rss_overhead_ratio:2.01
rss_overhead_bytes:1636786176
mem_fragmentation_ratio:2.05
mem_fragmentation_bytes:1674466472
mem_not_counted_for_evict:4
mem_replication_backlog:1048576
mem_clients_slaves:20512
mem_clients_normal:246168
mem_aof_buffer:8
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
current_cow_size:0
current_fork_perc:0.00%
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:31
rdb_bgsave_in_progress:0
rdb_last_save_time:1619814609
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:22
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:1953128448
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:13
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:4145692672
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:1385976253
aof_base_size:1357412873
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0

# Stats
total_connections_received:115
total_commands_processed:206854
instantaneous_ops_per_sec:1
total_net_input_bytes:1338983934
total_net_output_bytes:1409093297
instantaneous_input_kbps:0.05
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:2
sync_partial_ok:0
sync_partial_err:2
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:305
evicted_keys:0
keyspace_hits:168156
keyspace_misses:27452
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:68586
total_forks:10
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:45
dump_payload_sanitizations:0
total_reads_processed:118424
total_writes_processed:4568
io_threaded_reads_processed:0
io_threaded_writes_processed:0

# Replication
role:master
connected_slaves:1
slave0:ip=172.19.0.4,port=30006,state=online,offset=1388506349,lag=1
master_failover_state:no-failover
master_replid:4eadc882d7d565add2837b3a8fc3e0f677708939
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1388506349
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1387457774
repl_backlog_histlen:1048576

# CPU
used_cpu_sys:279.881534
used_cpu_user:256.164560
used_cpu_sys_children:14.473913
used_cpu_user_children:95.306692
used_cpu_sys_main_thread:0.066528
used_cpu_user_main_thread:0.896178

# Modules
module:name=ai,ver=999999,api=1,filters=0,usedby=[rg],using=[],options=[handle-io-errors]
module:name=rg,ver=999999,api=1,filters=2,usedby=[rg],using=[rgai],options=[]

# Commandstats
cmdstat_rg.dumpregistrations:calls=19,usec=1023,usec_per_call=53.84,rejected_calls=0,failed_calls=0
cmdstat_rg.configset:calls=1,usec=9,usec_per_call=9.00,rejected_calls=0,failed_calls=0
cmdstat_psync:calls=2,usec=4900,usec_per_call=2450.00,rejected_calls=0,failed_calls=0
cmdstat_rg.hello:calls=9,usec=15,usec_per_call=1.67,rejected_calls=0,failed_calls=0
cmdstat_sismember:calls=94946,usec=39735,usec_per_call=0.42,rejected_calls=0,failed_calls=0
cmdstat_xtrim:calls=11592,usec=67083,usec_per_call=5.79,rejected_calls=0,failed_calls=0
cmdstat_RG.INNERHOOK:calls=17,usec=976,usec_per_call=57.41,rejected_calls=0,failed_calls=0
cmdstat_RG.INNERMSGCOMMAND:calls=88,usec=1389,usec_per_call=15.78,rejected_calls=0,failed_calls=0
cmdstat_hget:calls=1,usec=6,usec_per_call=6.00,rejected_calls=1,failed_calls=0
cmdstat_ai.modelset:calls=1,usec=3120222,usec_per_call=3120222.00,rejected_calls=0,failed_calls=0
cmdstat_replconf:calls=36248,usec=28341,usec_per_call=0.78,rejected_calls=0,failed_calls=0
cmdstat_scan:calls=1,usec=6,usec_per_call=6.00,rejected_calls=0,failed_calls=0
cmdstat_xadd:calls=11592,usec=140064,usec_per_call=12.08,rejected_calls=0,failed_calls=0
cmdstat_ai.tensorset:calls=39,usec=390,usec_per_call=10.00,rejected_calls=0,failed_calls=0
cmdstat_set:calls=21,usec=315,usec_per_call=15.00,rejected_calls=0,failed_calls=0
cmdstat_ai.info:calls=1,usec=12,usec_per_call=12.00,rejected_calls=0,failed_calls=0
cmdstat_xack:calls=11592,usec=19057,usec_per_call=1.64,rejected_calls=0,failed_calls=0
cmdstat_xinfo:calls=1,usec=13,usec_per_call=13.00,rejected_calls=0,failed_calls=0
cmdstat_ai.modelrun:calls=13,usec=803,usec_per_call=61.77,rejected_calls=0,failed_calls=0
cmdstat_config:calls=16,usec=176,usec_per_call=11.00,rejected_calls=0,failed_calls=0
cmdstat_keys:calls=8,usec=486,usec_per_call=60.75,rejected_calls=0,failed_calls=0
cmdstat_command:calls=14,usec=1188,usec_per_call=84.86,rejected_calls=0,failed_calls=0
cmdstat_hgetall:calls=703,usec=26009,usec_per_call=37.00,rejected_calls=0,failed_calls=0
cmdstat_hset:calls=696,usec=7443,usec_per_call=10.69,rejected_calls=0,failed_calls=0
cmdstat_xlen:calls=11592,usec=3250,usec_per_call=0.28,rejected_calls=0,failed_calls=0
cmdstat_info:calls=6,usec=279,usec_per_call=46.50,rejected_calls=0,failed_calls=0
cmdstat_rg.infocluster:calls=1,usec=8,usec_per_call=8.00,rejected_calls=0,failed_calls=0
cmdstat_rg.refreshcluster:calls=1,usec=34,usec_per_call=34.00,rejected_calls=0,failed_calls=0
cmdstat_xreadgroup:calls=11595,usec=188596,usec_per_call=16.27,rejected_calls=0,failed_calls=1
cmdstat_rg.trigger:calls=34,usec=1700,usec_per_call=50.00,rejected_calls=0,failed_calls=5
cmdstat_ping:calls=31,usec=22,usec_per_call=0.71,rejected_calls=0,failed_calls=0
cmdstat_rg.pyexecute:calls=29,usec=309109,usec_per_call=10658.93,rejected_calls=0,failed_calls=2
cmdstat_xgroup:calls=1,usec=4,usec_per_call=4.00,rejected_calls=0,failed_calls=0
cmdstat_get:calls=13739,usec=24534,usec_per_call=1.79,rejected_calls=1,failed_calls=0
cmdstat_ai.tensorget:calls=40,usec=216,usec_per_call=5.40,rejected_calls=0,failed_calls=1
cmdstat_cluster:calls=23,usec=2277,usec_per_call=99.00,rejected_calls=0,failed_calls=0
cmdstat_rg.unregister:calls=53,usec=1197,usec_per_call=22.58,rejected_calls=0,failed_calls=30
cmdstat_sadd:calls=2088,usec=4409,usec_per_call=2.11,rejected_calls=0,failed_calls=0

# Errorstats
errorstat_ERR:count=8
errorstat_Error:count=1
errorstat_MOVED:count=1
errorstat_NOGROUP:count=1
errorstat_['Traceback:count=1
errorstat_execution:count=30
errorstat_object:count=3

# Cluster
cluster_enabled:1

# Keyspace
db0:keys=723,expires=0,avg_ttl=0

------ CLIENT LIST OUTPUT ------
id=148777 addr=172.19.0.1:48132 laddr=172.19.0.4:30001 fd=41 name= age=1771 idle=1 flags=b db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=40954 argv-mem=56 obl=0 oll=0 omem=0 tot-mem=61520 events=r cmd=RG.INNERHOOK user=default redir=-1
id=23 addr=172.19.0.4:52146 laddr=172.19.0.4:30001 fd=50 name= age=36361 idle=36361 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20496 events=r cmd=rg.hello user=default redir=-1
id=24 addr=172.19.0.4:52156 laddr=172.19.0.4:30001 fd=51 name= age=36361 idle=35609 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20536 events=r cmd=RG.INNERMSGCOMMAND user=default redir=-1
id=25 addr=172.19.0.4:52166 laddr=172.19.0.4:30001 fd=52 name= age=36361 idle=36361 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20496 events=r cmd=rg.hello user=default redir=-1
id=26 addr=172.19.0.4:52174 laddr=172.19.0.4:30001 fd=53 name= age=36361 idle=35609 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20536 events=r cmd=RG.INNERMSGCOMMAND user=default redir=-1
id=27 addr=172.19.0.4:52182 laddr=172.19.0.4:30001 fd=54 name= age=36361 idle=36361 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20496 events=r cmd=rg.hello user=default redir=-1
id=19 addr=172.19.0.4:52108 laddr=172.19.0.4:30001 fd=46 name= age=36361 idle=35610 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20536 events=r cmd=RG.INNERMSGCOMMAND user=default redir=-1
id=20 addr=172.19.0.4:52120 laddr=172.19.0.4:30001 fd=47 name= age=36361 idle=36361 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20496 events=r cmd=rg.hello user=default redir=-1
id=148752 addr=172.19.0.1:47666 laddr=172.19.0.4:30001 fd=39 name= age=3119 idle=3119 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20520 events=r cmd=rg.trigger user=default redir=-1
id=21 addr=172.19.0.4:52128 laddr=172.19.0.4:30001 fd=48 name= age=36361 idle=35609 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20536 events=r cmd=RG.INNERMSGCOMMAND user=default redir=-1
id=41221 addr=172.19.0.4:55954 laddr=172.19.0.4:30001 fd=63 name= age=36011 idle=1 flags=S db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20512 events=r cmd=replconf user=default redir=-1
id=148742 addr=172.19.0.1:47664 laddr=172.19.0.4:30001 fd=8 name= age=3119 idle=3119 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20520 events=r cmd=rg.trigger user=default redir=-1
id=22 addr=172.19.0.4:52138 laddr=172.19.0.4:30001 fd=49 name= age=36361 idle=36361 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20496 events=r cmd=rg.hello user=default redir=-1

------ MODULES INFO OUTPUT ------
# ai_git
ai_git_sha:43b91ff882d001279274052a5a41b08ec29eb94f

# ai_load_time_configs
ai_threads_per_queue:1
ai_inter_op_parallelism:0
ai_intra_op_parallelism:0

# ai_memory_usage
ai_onnxruntime_memory:0
ai_onnxruntime_memory_access_num:0

# ai_cpu
ai_self_used_cpu_sys:279.881534
ai_self_used_cpu_user:256.164720
ai_children_used_cpu_sys:14.473913
ai_children_used_cpu_user:95.306692
ai_main_thread_used_cpu_sys:0.881534
ai_main_thread_used_cpu_user:0.164720
ai_queue_CPU_bthread_n1_used_cpu_total:316.766965

------ FAST MEMORY TEST ------
10:M 30 Apr 2021 21:27:20.011 # main thread terminated
10:M 30 Apr 2021 21:27:20.011 # Bio thread for job type #0 terminated
10:M 30 Apr 2021 21:27:20.011 # Bio thread for job type #1 terminated
10:M 30 Apr 2021 21:27:20.011 # Bio thread for job type #2 terminated