pxc-pod-60gb-prob.log

2024-05-08T15:31:10.521707Z 0 [Note] [MY-000000] [WSREP] Initiating SST/IST transfer on JOINER side (wsrep_sst_xtrabackup-v2 --role 'joiner' --address '10.64.6.3' --datadir '/var/lib/mysql/' --basedir '/usr/' --plugindir '/usr/lib64/mysql/plugin/' --defaults-file '/etc/my.cnf' --defaults-group-suffix '' --parent '1' --mysqld-version '8.0.32-24.2'  --binlog 'binlog' )
2024-05-08T15:31:11.024215Z 0 [Warning] [MY-000000] [WSREP-SST] Found a stale sst_in_progress file: /var/lib/mysql//sst_in_progress
2024-05-08T15:31:11.374268Z 1 [Note] [MY-000000] [WSREP] Prepared SST request: xtrabackup-v2|10.64.6.3:4444/xtrabackup_sst//1
2024-05-08T15:31:11.374344Z 1 [Note] [MY-000000] [Galera] Check if state gap can be serviced using IST
2024-05-08T15:31:11.374367Z 1 [Note] [MY-000000] [Galera] Local UUID: 00000000-0000-0000-0000-000000000000 != Group UUID: 60e73ca4-0d0d-11ef-9cf4-73add5454eb5
2024-05-08T15:31:11.374388Z 1 [Note] [MY-000000] [Galera] ####### IST uuid:00000000-0000-0000-0000-000000000000 f: 0, l: 120366, STRv: 3
2024-05-08T15:31:11.374487Z 1 [Note] [MY-000000] [Galera] IST receiver addr using ssl://10.64.6.3:4568
2024-05-08T15:31:11.374576Z 1 [Note] [MY-000000] [Galera] IST receiver using ssl
2024-05-08T15:31:11.375022Z 1 [Note] [MY-000000] [Galera] Prepared IST receiver for 0-120366, listening at: ssl://10.64.6.3:4568
2024-05-08T15:31:11.376847Z 0 [Note] [MY-000000] [Galera] Member 2.0 (mysql-ofd-pxc-2) requested state transfer from 'mysql-ofd-pxc-1,'. Selected 1.0 (mysql-ofd-pxc-1)(SYNCED) as donor.
2024-05-08T15:31:11.376912Z 0 [Note] [MY-000000] [Galera] Shifting PRIMARY -> JOINER (TO: 120366)
2024-05-08T15:31:11.377095Z 1 [Note] [MY-000000] [Galera] Requesting state transfer: success, donor: 1
2024-05-08T15:31:11.377169Z 1 [Note] [MY-000000] [Galera] Resetting GCache seqno map due to different histories.
2024-05-08T15:31:11.377266Z 1 [Note] [MY-000000] [Galera] GCache history reset: 60e73ca4-0d0d-11ef-9cf4-73add5454eb5:0 -> 60e73ca4-0d0d-11ef-9cf4-73add5454eb5:120366
2024-05-08T15:31:12.144124Z 0 [Note] [MY-000000] [WSREP-SST]    joiner: => Rate:[ 167 B/s] Avg:[ 167 B/s] Elapsed:0:00:01  Bytes:  171 B 
2024-05-08T15:31:13.234929Z 0 [Note] [MY-000000] [WSREP-SST] Proceeding with SST.........
2024-05-08T15:31:13.253084Z 0 [Note] [MY-000000] [WSREP-SST] ............Waiting for SST streaming to complete!
2024-05-08T15:31:21.508046Z 0 [Note] [MY-000000] [Galera] (e6b92080-bc6b, 'ssl://0.0.0.0:4567') turning message relay requesting off
   joiner: => Rate:[ 169MiB/s] Avg:[ 155MiB/s] Elapsed:0:02:10  Bytes: 18.1GiB :[0.00 B/s] Avg:[0.00 B/s] Elapsed:0:00:10  Bytes: 0.00 B 
   joiner: => Rate:[ 172MiB/s] Avg:[ 162MiB/s] Elapsed:0:04:10  Bytes: 39.7GiB 
   joiner: => Rate:[ 174MiB/s] Avg:[ 165MiB/s] Elapsed:0:06:20  Bytes: 61.5GiB es: 41.3GiB 
   joiner: => Rate:[ 166MiB/s] Avg:[ 166MiB/s] Elapsed:0:07:27  Bytes: 72.7GiB psed:0:06:30  Bytes: 63.2GiB 
2024-05-08T15:38:40.269486Z 0 [Note] [MY-000000] [Galera] 1.0 (mysql-ofd-pxc-1): State transfer to 2.0 (mysql-ofd-pxc-2) complete.
2024-05-08T15:38:40.271282Z 0 [Note] [MY-000000] [Galera] Member 1.0 (mysql-ofd-pxc-1) synced with group.
2024-05-08T15:38:40.276028Z 0 [Note] [MY-000000] [WSREP-SST] Preparing the backup at /var/lib/mysql//sst-xb-tmpdir
2024-05-08T15:38:43.891124Z 0 [Note] [MY-000000] [WSREP-SST] Moving the backup to /var/lib/mysql/
2024-05-08T15:38:44.020596Z 0 [Note] [MY-000000] [WSREP-SST] Running post-processing...........
2024-05-08T15:38:44.030042Z 0 [Note] [MY-000000] [WSREP-SST] Skipping mysql_upgrade (sst): local version (8.0.32) == donor version (8.0.32)
2024-05-08T15:38:44.099882Z 0 [Note] [MY-000000] [WSREP-SST] Waiting for server instance to start.....  This may take some time
2024-05-08T15:38:48.756892Z 0 [ERROR] [MY-000000] [WSREP-SST] ******************* FATAL ERROR ********************** 
2024-05-08T15:38:48.756933Z 0 [ERROR] [MY-000000] [WSREP-SST] Failed to start the mysql server that checks for async replication.
2024-05-08T15:38:48.756942Z 0 [ERROR] [MY-000000] [WSREP-SST] Check the parameters and retry
2024-05-08T15:38:48.756969Z 0 [ERROR] [MY-000000] [WSREP-SST] Line 521  pid:2097
2024-05-08T15:38:48.759674Z 0 [ERROR] [MY-000000] [WSREP-SST] ------------ mysql error log (START) ------------
	---- Starting the MySQL server used for post-processing ----
	2024-05-08T15:38:44.433289Z 0 [Warning] [MY-011068] [Server] The syntax '--skip-host-cache' is deprecated and will be removed in a future release. Please use SET GLOBAL host_cache_size=0 instead.
	2024-05-08T15:38:44.433310Z 0 [Warning] [MY-011068] [Server] The syntax 'wsrep_slave_threads' is deprecated and will be removed in a future release. Please use wsrep_applier_threads instead.
	2024-05-08T15:38:44.433366Z 0 [Warning] [MY-011069] [Server] The syntax '--replica-parallel-type' is deprecated and will be removed in a future release.
	2024-05-08T15:38:44.433395Z 0 [Warning] [MY-011068] [Server] The syntax 'wsrep_slave_threads' is deprecated and will be removed in a future release. Please use wsrep_applier_threads instead.
	2024-05-08T15:38:44.433414Z 0 [Warning] [MY-011068] [Server] The syntax 'skip_slave_start' is deprecated and will be removed in a future release. Please use skip_replica_start instead.
	2024-05-08T15:38:44.433477Z 0 [Warning] [MY-000000] [WSREP] --wsrep-sync-wait=3 takes precedence over --wsrep-causal-reads=OFF. WSREP_SYNC_WAIT_BEFORE_READ is on
	2024-05-08T15:38:44.436029Z 0 [Warning] [MY-010097] [Server] Insecure configuration for --secure-log-path: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.
	2024-05-08T15:38:44.436068Z 0 [Warning] [MY-000000] [WSREP] Node is not a cluster node. Disabling pxc_strict_mode
	2024-05-08T15:38:44.436726Z 0 [System] [MY-010116] [Server] /usr/sbin/mysqld (mysqld 8.0.32-24.2) starting as process 2097
	2024-05-08T15:38:44.440276Z 0 [Warning] [MY-010068] [Server] CA certificate /etc/mysql/ssl-internal/ca.crt is self signed.
	2024-05-08T15:38:44.440325Z 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channel.
	2024-05-08T15:38:44.445926Z 0 [Warning] [MY-010075] [Server] No existing UUID has been found, so we assume that this is the first time that this server has been started. Generating a new UUID: 0da8e1ac-0d51-11ef-823d-c67a8be7f65a.
	2024-05-08T15:38:44.453071Z 1 [System] [MY-013576] [InnoDB] InnoDB initialization has started.
	2024-05-08T15:38:45.446991Z 1 [System] [MY-013577] [InnoDB] InnoDB initialization has ended.
	2024-05-08T15:38:45.762221Z 1 [Note] [MY-000000] [WSREP] wsrep_init_schema_and_SR (nil)
	2024-05-08T15:38:45.789671Z 1 [System] [MY-000000] [WSREP] PXC upgrade completed successfully
	2024-05-08T15:38:45.874069Z 0 [System] [MY-010229] [Server] Starting XA crash recovery...
	2024-05-08T15:38:45.898370Z 0 [System] [MY-010232] [Server] XA crash recovery finished.
	2024-05-08T15:38:46.011225Z 0 [Warning] [MY-010068] [Server] CA certificate /etc/mysql/ssl-internal/ca.crt is self signed.
	2024-05-08T15:38:46.011268Z 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channel.
	2024-05-08T15:38:46.024919Z 0 [Warning] [MY-013595] [Server] Failed to initialize TLS for channel: mysql_admin. See below for the description of exact issue.
	2024-05-08T15:38:46.024959Z 0 [Warning] [MY-010069] [Server] Failed to set up SSL because of the following SSL library error: SSL context is not usable without certificate and private key
	2024-05-08T15:38:46.024978Z 0 [System] [MY-013603] [Server] No TLS configuration was given for channel mysql_admin; re-using TLS configuration of channel mysql_main.
	2024-05-08T15:38:46.054132Z 0 [Warning] [MY-011810] [Server] Insecure configuration for --pid-file: Location '/tmp' in the path is accessible to all OS users. Consider choosing a different directory.
	2024-05-08T15:38:46.057505Z 0 [ERROR] [MY-000067] [Server] unknown variable 'compress=lz4'.
	2024-05-08T15:38:46.057623Z 0 [ERROR] [MY-010119] [Server] Aborting
	2024-05-08T15:38:47.748412Z 0 [System] [MY-010910] [Server] /usr/sbin/mysqld: Shutdown complete (mysqld 8.0.32-24.2)  Percona XtraDB Cluster (GPL), Release rel24, Revision 2119e75, WSREP version 26.1.4.3.

2024-05-08T15:38:48.759719Z 0 [ERROR] [MY-000000] [WSREP-SST] ------------ mysql error log (END) ------------
2024-05-08T15:38:48.759733Z 0 [ERROR] [MY-000000] [WSREP-SST] ****************************************************** 
2024-05-08T15:38:48.760150Z 0 [Note] [MY-000000] [WSREP-SST] ...........post-processing failed.  Exiting
2024-05-08T15:38:48.760226Z 0 [ERROR] [MY-000000] [WSREP-SST] Cleanup after exit with status:3
2024-05-08T15:38:48.770668Z 0 [ERROR] [MY-000000] [WSREP] Process completed with error: wsrep_sst_xtrabackup-v2 --role 'joiner' --address '10.64.6.3' --datadir '/var/lib/mysql/' --basedir '/usr/' --plugindir '/usr/lib64/mysql/plugin/' --defaults-file '/etc/my.cnf' --defaults-group-suffix '' --parent '1' --mysqld-version '8.0.32-24.2'  --binlog 'binlog' : 3 (No such process)
2024-05-08T15:38:48.770702Z 0 [ERROR] [MY-000000] [WSREP] Failed to read uuid:seqno from joiner script.
2024-05-08T15:38:48.770712Z 0 [ERROR] [MY-000000] [WSREP] SST script aborted with error 3 (No such process)
2024-05-08T15:38:48.770802Z 3 [Note] [MY-000000] [Galera] Processing SST received
2024-05-08T15:38:48.770846Z 3 [Note] [MY-000000] [Galera] SST received: 00000000-0000-0000-0000-000000000000:-1
2024-05-08T15:38:48.770869Z 3 [System] [MY-000000] [WSREP] SST completed
2024-05-08T15:38:48.771030Z 1 [Note] [MY-000000] [Galera]  str_proto_ver_: 3 sst_seqno_: -1 cc_seqno: 120366 req->ist_len(): 68
2024-05-08T15:38:48.771061Z 1 [ERROR] [MY-000000] [Galera] Application received wrong state: 
	Received: 00000000-0000-0000-0000-000000000000
	Required: 60e73ca4-0d0d-11ef-9cf4-73add5454eb5
2024-05-08T15:38:48.771072Z 1 [ERROR] [MY-000000] [Galera] Application state transfer failed. This is unrecoverable condition, restart required.
2024-05-08T15:38:48.771084Z 1 [Note] [MY-000000] [Galera] ReplicatorSMM::abort()
2024-05-08T15:38:48.771096Z 1 [Note] [MY-000000] [Galera] Closing send monitor...
2024-05-08T15:38:48.771107Z 1 [Note] [MY-000000] [Galera] Closed send monitor.
2024-05-08T15:38:48.771120Z 1 [Note] [MY-000000] [Galera] gcomm: terminating thread
2024-05-08T15:38:48.771144Z 1 [Note] [MY-000000] [Galera] gcomm: joining thread
2024-05-08T15:38:48.771329Z 1 [Note] [MY-000000] [Galera] gcomm: closing backend
2024-05-08T15:38:48.773325Z 1 [Note] [MY-000000] [Galera] Current view of cluster as seen by this node
view (view_id(NON_PRIM,72aaf26f-a31a,15)
memb {
	e6b92080-bc6b,0
	}
joined {
	}
left {
	}
partitioned {
	72aaf26f-a31a,0
	b4435d17-aac3,0
	}
)
2024-05-08T15:38:48.773383Z 1 [Note] [MY-000000] [Galera] PC protocol downgrade 1 -> 0
2024-05-08T15:38:48.773405Z 1 [Note] [MY-000000] [Galera] Current view of cluster as seen by this node
view ((empty))
2024-05-08T15:38:48.773639Z 1 [Note] [MY-000000] [Galera] gcomm: closed
2024-05-08T15:38:48.773678Z 0 [Note] [MY-000000] [Galera] New COMPONENT: primary = no, bootstrap = no, my_idx = 0, memb_num = 1
2024-05-08T15:38:48.773797Z 0 [Note] [MY-000000] [Galera] Flow-control interval: [96, 96]
2024-05-08T15:38:48.773810Z 0 [Note] [MY-000000] [Galera] Received NON-PRIMARY.
2024-05-08T15:38:48.773821Z 0 [Note] [MY-000000] [Galera] Shifting JOINER -> OPEN (TO: 120366)
2024-05-08T15:38:48.773835Z 0 [Note] [MY-000000] [Galera] New SELF-LEAVE.
2024-05-08T15:38:48.773866Z 0 [Note] [MY-000000] [Galera] Flow-control interval: [0, 0]
2024-05-08T15:38:48.773945Z 0 [Note] [MY-000000] [Galera] Received SELF-LEAVE. Closing connection.
2024-05-08T15:38:48.773964Z 0 [Note] [MY-000000] [Galera] Shifting OPEN -> CLOSED (TO: 120366)
2024-05-08T15:38:48.773979Z 0 [Note] [MY-000000] [Galera] RECV thread exiting 0: Success
2024-05-08T15:38:48.774044Z 1 [Note] [MY-000000] [Galera] recv_thread() joined.
2024-05-08T15:38:48.774061Z 1 [Note] [MY-000000] [Galera] Closing replication queue.
2024-05-08T15:38:48.774070Z 1 [Note] [MY-000000] [Galera] Closing slave action queue.
2024-05-08T15:38:48.774085Z 1 [Note] [MY-000000] [Galera] mysqld: Terminated.
2024-05-08T15:38:48.774094Z 1 [Note] [MY-000000] [WSREP] Initiating SST cancellation
2024-05-08T15:38:48Z UTC - mysqld got signal 11 ;
Most likely, you have hit a bug, but this error can also be caused by malfunctioning hardware.
BuildID[sha1]=df9f6877fc91c9a71d439f27569eabdef408f622
Server Version: 8.0.32-24.2 Percona XtraDB Cluster (GPL), Release rel24, Revision 2119e75, WSREP version 26.1.4.3, wsrep_26.1.4.3

Thread pointer: 0x7c9ecc000b60
Attempting backtrace. You can use the following information to find out
where mysqld died. If you see no messages after this, something went
terribly wrong...
stack_bottom = 7c9edbdd7c70 thread_stack 0x100000
/usr/sbin/mysqld(my_print_stacktrace(unsigned char const*, unsigned long)+0x41) [0x2253a31]
/usr/sbin/mysqld(print_fatal_signal(int)+0x39f) [0x1262d0f]
/usr/sbin/mysqld(handle_fatal_signal+0xd8) [0x1262df8]
/lib64/libpthread.so.0(+0x12ce0) [0x7c9f63163ce0]
/lib64/libc.so.6(abort+0x203) [0x7c9f614e5ee1]
/usr/lib64/galera4/libgalera_smm.so(+0x17935) [0x7c9f54843935]
/usr/lib64/galera4/libgalera_smm.so(+0x207d92) [0x7c9f54a33d92]
/usr/lib64/galera4/libgalera_smm.so(+0x226d59) [0x7c9f54a52d59]
/usr/lib64/galera4/libgalera_smm.so(+0x209905) [0x7c9f54a35905]
/usr/lib64/galera4/libgalera_smm.so(+0x212077) [0x7c9f54a3e077]
/usr/lib64/galera4/libgalera_smm.so(+0x2122be) [0x7c9f54a3e2be]
/usr/lib64/galera4/libgalera_smm.so(+0x1f07d2) [0x7c9f54a1c7d2]
/usr/lib64/galera4/libgalera_smm.so(+0x2113c4) [0x7c9f54a3d3c4]
/usr/lib64/galera4/libgalera_smm.so(+0x22e262) [0x7c9f54a5a262]
/usr/sbin/mysqld(wsrep::wsrep_provider_v26::run_applier(wsrep::high_priority_service*)+0x12) [0x2c25002]
/usr/sbin/mysqld() [0x12b46dd]
/usr/sbin/mysqld(start_wsrep_THD+0x371) [0xf5be41]
/usr/sbin/mysqld() [0x27b1249]
/lib64/libpthread.so.0(+0x81cf) [0x7c9f631591cf]
/lib64/libc.so.6(clone+0x43) [0x7c9f614fddd3]

Trying to get some variables.
Some pointers may be invalid and cause the dump to abort.
Query (0): is an invalid pointer
Connection ID (thread ID): 1
Status: NOT_KILLED

You may download the Percona XtraDB Cluster operations manual by visiting
http://www.percona.com/software/percona-xtradb-cluster/. You may find information
in the manual which will help you identify the cause of the crash.
Writing a core file using lib coredumper
PATH: (null)

params

[mysqld]
binlog_cache_size = 131072
binlog_expire_logs_seconds = 604800
binlog_format = ROW
binlog_stmt_cache_size = 131072
global-connection-memory-limit = 18446744073709551615
global-connection-memory-tracking = false
innodb_adaptive_hash_index = True
innodb_buffer_pool_chunk_size = 2097152
innodb_buffer_pool_instances = 6
innodb_buffer_pool_size = 5512928528
innodb_flush_log_at_trx_commit = 2
innodb_flush_method = O_DIRECT
innodb_io_capacity_max = 1800
innodb_monitor_enable = ALL
innodb_page_cleaners = 6
innodb_parallel_read_threads = 3
innodb_purge_threads = 4
innodb_redo_log_capacity = 1710392192
join_buffer_size = 524288
max_connections = 802
max_heap_table_size = 16777216
read_rnd_buffer_size = 393216
replica_compressed_protocol = 1
replica_exec_mode = STRICT
replica_parallel_type = LOGICAL_CLOCK
replica_parallel_workers = 4
replica_preserve_commit_order = ON
sort_buffer_size = 524288
sql_mode = 'ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION,TRADITIONAL,STRICT_ALL_TABLES'
sync_binlog = 1
table_definition_cache = 4096
table_open_cache = 4096
table_open_cache_instances = 4
tablespace_definition_cache = 512
thread_cache_size = 24
thread_pool_size = 6
thread_stack = 1048576
tmp_table_size = 16777216
wsrep_slave_threads = 1
wsrep_sync_wait = 3
wsrep_trx_fragment_size = 1048576
wsrep_trx_fragment_unit = bytes
wsrep-provider-options = evs.suspect_timeout=PT45S;gcs.max_packet_size=98365;evs.delay_margin=PT22S;evs.inactive_timeout=PT90S;evs.join_retrans_period=PT3S;gmcast.peer_timeout=PT11S;gcache.size=1966951020;evs.send_window=768;evs.user_send_window=768;evs.max_install_timeouts=3;pc.linger=PT45S;evs.stats_report_period=PT1M;gcs.fc_limit=96;gmcast.time_wait=PT13S;evs.inactive_check_period=PT3S;pc.announce_timeout=PT45S;pc.recovery=true;gcache.recover=yes;evs.delayed_keep_period=PT545S;