Statestore log

vtran@blackbox:~$ cat /data/vtran/Impala/logs/cluster/statestored.INFO
Log file created at: 2018/02/21 18:50:05
Running on machine: blackbox
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I0221 18:50:05.512274 98713 logging.cc:120] stdout will be logged to this file.
E0221 18:50:05.512434 98713 logging.cc:121] stderr will be logged to this file.
I0221 18:50:05.512670 98713 minidump.cc:231] Setting minidump size limit to 20971520.
I0221 18:50:05.512708 98713 atomicops-internals-x86.cc:98] vendor GenuineIntel  family 6  model 14  sse2 1  cmpxchg16b 1
I0221 18:50:05.515506 98713 authentication.cc:1078] Internal communication is not authenticated
I0221 18:50:05.515512 98713 authentication.cc:1099] External communication is not authenticated
I0221 18:50:05.515821 98713 init.cc:231] statestored version 3.0.0-SNAPSHOT DEBUG (build 74dd9e9bfe32541e9ab6bddb8b839f3e491361f1)
Built on Wed Feb 21 13:12:06 EST 2018
I0221 18:50:05.515827 98713 init.cc:232] Using hostname: blackbox
I0221 18:50:05.516355 98713 logging.cc:156] Flags (see also /varz are on debug webserver):
--flagfile=
--fromenv=
--tryfromenv=
--undefok=
--tab_completion_columns=80
--tab_completion_word=
--help=false
--helpfull=false
--helpmatch=
--helpon=
--helppackage=false
--helpshort=false
--helpxml=false
--version=false
--catalog_service_port=26000
--initial_hms_cnxn_timeout_s=120
--load_catalog_in_background=false
--max_hdfs_partitions_parallel_load=5
--max_nonhdfs_partitions_parallel_load=20
--num_metadata_loading_threads=16
--sentry_catalog_polling_frequency_s=60
--sentry_config=
--asm_module_dir=
--disable_optimization_passes=false
--dump_ir=false
--opt_module_dir=
--perf_map=false
--print_llvm_ir_instruction_count=false
--unopt_module_dir=
--abort_on_config_error=true
--be_port=22000
--be_principal=
--buffer_pool_clean_pages_limit=10%
--buffer_pool_limit=85%
--compact_catalog_topic=false
--disable_kudu=false
--disable_mem_pools=false
--enable_minidumps=true
--enable_process_lifetime_heap_profiling=false
--enable_stats_extrapolation=false
--fault_injection_rpc_delay_ms=0
--fault_injection_rpc_exception_type=0
--fault_injection_rpc_type=0
--heap_profile_dir=
--hostname=blackbox
--inc_stats_size_limit_bytes=209715200
--keytab_file=
--krb5_conf=
--krb5_debug_file=
--kudu_operation_timeout_ms=180000
--load_auth_to_local_rules=false
--log_filename=statestored
--max_log_files=10
--max_minidumps=9
--mem_limit=80%
--min_buffer_size=8192
--minidump_path=/data/vtran/Impala/logs/cluster/minidumps/statestored
--minidump_size_limit_hint_kb=20480
--principal=
--read_size=8388608
--redaction_rules_file=
--redirect_stdout_stderr=true
--reserved_words_version=3.0.0
--skip_file_runtime_filtering=false
--stress_catalog_init_delay_ms=0
--stress_datastream_recvr_delay_ms=0
--stress_fn_ctx_alloc=0
--stress_scratch_write_delay_ms=0
--tcmalloc_max_total_thread_cache_bytes=0
--thread_creation_fault_injection=false
--local_library_dir=/tmp
--max_audit_event_log_files=0
--memory_maintenance_sleep_time_ms=10000
--pause_monitor_sleep_time_ms=500
--pause_monitor_warn_threshold_ms=10000
--data_stream_sender_buffer_size=16384
--data_source_batch_size=1024
--exchg_node_buffer_size_bytes=10485760
--enable_quadratic_probing=true
--skip_lzo_version_check=false
--parquet_min_filter_reject_ratio=0.10000000000000001
--max_row_batches=0
--kudu_max_row_batches=0
--kudu_read_mode=READ_LATEST
--kudu_scanner_keep_alive_period_sec=15
--pick_only_leaders_for_tests=false
--kudu_error_buffer_size=10485760
--kudu_mutation_buffer_size=10485760
--convert_legacy_hive_parquet_utc_timestamps=false
--max_page_header_size=8388608
--runtime_filter_wait_time_ms=1000
--accepted_cnxn_queue_depth=10000
--enable_ldap_auth=false
--internal_principals_whitelist=hdfs
--kerberos_reinit_interval=60
--ldap_allow_anonymous_binds=false
--ldap_baseDN=
--ldap_bind_pattern=
--ldap_ca_certificate=
--ldap_domain=
--ldap_manual_config=false
--ldap_passwords_in_clear_ok=false
--ldap_tls=false
--ldap_uri=
--sasl_path=
--use_kudu_kinit=true
--num_acceptor_threads=2
--num_reactor_threads=0
--rpc_negotiation_thread_count=4
--rpc_negotiation_timeout_ms=60000
--rpc_retry_interval_ms=5
--concurrent_scratch_ios_per_device=2
--madvise_huge_pages=true
--mmap_buffers=false
--insert_inherit_permissions=false
--datastream_sender_timeout_ms=120000
--backend_client_connection_num_retries=3
--backend_client_rpc_timeout_ms=300000
--catalog_client_connection_num_retries=3
--catalog_client_rpc_timeout_ms=0
--catalog_service_host=localhost
--coordinator_rpc_threads=12
--disable_admission_control=false
--enable_webserver=true
--num_hdfs_worker_threads=16
--state_store_host=localhost
--state_store_subscriber_port=23000
--status_report_interval=5
--s3a_access_key_cmd=
--s3a_secret_key_cmd=
--max_cached_file_handles=0
--max_free_io_buffers=128
--num_adls_io_threads=16
--num_disks=0
--num_file_handle_cache_partitions=16
--num_io_threads_per_rotational_disk=0
--num_io_threads_per_solid_state_disk=0
--num_remote_hdfs_io_threads=8
--num_s3_io_threads=16
--num_threads_per_disk=0
--unused_file_handle_timeout_sec=270
--adls_read_chunk_size=131072
--use_hdfs_pread=false
--datastream_service_deserialization_queue_size=10000
--datastream_service_num_deserialization_threads=16
--log_mem_usage_interval=0
--max_filter_error_rate=0.75
--num_threads_per_core=3
--use_local_tz_for_unix_timestamp_conversions=false
--allow_multiple_scratch_dirs_per_device=false
--disk_spill_encryption=false
--scratch_dirs=/tmp
--queue_wait_timeout_ms=60000
--default_pool_max_queued=200
--default_pool_max_requests=-1
--default_pool_mem_limit=
--disable_pool_max_requests=false
--disable_pool_mem_limits=false
--fair_scheduler_allocation_path=
--llama_site_path=
--require_username=false
--datastream_service_num_svc_threads=0
--datastream_service_queue_mem_limit=5%
--authorization_policy_file=
--authorization_policy_provider_class=org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider
--authorized_proxy_user_config=
--authorized_proxy_user_config_delimiter=,
--kudu_master_hosts=
--server_name=
--abort_on_failed_audit_event=true
--abort_on_failed_lineage_event=true
--audit_event_log_dir=
--beeswax_port=21000
--cancellation_thread_pool_size=5
--default_query_options=
--fe_service_threads=64
--hs2_port=21050
--idle_query_timeout=0
--idle_session_timeout=0
--is_coordinator=true
--is_executor=true
--lineage_event_log_dir=
--log_query_to_file=true
--max_audit_event_log_file_size=5000
--max_lineage_log_file_size=5000
--max_profile_log_file_size=5000
--max_profile_log_files=10
--max_result_cache_size=100000
--profile_log_dir=
--query_log_size=25
--ssl_cipher_list=
--ssl_client_ca_certificate=/data/vtran/Impala/be/src/testutil/server-cert.pem
--ssl_minimum_version=tlsv1
--ssl_private_key=/data/vtran/Impala/be/src/testutil/server-key.pem
--ssl_private_key_password_cmd=
--ssl_server_certificate=/data/vtran/Impala/be/src/testutil/server-cert.pem
--stress_metadata_loading_pause_injection_ms=0
--statestore_subscriber_cnxn_attempts=10
--statestore_subscriber_cnxn_retry_interval_ms=3000
--statestore_subscriber_timeout_seconds=30
--state_store_port=24000
--statestore_heartbeat_frequency_ms=1000
--statestore_heartbeat_tcp_timeout_seconds=3
--statestore_max_missed_heartbeats=10
--statestore_max_subscribers=10000
--statestore_num_heartbeat_threads=10
--statestore_num_priority_update_threads=10
--statestore_num_update_threads=10
--statestore_priority_update_frequency_ms=100
--statestore_update_frequency_ms=2000
--statestore_update_tcp_timeout_seconds=300
--force_lowercase_usernames=false
--sasl_connect_tcp_timeout_ms=300000
--num_cores=0
--web_log_bytes=1048576
--non_impala_java_vlog=0
--periodic_counter_update_period_ms=500
--enable_webserver_doc_root=true
--webserver_authentication_domain=
--webserver_certificate_file=
--webserver_doc_root=/data/vtran/Impala
--webserver_interface=
--webserver_password_file=
--webserver_port=25010
--webserver_private_key_file=
--webserver_private_key_password_cmd=
--webserver_x_frame_options=DENY
--alsologtoemail=
--alsologtostderr=false
--colorlogtostderr=false
--drop_log_memory=true
--log_backtrace_at=
--log_dir=/data/vtran/Impala/logs/cluster
--log_link=
--log_prefix=true
--logbuflevel=0
--logbufsecs=5
--logemaillevel=999
--logmailer=/bin/mail
--logtostderr=false
--max_log_size=1800
--minloglevel=0
--stderrthreshold=4
--stop_logging_if_full_disk=false
--symbolize_stacktrace=true
--v=1
--vmodule=
I0221 18:50:05.516441 98713 init.cc:237] Cpu Info:
  Model: Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
  Cores: 16
  Max Possible Cores: 16
  L1 Cache: 32.00 KB (Line: 64.00 B)
  L2 Cache: 256.00 KB (Line: 64.00 B)
  L3 Cache: 25.00 MB (Line: 64.00 B)
  Hardware Supports:
    ssse3
    sse4_1
    sse4_2
    popcnt
    avx
  Numa Nodes: 1
  Numa Nodes of Cores: 0->0 | 1->0 | 2->0 | 3->0 | 4->0 | 5->0 | 6->0 | 7->0 | 8->0 | 9->0 | 10->0 | 11->0 | 12->0 | 13->0 | 14->0 | 15->0 |
I0221 18:50:05.516460 98713 init.cc:238] Disk Info:
  Num disks 3:
    xvda (rotational=false)
    xvdb (rotational=false)
    xvdc (rotational=false)

I0221 18:50:05.516508 98713 init.cc:239] Physical Memory: 29.44 GB
Transparent Huge Pages Config:
  enabled: [always] madvise never
  defrag: always madvise [never]
  khugepaged defrag: 1
I0221 18:50:05.516513 98713 init.cc:240] OS version: Linux version 4.4.0-104-generic (buildd@lgw01-amd64-022) (gcc version 5.4.0 20160609 (Ubuntu 5.4.0-6ubuntu1~16.04.5) ) #127-Ubuntu SMP Mon Dec 11 12:16:42 UTC 2017
Clock: clocksource: 'xen', clockid_t: CLOCK_MONOTONIC_COARSE
I0221 18:50:05.516516 98713 init.cc:241] Process ID: 98713
I0221 18:50:05.516587 98713 webserver.cc:223] Starting webserver on 0.0.0.0:25010
I0221 18:50:05.516594 98713 webserver.cc:237] Document root: /data/vtran/Impala
I0221 18:50:05.516688 98713 webserver.cc:327] Webserver started
I0221 18:50:05.522306 98713 statestored-main.cc:99] Enabling SSL for Statestore
I0221 18:50:05.523870 98713 thrift-server.cc:451] ThriftServer 'StatestoreService' started on port: 24000s
I0221 18:50:08.170115 98936 statestore.cc:531] Creating new topic: ''catalog-update' on behalf of subscriber: 'catalog-server@blackbox:26000
I0221 18:50:08.170137 98936 statestore.cc:539] Registering: catalog-server@blackbox:26000
I0221 18:50:08.170195 98936 statestore.cc:563] Subscriber 'catalog-server@blackbox:26000' registered (registration id: c546da8a10bfcfdf:c2f9a57bcb98d9a2)
I0221 18:50:08.174201 98753 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23020
I0221 18:50:08.174206 98733 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23020
I0221 18:50:08.176862 98733 client-cache.h:304] RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:08.176890 98733 client-cache.cc:170] Broken Connection, destroy client for blackbox:23020
I0221 18:50:08.176901 98753 client-cache.h:304] RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:08.176914 98753 client-cache.cc:170] Broken Connection, destroy client for blackbox:23020
I0221 18:50:08.177078 98753 statestore.cc:866] Unable to send heartbeat message to subscriber catalog-server@blackbox:26000, received error: RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:08.177120 98733 statestore.cc:866] Unable to send topic update message to subscriber catalog-server@blackbox:26000, received error: RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.180138 98754 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23020
I0221 18:50:09.183082 98754 client-cache.h:304] RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:09.183147 98754 client-cache.cc:170] Broken Connection, destroy client for blackbox:23020
I0221 18:50:09.183352 98754 statestore.cc:866] Unable to send heartbeat message to subscriber catalog-server@blackbox:26000, received error: RPC Error: Client for blackbox:23020 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:09.321941 99113 statestore.cc:531] Creating new topic: ''impala-request-queue' on behalf of subscriber: 'impalad@blackbox:22000
I0221 18:50:09.321975 99113 statestore.cc:531] Creating new topic: ''impala-membership' on behalf of subscriber: 'impalad@blackbox:22000
I0221 18:50:09.321985 99113 statestore.cc:539] Registering: impalad@blackbox:22000
I0221 18:50:09.322077 99113 statestore.cc:563] Subscriber 'impalad@blackbox:22000' registered (registration id: cc4438bb418a0740:36fd9500b1087583)
I0221 18:50:09.327312 98735 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.329787 98746 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.330566 98756 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.330704 98735 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.330724 98735 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.330935 98735 statestore.cc:866] Unable to send topic update message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.332270 98746 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.332281 98746 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.332422 98746 statestore.cc:866] Unable to send priority topic update message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.333046 98756 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:09.333056 98756 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.333191 98756 statestore.cc:866] Unable to send heartbeat message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala18THeartbeatResponseE, send: not done
I0221 18:50:09.435464 98747 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.437741 98747 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.437786 98747 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.437928 98747 statestore.cc:866] Unable to send priority topic update message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.540982 98745 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.543517 98745 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.543534 98745 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.543712 98745 statestore.cc:866] Unable to send priority topic update message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.647295 98751 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000
I0221 18:50:09.650938 98751 client-cache.h:304] RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.650951 98751 client-cache.cc:170] Broken Connection, destroy client for blackbox:23000
I0221 18:50:09.651127 98751 statestore.cc:866] Unable to send priority topic update message to subscriber impalad@blackbox:22000, received error: RPC Error: Client for blackbox:23000 hit an unexpected exception: authorize: cannot authorize peer, type: N6apache6thrift9transport13TSSLExceptionE, rpc: N6impala20TUpdateStateResponseE, send: not done
I0221 18:50:09.754004 98743 client-cache.cc:81] ReopenClient(): re-creating client for blackbox:23000