Last active
August 26, 2021 01:08
-
-
Save Ehco1996/61d416c2653bb43200959f63d566d6ed to your computer and use it in GitHub Desktop.
给 DM worker 的 log 脱敏
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import sys | |
| def redact_log(file_name): | |
| new_file_name = file_name + ".redacted.log" | |
| print("start to redact_log file=", file_name, "new file=", new_file_name) | |
| f1 = open(file=file_name, mode="r") | |
| f2 = open(file=new_file_name, mode="w") | |
| for line in f1.readlines(): | |
| if "argument" in line: | |
| logs = line.split(" ") | |
| need_write = [] | |
| for log in logs: | |
| if not "argument" in log: | |
| need_write.append(log) | |
| line = " ".join(need_write) | |
| f2.write(line) | |
| f1.close() | |
| f2.close() | |
| print("redact log file=", file_name, "finished") | |
| if __name__ == "__main__": | |
| file_name = sys.argv[1] | |
| redact_log(file_name=file_name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2021/08/26 09:06:08.610 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":2,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=false] | |
| [2021/08/26 09:06:08.620 +08:00] [WARN] [task.go:925] ["session variable 'time_zone' is overwritten by default UTC timezone."] [time_zone=+00:00] | |
| [2021/08/26 09:06:08.620 +08:00] [WARN] [task.go:925] ["session variable 'time_zone' is overwritten by default UTC timezone."] [time_zone=+00:00] | |
| [2021/08/26 09:06:08.631 +08:00] [INFO] [worker.go:694] ["start to create subtask"] [sourceID=mysql-replica-01] [task=test] | |
| [2021/08/26 09:06:08.634 +08:00] [INFO] [worker.go:426] ["subtask created"] [component="worker controller"] [config="{\"is-sharding\":true,\"shard-mode\":\"optimistic\",\"online-ddl\":false,\"online-ddl-scheme\":\"\",\"case-sensitive\":false,\"name\":\"test\",\"mode\":\"all\",\"ignore-checking-items\":null,\"source-id\":\"mysql-replica-01\",\"server-id\":429553783,\"flavor\":\"mysql\",\"meta-schema\":\"dm_meta\",\"heartbeat-update-interval\":1,\"heartbeat-report-interval\":10,\"enable-heartbeat\":false,\"meta\":null,\"timezone\":\"\",\"relay-dir\":\"relay-dir\",\"use-relay\":false,\"from\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"max-allowed-packet\":null,\"session\":{\"time_zone\":\"+00:00\"},\"security\":null},\"to\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"test\",\"max-allowed-packet\":null,\"session\":{\"tidb_txn_mode\":\"optimistic\",\"time_zone\":\"+00:00\"},\"security\":null},\"route-rules\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"tb*\",\"target-schema\":\"shardddl\",\"target-table\":\"tb\"},{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"\",\"target-schema\":\"shardddl\",\"target-table\":\"\"}],\"filter-rules\":[],\"mapping-rule\":[],\"expression-filter\":[],\"black-white-list\":null,\"block-allow-list\":{\"do-tables\":null,\"do-dbs\":[\"shardddl1\",\"shardddl2\"],\"ignore-tables\":null,\"ignore-dbs\":null},\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\",\"pool-size\":16,\"dir\":\"./dumped_data.test\",\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false,\"log-level\":\"\",\"log-file\":\"\",\"log-format\":\"\",\"log-rotate\":\"\",\"pprof-addr\":\"\",\"status-addr\":\"\",\"config-file\":\"\",\"clean-dump-file\":true,\"ansi-quotes\":false}"] | |
| [2021/08/26 09:06:08.636 +08:00] [INFO] [syncer.go:3403] ["use timezone"] [task=test] [unit="binlog replication"] [location=UTC] | |
| [2021/08/26 09:06:08.647 +08:00] [INFO] [dumpling.go:304] ["found upstream SQL mode"] [task=test] [unit=dump] ["SQL mode"=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION] | |
| [2021/08/26 09:06:08.651 +08:00] [INFO] [dumpling.go:67] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\"\",\"CsvDelimiter\":\"\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"HasTiKV\":false,\"ServerType\":0,\"ServerVersion\":null},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":67108864,\"StatementSize\":1000000,\"SessionParams\":{\"time_zone\":\"+00:00\"},\"Tables\":null}"] | |
| [2021/08/26 09:06:08.670 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE SCHEMA IF NOT EXISTS `dm_meta`"] [argument="[]"] | |
| [2021/08/26 09:06:08.680 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE TABLE IF NOT EXISTS `dm_meta`.`test_loader_checkpoint` (\n\t\tid char(32) NOT NULL,\n\t\tfilename varchar(255) NOT NULL,\n\t\tcp_schema varchar(128) NOT NULL,\n\t\tcp_table varchar(128) NOT NULL,\n\t\toffset bigint NOT NULL,\n\t\tend_pos bigint NOT NULL,\n\t\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n\t\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n\t\tUNIQUE KEY uk_id_f (id,filename)\n\t);\n"] [argument="[]"] | |
| [2021/08/26 09:06:08.690 +08:00] [DEBUG] [loader.go:911] ["all route rules"] [task=test] [unit=load] ["schema route rules"="{\"shardddl*\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"\",\"target-schema\":\"shardddl\",\"target-table\":\"\"}]}"] ["table route rules"="{\"shardddl*\":{\"tb*\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"tb*\",\"target-schema\":\"shardddl\",\"target-table\":\"tb\"}]}}"] | |
| [2021/08/26 09:06:08.694 +08:00] [INFO] [loader.go:540] ["loader's sql_mode is"] [task=test] [unit=load] [sqlmode=IGNORE_SPACE,ALLOW_INVALID_DATES,ONLY_FULL_GROUP_BY,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO] | |
| [2021/08/26 09:06:08.744 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit="binlog replication"] [scope=upstream] ["server info string"=5.7.34-log] | |
| [2021/08/26 09:06:08.744 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit="binlog replication"] [scope=upstream] [type=MySQL] | |
| [2021/08/26 09:06:08.744 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit="binlog replication"] [scope=upstream] [version=5.7.34-log] | |
| [2021/08/26 09:06:08.746 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit="binlog replication"] [scope=downstream] ["server info string"=5.7.25-TiDB-v5.0.1] | |
| [2021/08/26 09:06:08.746 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit="binlog replication"] [scope=downstream] [type=TiDB] | |
| [2021/08/26 09:06:08.746 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit="binlog replication"] [scope=downstream] [version=5.0.1] | |
| [2021/08/26 09:06:08.746 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [component=schema-tracker] [task=test] [query="SHOW VARIABLES LIKE 'tidb_skip_utf8_check'"] [argument="[]"] | |
| [2021/08/26 09:06:09.618 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:09.672 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:10.039 +08:00] [INFO] [common.go:110] ["no tables need to sync"] [schema=shardddl2] | |
| [2021/08/26 09:06:10.049 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="CREATE SCHEMA IF NOT EXISTS `dm_meta`"] [argument="[]"] | |
| [2021/08/26 09:06:10.050 +08:00] [INFO] [checkpoint.go:741] ["create checkpoint schema"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [statement="CREATE SCHEMA IF NOT EXISTS `dm_meta`"] | |
| [2021/08/26 09:06:10.050 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="CREATE TABLE IF NOT EXISTS `dm_meta`.`test_syncer_checkpoint` (\n\t\t\tid VARCHAR(32) NOT NULL,\n\t\t\tcp_schema VARCHAR(128) NOT NULL,\n\t\t\tcp_table VARCHAR(128) NOT NULL,\n\t\t\tbinlog_name VARCHAR(128),\n\t\t\tbinlog_pos INT UNSIGNED,\n\t\t\tbinlog_gtid TEXT,\n\t\t\texit_safe_binlog_name VARCHAR(128) DEFAULT '',\n\t\t\texit_safe_binlog_pos INT UNSIGNED DEFAULT 0,\n\t\t\texit_safe_binlog_gtid TEXT,\n\t\t\ttable_info JSON NOT NULL,\n\t\t\tis_global BOOLEAN,\n\t\t\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n\t\t\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n\t\t\tUNIQUE KEY uk_id_schema_table (id, cp_schema, cp_table)\n\t\t)"] [argument="[]"] | |
| [2021/08/26 09:06:10.062 +08:00] [INFO] [checkpoint.go:765] ["create checkpoint table"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [statements="[\"CREATE TABLE IF NOT EXISTS `dm_meta`.`test_syncer_checkpoint` (\\n\\t\\t\\tid VARCHAR(32) NOT NULL,\\n\\t\\t\\tcp_schema VARCHAR(128) NOT NULL,\\n\\t\\t\\tcp_table VARCHAR(128) NOT NULL,\\n\\t\\t\\tbinlog_name VARCHAR(128),\\n\\t\\t\\tbinlog_pos INT UNSIGNED,\\n\\t\\t\\tbinlog_gtid TEXT,\\n\\t\\t\\texit_safe_binlog_name VARCHAR(128) DEFAULT '',\\n\\t\\t\\texit_safe_binlog_pos INT UNSIGNED DEFAULT 0,\\n\\t\\t\\texit_safe_binlog_gtid TEXT,\\n\\t\\t\\ttable_info JSON NOT NULL,\\n\\t\\t\\tis_global BOOLEAN,\\n\\t\\t\\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\\n\\t\\t\\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\\n\\t\\t\\tUNIQUE KEY uk_id_schema_table (id, cp_schema, cp_table)\\n\\t\\t)\"]"] | |
| [2021/08/26 09:06:10.062 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit="binlog replication"] [query="SELECT cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global FROM `dm_meta`.`test_syncer_checkpoint` WHERE id = ?"] [argument="[mysql-replica-01]"] | |
| [2021/08/26 09:06:10.083 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit=load] [query="SELECT COUNT(id) FROM `dm_meta`.`test_loader_checkpoint` WHERE `id` = ?"] [argument="[mysql-replica-01]"] | |
| [2021/08/26 09:06:10.084 +08:00] [DEBUG] [checkpoint.go:446] ["checkpoint record"] [task=test] [unit=load] [component="remote checkpoint"] [count=0] | |
| [2021/08/26 09:06:10.085 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Dump] | |
| [2021/08/26 09:06:10.100 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit=dump] ["server info string"=5.7.34-log] | |
| [2021/08/26 09:06:10.100 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit=dump] [type=MySQL] | |
| [2021/08/26 09:06:10.101 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit=dump] [version=5.7.34-log] | |
| [2021/08/26 09:06:10.103 +08:00] [INFO] [dump.go:93] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\"\",\"CsvDelimiter\":\"\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"HasTiKV\":false,\"ServerType\":1,\"ServerVersion\":\"5.7.34-log\"},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":67108864,\"StatementSize\":1000000,\"SessionParams\":{\"time_zone\":\"+00:00\"},\"Tables\":null}"] | |
| [2021/08/26 09:06:10.230 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:10.573 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n {\n \"name\": \"test\",\n \"stage\": \"Running\",\n \"unit\": \"Dump\",\n \"result\": null,\n \"unresolvedDDLLockID\": \"\",\n \"dump\": {\n\n }\n }\n ]\n}"] | |
| [2021/08/26 09:06:11.267 +08:00] [DEBUG] [block_allow_list.go:16] ["start to filter tables"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.267 +08:00] [DEBUG] [block_allow_list.go:38] ["ignore table"] [task=test] [unit=dump] [tables="tables list\n\nschema adjust_gtid :[t1, ]schema dm_syncer_ignore_db :[dm_syncer_ignore_db_1, dm_syncer_ignore_db_2, ]schema metrics :[t1, ]schema mianbaoduo_dev :[affiliates, agreeProduct, apiLog, ban_user, banner, chat, chat_record, chat_report, checkCode, comments, copyright, copyright_pro, discounts, ditaner, download_handle, fans, files, geo, geo_china, groupbuy, hermit, hermit_result, inBucket, json_set, landingShow, mentions, mianbaoTag, orders, partner, partner_category, partner_invite, partner_manage, partner_product, partner_product_tag, payCut, payLog, pay_complaint, pixabay, prizes, productComplaint, productDraft, productView, product_complaint, product_diff_price, product_setting, products, productsLog, promotion, protected_pdf, qrcodeLog, qy_wx_live, recommend, reuse_discount, rewards, searchTag, search_word_filter, seckill_products, sendRecords, service_service, service_serviceorder, tags, tags_link, traffic_detail, updateLog, user, user_financial, wx_reply, ]schema Upper_Db_IGNORE :[ignore_table, ]schema dm_syncer :[dm_syncer_1, t1, ]schema downstream_more_column :[t1, ]schema only_dml :[t1, ]schema filters :[t1, ]schema full/mode :[t1, tb\\/1, ]schema openapi :[t, ]schema gtid :[t1, ]schema test_tcp_reader_db :[test_tcp_reader_table, ]schema dmctl_command :[t1, ]schema dm_syncer_do_db :[dm_syncer_do_db_1, dm_syncer_do_db_2, dm_syncer_do_db_3, ]schema sharding_seq_tmp :[t1, ]schema expr_filter :[t1, ]schema sys :[sys_config, ]schema information_schema :[CHARACTER_SETS, COLLATIONS, COLLATION_CHARACTER_SET_APPLICABILITY, COLUMNS, COLUMN_PRIVILEGES, ENGINES, EVENTS, FILES, GLOBAL_STATUS, GLOBAL_VARIABLES, KEY_COLUMN_USAGE, OPTIMIZER_TRACE, PARAMETERS, PARTITIONS, PLUGINS, PROCESSLIST, PROFILING, REFERENTIAL_CONSTRAINTS, ROUTINES, SCHEMATA, SCHEMA_PRIVILEGES, SESSION_STATUS, SESSION_VARIABLES, STATISTICS, TABLES, TABLESPACES, TABLE_CONSTRAINTS, TABLE_PRIVILEGES, TRIGGERS, USER_PRIVILEGES, VIEWS, INNODB_LOCKS, INNODB_TRX, INNODB_SYS_DATAFILES, INNODB_FT_CONFIG, INNODB_SYS_VIRTUAL, INNODB_CMP, INNODB_FT_BEING_DELETED, INNODB_CMP_RESET, INNODB_CMP_PER_INDEX, INNODB_CMPMEM_RESET, INNODB_FT_DELETED, INNODB_BUFFER_PAGE_LRU, INNODB_LOCK_WAITS, INNODB_TEMP_TABLE_INFO, INNODB_SYS_INDEXES, INNODB_SYS_TABLES, INNODB_SYS_FIELDS, INNODB_CMP_PER_INDEX_RESET, INNODB_BUFFER_PAGE, INNODB_FT_DEFAULT_STOPWORD, INNODB_FT_INDEX_TABLE, INNODB_FT_INDEX_CACHE, INNODB_SYS_TABLESPACES, INNODB_METRICS, INNODB_SYS_FOREIGN_COLS, INNODB_CMPMEM, INNODB_BUFFER_POOL_STATS, INNODB_SYS_COLUMNS, INNODB_SYS_FOREIGN, INNODB_SYS_TABLESTATS, ]schema drop_column_with_index :[t1, ]schema full_mode :[t1, ]schema load_task2 :[t1, ]schema sharding_seq :[t1, t2, ]schema load_interrupt :[t1, ]schema all_mode :[t1, ]schema mysql :[columns_priv, db, engine_cost, event, func, general_log, gtid_executed, help_category, help_keyword, help_relation, help_topic, innodb_index_stats, innodb_table_stats, ndb_binlog_index, plugin, proc, procs_priv, proxies_priv, server_cost, servers, slave_master_info, slave_relay_log_info, slave_worker_info, slow_log, tables_priv, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type, user, ]schema ignore_db :[ignore_table, ]schema dm_benchmark :[sbtest1, ]schema performance_schema :[accounts, cond_instances, events_stages_current, events_stages_history, events_stages_history_long, events_stages_summary_by_account_by_event_name, events_stages_summary_by_host_by_event_name, events_stages_summary_by_thread_by_event_name, events_stages_summary_by_user_by_event_name, events_stages_summary_global_by_event_name, events_statements_current, events_statements_history, events_statements_history_long, events_statements_summary_by_account_by_event_name, events_statements_summary_by_digest, events_statements_summary_by_host_by_event_name, events_statements_summary_by_program, events_statements_summary_by_thread_by_event_name, events_statements_summary_by_user_by_event_name, events_statements_summary_global_by_event_name, events_transactions_current, events_transactions_history, events_transactions_history_long, events_transactions_summary_by_account_by_event_name, events_transactions_summary_by_host_by_event_name, events_transactions_summary_by_thread_by_event_name, events_transactions_summary_by_user_by_event_name, events_transactions_summary_global_by_event_name, events_waits_current, events_waits_history, events_waits_history_long, events_waits_summary_by_account_by_event_name, events_waits_summary_by_host_by_event_name, events_waits_summary_by_instance, events_waits_summary_by_thread_by_event_name, events_waits_summary_by_user_by_event_name, events_waits_summary_global_by_event_name, file_instances, file_summary_by_event_name, file_summary_by_instance, global_status, global_variables, host_cache, hosts, memory_summary_by_account_by_event_name, memory_summary_by_host_by_event_name, memory_summary_by_thread_by_event_name, memory_summary_by_user_by_event_name, memory_summary_global_by_event_name, metadata_locks, mutex_instances, objects_summary_global_by_type, performance_timers, prepared_statements_instances, replication_applier_configuration, replication_applier_status, replication_applier_status_by_coordinator, replication_applier_status_by_worker, replication_connection_configuration, replication_connection_status, replication_group_member_stats, replication_group_members, rwlock_instances, session_account_connect_attrs, session_connect_attrs, session_status, session_variables, setup_actors, setup_consumers, setup_instruments, setup_objects, setup_timers, socket_instances, socket_summary_by_event_name, socket_summary_by_instance, status_by_account, status_by_host, status_by_thread, status_by_user, table_handles, table_io_waits_summary_by_index_usage, table_io_waits_summary_by_table, table_lock_waits_summary_by_table, threads, user_variables_by_thread, users, variables_by_thread, ]schema sharding_seq_opt :[t1, t2, ]schema DM_HEARTBEAT :[HEARTBEAT, ]schema fake_rotate_event :[t1, ]schema load_task4 :[t1, ]schema online_ddl :[_ignore_old, ignore, t1, t2, ]schema Upper_DB1 :[Do_Table1, ]schema test :[_loader_checkpoint, syncer_ut_onlineddl, ]schema tls :[t, ]schema load_task1 :[t1, ]schema Upper_DB :[Do_Table, Do_table_ignore, lower_Table1, lower_table, ]schema dmctl :[t_1, t_2, tb_1, tb_2, ]"] | |
| [2021/08/26 09:06:11.268 +08:00] [DEBUG] [dump.go:1207] ["no need to build region info because database is not TiDB 3.x"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.272 +08:00] [INFO] [dump.go:191] ["All the dumping transactions have started. Start to unlock tables"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.272 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of dababase 'shardddl2'"] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of dababase 'shardddl1'"] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:310] ["start dumping table..."] [task=test] [unit=dump] [database=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl2] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl2] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of table 'shardddl1'.'tb1'"] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.275 +08:00] [INFO] [dump.go:490] ["get estimated rows count"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [estimateCount=1] | |
| [2021/08/26 09:06:11.275 +08:00] [WARN] [dump.go:496] ["skip concurrent dump due to estimate count < rows"] [task=test] [unit=dump] ["estimate count"=1] [conf.rows=200000] [database=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="data of table 'shardddl1'.'tb1'(0/1)"] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [writer.go:168] ["trying to dump table chunk"] [task=test] [unit=dump] [retryTime=1] [db=shardddl1] [table=tb1] [chunkIndex=0] [] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [ir_impl.go:207] ["try to start tableData"] [task=test] [unit=dump] [query="SELECT `a`,`b` FROM `shardddl1`.`tb1` ORDER BY `a`"] | |
| [2021/08/26 09:06:11.276 +08:00] [WARN] [writer.go:230] ["no data written in table chunk"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [chunkIdx=0] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [dump.go:274] ["finish dumping table data task"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [chunkIdx=0] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [status.go:27] ["stopping log progress"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.276 +08:00] [INFO] [collector.go:232] ["backup success summary"] [task=test] [unit=dump] [total-ranges=4] [ranges-succeed=4] [ranges-failed=0] [total-take=3.797736ms] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/metadata] | |
| [2021/08/26 09:06:11.276 +08:00] [INFO] [dumpling.go:140] ["dump data finished"] [task=test] [unit=dump] ["cost time"=1.187529872s] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Dump] [stage=Finished] [status={}] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:303] ["switching to next unit"] [subtask=test] [unit=Dump] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Load] | |
| [2021/08/26 09:06:11.279 +08:00] [INFO] [loader.go:1551] ["put load worker in etcd"] [task=test] [unit=load] [task=test] [source=mysql-replica-01] [worker=worker1] | |
| [2021/08/26 09:06:11.279 +08:00] [DEBUG] [loader.go:1073] ["collected files"] [task=test] [unit=load] [files="{\"metadata\":{},\"shardddl1-schema-create.sql\":{},\"shardddl1.tb1-schema.sql\":{},\"shardddl2-schema-create.sql\":{}}"] | |
| [2021/08/26 09:06:11.279 +08:00] [INFO] [loader.go:1046] ["prepare loading"] [task=test] [unit=load] ["cost time"=477.757µs] | |
| [2021/08/26 09:06:11.280 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit=load] [query="SELECT `filename`,`cp_schema`,`cp_table`,`offset`,`end_pos` from `dm_meta`.`test_loader_checkpoint` where `id`=?"] [argument="[mysql-replica-01]"] | |
| [2021/08/26 09:06:11.281 +08:00] [INFO] [checkpoint.go:168] ["load checkpoint"] [task=test] [unit=load] [component="remote checkpoint"] ["cost time"=1.052436ms] | |
| [2021/08/26 09:06:11.281 +08:00] [INFO] [checkpoint.go:301] ["calculate checkpoint finished."] [task=test] [unit=load] [component="remote checkpoint"] ["finished tables"={}] | |
| [2021/08/26 09:06:11.288 +08:00] [INFO] [loader.go:1330] ["start to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.288 +08:00] [INFO] [loader.go:1330] ["start to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.288 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] | |
| [2021/08/26 09:06:11.288 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] | |
| [2021/08/26 09:06:11.289 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [argument="[]"] | |
| [2021/08/26 09:06:11.289 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [argument="[]"] | |
| [2021/08/26 09:06:11.290 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [argument="[]"] [error="Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [argument="[]"] [error="Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;]"] [arguments="[]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;, RawCause: Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [INFO] [loader.go:1104] ["database already exists, skip it"] [task=test] [unit=load] ["db schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.290 +08:00] [INFO] [loader.go:1335] ["finish to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.291 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;]"] [arguments="[]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;, RawCause: Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.291 +08:00] [INFO] [loader.go:1104] ["database already exists, skip it"] [task=test] [unit=load] ["db schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.291 +08:00] [INFO] [loader.go:1335] ["finish to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.296 +08:00] [INFO] [loader.go:1370] ["start to create table"] [task=test] [unit=load] ["table file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.296 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"] | |
| [2021/08/26 09:06:11.296 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="USE `shardddl`;"] [argument="[]"] | |
| [2021/08/26 09:06:11.297 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"] [argument="[]"] | |
| [2021/08/26 09:06:11.297 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"] [argument="[]"] [error="Error 1050: Table 'shardddl.tb' already exists"] | |
| [2021/08/26 09:06:11.298 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[USE `shardddl`; CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;]"] [arguments="[]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;, RawCause: Error 1050: Table 'shardddl.tb' already exists"] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1121] ["table already exists, skip it"] [task=test] [unit=load] ["table schema file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1375] ["finish to create table"] [task=test] [unit=load] ["table file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1422] ["finish to create tables"] [task=test] [unit=load] ["cost time"=15.998743ms] | |
| [2021/08/26 09:06:11.298 +08:00] [DEBUG] [loader.go:1429] ["restoring table data"] [task=test] [unit=load] [schema=shardddl1] [table=tb1] ["data files"={}] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1467] ["all data files have been dispatched, waiting for them finished"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:739] ["all data files have been finished"] [task=test] [unit=load] ["cost time"=17.58735ms] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [loader.go:1517] ["clean dump files"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [status.go:69] ["progress status of load"] [task=test] [unit=load] [finished_bytes=0] [total_bytes=0] [total_file_count=3] [progress="100.00 %"] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Load] [stage=Finished] [status="{\"progress\":\"100.00 %\",\"metaBinlog\":\"(mysql-bin.000001, 1094)\",\"metaBinlogGTID\":\"b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5\"}"] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [loader.go:787] ["stop importing data process"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [DEBUG] [loader.go:791] ["all workers have been closed"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [DEBUG] [loader.go:794] ["all loader's go-routines have been closed"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:303] ["switching to next unit"] [subtask=test] [unit=Load] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:11.303 +08:00] [INFO] [checkpoint.go:960] ["loaded checkpoints from meta"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] ["global checkpoint"="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5(flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:11.304 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1094 b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5 0 null true]"] | |
| [2021/08/26 09:06:11.305 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5(flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:11.306 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:11.308 +08:00] [INFO] [syncer.go:3548] ["delete load worker in etcd for all mode"] [task=test] [unit="binlog replication"] [task=test] [source=mysql-replica-01] | |
| [2021/08/26 09:06:11.308 +08:00] [INFO] [syncer.go:1530] ["try to remove all dump files"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:11.310 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5"] | |
| [2021/08/26 09:06:11.318 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=161] | |
| [2021/08/26 09:06:11.319 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:11.319 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:11.320 +08:00] [INFO] [mode.go:57] ["enable safe-mode because of task initialization"] [task=test] [unit="binlog replication"] ["duration in seconds"=60] | |
| [2021/08/26 09:06:11.362 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:11.363 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.352 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:12.481 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1159,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.482 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1227,\"Flags\":8}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1281,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1325,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1325), gtid-set: "] ["raw event data"="[[1,\"aaa\"]]"] | |
| [2021/08/26 09:06:12.502 +08:00] [INFO] [optimist.go:175] ["no init schema exists"] [task=test] [unit="binlog replication"] [component="shard DDL optimist"] [schema=shardddl] [table=tb] [revision=19] | |
| [2021/08/26 09:06:12.504 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit="binlog replication"] [query="SHOW CREATE TABLE `shardddl`.`tb`"] [argument="[]"] | |
| [2021/08/26 09:06:12.507 +08:00] [DEBUG] [syncer.go:809] ["reverse-synchronized table schema"] [task=test] [unit="binlog replication"] [origSchema=shardddl1] [origTable=tb1] [renamedSchema=shardddl] [renamedTable=tb] [sql="CREATE TABLE IF NOT EXISTS `shardddl1`.`tb1` (`a` INT(11) NOT NULL,`b` VARCHAR(10) DEFAULT NULL,PRIMARY KEY(`a`) CLUSTERED) ENGINE = InnoDB DEFAULT CHARACTER SET = LATIN1 DEFAULT COLLATE = LATIN1_BIN"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=1`shardddl`.`tb`] [keys="[1`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=3] [key=1`shardddl`.`tb`] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1325), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1356,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1227), gtid-set: "] [location="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:12.526 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:12.536 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[1 aaa]"] | |
| [2021/08/26 09:06:12.586 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1421,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.586 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":115,\"LogPos\":1536,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.593 +08:00] [INFO] [syncer.go:2314] [task=test] [unit="binlog replication"] [event=query] [statement="alter table shardddl1.tb1 change a c int"] [schema=] ["last location"="position: (mysql-bin.000001, 1356), gtid-set: "] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.594 +08:00] [INFO] [syncer.go:2326] ["resolve sql"] [task=test] [unit="binlog replication"] [event=query] ["raw statement"="alter table shardddl1.tb1 change a c int"] [statements="[\"ALTER TABLE `shardddl1`.`tb1` CHANGE COLUMN `a` `c` INT\"]"] [schema=] ["last location"="position: (mysql-bin.000001, 1536), gtid-set: "] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [INFO] [syncer.go:2425] ["prepare to handle ddls"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT\"]"] ["raw statement"="alter table shardddl1.tb1 change a c int"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:12.597 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1356 0 null true]"] | |
| [2021/08/26 09:06:12.598 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 shardddl1 tb1 mysql-bin.000001 1325 0 {\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handl..."] | |
| [2021/08/26 09:06:12.599 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:12.600 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:12.619 +08:00] [INFO] [optimist.go:182] ["putted a shard DDL info into etcd"] [task=test] [unit="binlog replication"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT\"],\"table-info-before\":{\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_cst_id\":0,\"update_timestamp\":427278984154185728,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0},\"table-info-after\":[{\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_cst_id\":0,\"update_timestamp\":427278984178302976,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0}],\"ignore-conflict\":false}"] | |
| [2021/08/26 09:06:12.643 +08:00] [INFO] [optimist.go:187] ["got a shard DDL lock operation"] [task=test] [unit="binlog replication"] [operation="{\"id\":\"test-`shardddl`.`tb`\",\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"ddls\":[],\"conflict-stage\":\"detected\",\"conflict-message\":\"[code=11111:class=functional:scope=internal:level=medium], Message: fail to try sync the optimistic shard ddl lock test-`shardddl`.`tb`: there will be conflicts if DDLs [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] are applied to the downstream. old table info: CREATE TABLE `tbl`(`a` INT(11) NOT NULL, `b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, PRIMARY KEY (`a`)) CHARSET LATIN1 COLLATE LATIN1_BIN, new table info: CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN, RawCause: at tuple index 2: at map key \\\"a\\\": column with no default value cannot be missing, Workaround: Please use `show-ddl-locks` command for more details.\",\"done\":false,\"cols\":[]}"] | |
| [2021/08/26 09:06:12.645 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1356 mysql-bin.000001 1536 null true]"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:12.655 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 1325), gtid-set: (flushed position: (mysql-bin.000001, 1325), gtid-set: )"] | |
| [2021/08/26 09:06:12.655 +08:00] [INFO] [checkpoint.go:694] ["rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [from="position: (mysql-bin.000001, 1325), gtid-set: "] [to="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:12.674 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:12.707 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Sync] [stage=Paused] [status="{\"totalEvents\":1,\"syncerBinlog\":\"(mysql-bin.000001, 1356)\",\"binlogType\":\"remote\"}"] | |
| [2021/08/26 09:06:12.707 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":36062,\"ErrClass\":\"sync-unit\",\"ErrScope\":\"internal\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (mysql-bin.000001, 1421), gtid-set: ], endLocation: [position: (mysql-bin.000001, 1536), gtid-set: ], origin SQL: [alter table shardddl1.tb1 change a c int]: fail to handle shard ddl [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] in optimistic mode, because schema conflict detected, conflict error: [code=11111:class=functional:scope=internal:level=medium], Message: fail to try sync the optimistic shard ddl lock test-`shardddl`.`tb`: there will be conflicts if DDLs [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] are applied to the downstream. old table info: CREATE TABLE `tbl`(`a` INT(11) NOT NULL, `b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, PRIMARY KEY (`a`)) CHARSET LATIN1 COLLATE LATIN1_BIN, new table info: CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN, RawCause: at tuple index 2: at map key \\\"a\\\": column with no default value cannot be missing, Workaround: Please use `show-ddl-locks` command for more details.\",\"Workaround\":\"Please use show-ddl-locks command for more details.\"}"] | |
| [2021/08/26 09:06:13.346 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:13.501 +08:00] [INFO] [server.go:917] [request=HandleError] [payload="op:Skip task:\"test\" "] | |
| [2021/08/26 09:06:13.502 +08:00] [INFO] [operator.go:93] ["set a new operator"] [task=test] [unit="binlog replication"] [component="error operator holder"] [position="(mysql-bin.000001, 1421)"] ["new operator"="uuid: e7119583-0894-4a95-9ce6-4cf90d2e8eb7, op: Skip, events: "] | |
| [2021/08/26 09:06:13.502 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:13.503 +08:00] [INFO] [subtask.go:525] ["resume with unit"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:13.537 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:13.540 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=170] | |
| [2021/08/26 09:06:13.541 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:13.542 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:13.542 +08:00] [INFO] [mode.go:37] ["enable safe-mode for safe mode exit point, will exit at"] [task=test] [unit="binlog replication"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1421,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.546 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":115,\"LogPos\":1536,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.546 +08:00] [INFO] [operator.go:167] ["match and apply a operator"] [task=test] [unit="binlog replication"] [component="error operator holder"] [startlocation="position: (mysql-bin.000001, 1421), gtid-set: "] [endlocation="position: (mysql-bin.000001, 1536), gtid-set: "] [operator="uuid: e7119583-0894-4a95-9ce6-4cf90d2e8eb7, op: Skip, events: "] | |
| [2021/08/26 09:06:13.546 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.546 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 1536), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:13.548 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1536 mysql-bin.000001 1536 null true]"] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1536), gtid-set: (flushed position: (mysql-bin.000001, 1536), gtid-set: )"] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1872] ["flush jobs when handle-error skip"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.550 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1601,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.554 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1669,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.554 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1536 0 null true]"] | |
| [2021/08/26 09:06:13.555 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1723,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1767,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["raw event data"="[[4,\"ddd\"]]"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=4`shardddl`.`tb`] [keys="[4`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=5] [key=4`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1798,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1669), gtid-set: "] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1863,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.558 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1931,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.558 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1985,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.560 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2029,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1767), gtid-set: "] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["raw event data"="[[7,\"ggg\"]]"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=7`shardddl`.`tb`] [keys="[7`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=7] [key=7`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2060,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1931), gtid-set: "] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2125,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2193,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":2247,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2291,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2029), gtid-set: "] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["raw event data"="[[10,\"jjj\"]]"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=10`shardddl`.`tb`] [keys="[10`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=14] [key=10`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2322,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2193), gtid-set: "] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.567 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[4 ddd]"] | |
| [2021/08/26 09:06:13.568 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[4 ddd]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.568 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] [arguments="[[4 ddd]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.568 +08:00] [INFO] [syncer.go:1464] ["received subtask's done"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.572 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[7 ggg]"] | |
| [2021/08/26 09:06:13.573 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[7 ggg]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.573 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] [arguments="[[7 ggg]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.574 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[10 jjj]"] | |
| [2021/08/26 09:06:13.574 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] [argument="[10 jjj]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.575 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] [arguments="[[10 jjj]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:1470] ["the last job is transaction end, done directly"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:1762] ["binlog replication main routine quit(context canceled)!"] [task=test] [unit="binlog replication"] ["last location"="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.576 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1536 mysql-bin.000001 2322 null true]"] | |
| [2021/08/26 09:06:13.577 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.579 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 2291), gtid-set: (flushed position: (mysql-bin.000001, 1325), gtid-set: )"] | |
| [2021/08/26 09:06:13.579 +08:00] [INFO] [checkpoint.go:694] ["rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [from="position: (mysql-bin.000001, 2291), gtid-set: "] [to="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:13.614 +08:00] [WARN] [syncer.go:699] ["something wrong with rollback global checkpoint"] [task=test] [unit="binlog replication"] ["previous position"="position: (mysql-bin.000001, 2322), gtid-set: "] ["current position"="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.615 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Sync] [stage=Paused] [status="{\"totalEvents\":1,\"syncerBinlog\":\"(mysql-bin.000001, 1536)\",\"binlogType\":\"remote\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 1767), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 2029), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 2291), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.632 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:13.747 +08:00] [INFO] [server.go:799] [request=OperateSchema] [payload="op:SetSchema task:\"test\" source:\"mysql-replica-01\" database:\"shardddl1\" table:\"tb1\" schema:\"CREATE TABLE `tb1` ( `c` int NOT NULL, `b` varchar(10) DEFAULT NULL, PRIMARY KEY (`c`)) ENGINE=InnoDB DEFAULT CHARSET=latin1\\n\" flush:true sync:true "] | |
| [2021/08/26 09:06:13.808 +08:00] [INFO] [schema.go:94] ["flush table info"] ["table info"="CREATE TABLE `shardddl1`.`tb1` (`c` INT NOT NULL,`b` VARCHAR(10) DEFAULT NULL,PRIMARY KEY(`c`)) ENGINE = InnoDB DEFAULT CHARACTER SET = LATIN1"] | |
| [2021/08/26 09:06:13.809 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 shardddl1 tb1 mysql-bin.000001 1325 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\"..."] | |
| [2021/08/26 09:06:13.815 +08:00] [INFO] [schema.go:110] ["sync info with operate-schema"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"\"],\"table-before\":\"CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN\",\"table-after\":\"CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN\",\"is-deleted\":false,\"version\":0,\"revision\":0,\"ignore-conflict\":true}"] | |
| [2021/08/26 09:06:14.108 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":2,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=false] | |
| [2021/08/26 09:06:14.109 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [worker.go:473] ["resume sub task"] [component="worker controller"] [task=test] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [subtask.go:525] ["resume with unit"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:14.147 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:14.149 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=174] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:37] ["enable safe-mode for safe mode exit point, will exit at"] [task=test] [unit="binlog replication"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1601,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1669,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1723,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1767,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["raw event data"="[[4,\"ddd\"]]"] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=4`shardddl`.`tb`] [keys="[4`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=5] [key=4`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1798,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1669), gtid-set: "] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1863,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1931,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1985,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2029,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1767), gtid-set: "] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["raw event data"="[[7,\"ggg\"]]"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=7`shardddl`.`tb`] [keys="[7`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=7] [key=7`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2060,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1931), gtid-set: "] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2125,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2193,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":2247,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2291,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2029), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["raw event data"="[[10,\"jjj\"]]"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=10`shardddl`.`tb`] [keys="[10`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=14] [key=10`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2322,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2193), gtid-set: "] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.164 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] [argument="[4 ddd]"] | |
| [2021/08/26 09:06:14.166 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] [argument="[7 ggg]"] | |
| [2021/08/26 09:06:14.168 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] [argument="[10 jjj]"] | |
| [2021/08/26 09:06:14.220 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:14.603 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:15.308 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:15.548 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2387,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.549 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":117,\"LogPos\":2504,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.550 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 1536 0 null true]"] | |
| [2021/08/26 09:06:15.553 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:15.554 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n {\n \"name\": \"test\",\n \"stage\": \"Running\",\n \"unit\": \"Sync\",\n \"result\": null,\n \"unresolvedDDLLockID\": \"\",\n \"sync\": {\n \"totalEvents\": \"4\",\n \"totalTps\": \"0\",\n \"recentTps\": \"0\",\n \"masterBinlog\": \"\",\n \"masterBinlogGtid\": \"\",\n \"syncerBinlog\": \"(mysql-bin.000001, 1536)\",\n \"syncerBinlogGtid\": \"\",\n \"blockingDDLs\": [\n ],\n \"unresolvedGroups\": [\n ],\n \"synced\": false,\n \"binlogType\": \"remote\",\n \"secondsBehindMaster\": \"0\"\n }\n }\n ]\n}"] | |
| [2021/08/26 09:06:15.555 +08:00] [INFO] [syncer.go:2314] [task=test] [unit="binlog replication"] [event=query] [statement="alter table shardddl1.tb1 add column d int"] [schema=] ["last location"="position: (mysql-bin.000001, 2322), gtid-set: "] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.556 +08:00] [INFO] [syncer.go:2326] ["resolve sql"] [task=test] [unit="binlog replication"] [event=query] ["raw statement"="alter table shardddl1.tb1 add column d int"] [statements="[\"ALTER TABLE `shardddl1`.`tb1` ADD COLUMN `d` INT\"]"] [schema=] ["last location"="position: (mysql-bin.000001, 2504), gtid-set: "] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2291), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [INFO] [syncer.go:2425] ["prepare to handle ddls"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 2322), gtid-set: (flushed position: (mysql-bin.000001, 1536), gtid-set: )"] | |
| [2021/08/26 09:06:15.559 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 2322 0 null true]"] | |
| [2021/08/26 09:06:15.560 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 shardddl1 tb1 mysql-bin.000001 2291 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\"..."] | |
| [2021/08/26 09:06:15.561 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 2322), gtid-set: (flushed position: (mysql-bin.000001, 2322), gtid-set: )"] | |
| [2021/08/26 09:06:15.561 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:15.596 +08:00] [INFO] [optimist.go:182] ["putted a shard DDL info into etcd"] [task=test] [unit="binlog replication"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"],\"table-info-before\":{\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\":\"\",\"L\":\"\"},\"idx_cols\":[{\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"length\":-1}],\"state\":5,\"comment\":\"\",\"index_type\":1,\"is_unique\":true,\"is_primary\":true,\"is_invisible\":false,\"is_global\":false}],\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":false,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":1,\"max_cst_id\":0,\"update_timestamp\":427278984489992196,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0},\"table-info-after\":[{\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\":\"\",\"L\":\"\"},\"idx_cols\":[{\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"length\":-1}],\"state\":5,\"comment\":\"\",\"index_type\":1,\"is_unique\":true,\"is_primary\":true,\"is_invisible\":false,\"is_global\":false}],\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":false,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":3,\"max_idx_id\":1,\"max_cst_id\":0,\"update_timestamp\":427278984958443520,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0}],\"ignore-conflict\":false}"] | |
| [2021/08/26 09:06:15.616 +08:00] [INFO] [optimist.go:187] ["got a shard DDL lock operation"] [task=test] [unit="binlog replication"] [operation="{\"id\":\"test-`shardddl`.`tb`\",\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"],\"conflict-stage\":\"none\",\"conflict-message\":\"\",\"done\":false,\"cols\":[]}"] | |
| [2021/08/26 09:06:15.619 +08:00] [INFO] [optimist.go:197] ["start to handle ddls in optimistic shard mode"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.620 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT"] [argument="[]"] | |
| [2021/08/26 09:06:15.650 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.650 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.651 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 2504 0 null true]"] | |
| [2021/08/26 09:06:15.651 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 shardddl1 tb1 mysql-bin.000001 2504 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_defau..."] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 2504), gtid-set: (flushed position: (mysql-bin.000001, 2504), gtid-set: )"] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [optimist.go:236] ["finish to handle ddls in optimistic shard mode"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.665 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2569,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2637,\"Flags\":8}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":2692,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":2740,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2740), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2740), gtid-set: "] ["raw event data"="[[13,\"mmm\",13]]"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=13`shardddl`.`tb`] [keys="[13`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=12] [key=13`shardddl`.`tb`] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2740), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2771,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2637), gtid-set: "] [location="position: (mysql-bin.000001, 2771), gtid-set: "] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2771), gtid-set: "] | |
| [2021/08/26 09:06:15.677 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:15.678 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] [argument="[13 mmm 13]"] | |
| [2021/08/26 09:06:15.837 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2836,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2904,\"Flags\":8}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":2959,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":3007,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3007), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2740), gtid-set: "] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 3007), gtid-set: "] ["raw event data"="[[16,\"ppp\",16]]"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=16`shardddl`.`tb`] [keys="[16`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=10] [key=16`shardddl`.`tb`] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3007), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":3038,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2904), gtid-set: "] [location="position: (mysql-bin.000001, 3038), gtid-set: "] | |
| [2021/08/26 09:06:15.840 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3038), gtid-set: "] | |
| [2021/08/26 09:06:15.850 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] [argument="[16 ppp 16]"] | |
| [2021/08/26 09:06:16.089 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":3103,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":3171,\"Flags\":8}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":3226,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":3274,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3274), gtid-set: "] ["old location"="position: (mysql-bin.000001, 3007), gtid-set: "] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 3274), gtid-set: "] ["raw event data"="[[19,\"sss\",19]]"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=19`shardddl`.`tb`] [keys="[19`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=0] [key=19`shardddl`.`tb`] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3274), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":3305,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 3171), gtid-set: "] [location="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:16.092 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:16.102 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] [argument="[19 sss 19]"] | |
| [2021/08/26 09:06:16.517 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:17.116 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":0,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=true] | |
| [2021/08/26 09:06:17.116 +08:00] [INFO] [worker.go:466] ["stop sub task"] [component="worker controller"] [task=test] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [subtask.go:463] [closing] [subtask=test] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [subtask.go:360] ["closing unit process"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1464] ["received subtask's done"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1470] ["the last job is transaction end, done directly"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1762] ["binlog replication main routine quit(context canceled)!"] [task=test] [unit="binlog replication"] ["last location"="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:17.118 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:17.119 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 mysql-bin.000001 3305 mysql-bin.000001 3305 null true]"] | |
| [2021/08/26 09:06:17.120 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] [argument="[mysql-replica-01 shardddl1 tb1 mysql-bin.000001 3274 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_defau..."] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 3305), gtid-set: (flushed position: (mysql-bin.000001, 3305), gtid-set: )"] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.123 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 3274), gtid-set: (flushed position: (mysql-bin.000001, 3274), gtid-set: )"] | |
| [2021/08/26 09:06:18.687 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:20.554 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] | |
| [2021/08/26 09:06:21.693 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:24.695 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:25.559 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] | |
| [2021/08/26 09:06:27.703 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:30.558 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] | |
| [2021/08/26 09:06:30.710 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:33.717 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:35.555 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] | |
| [2021/08/26 09:06:36.721 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:39.726 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:40.554 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2021/08/26 09:06:08.610 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":2,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=false] | |
| [2021/08/26 09:06:08.620 +08:00] [WARN] [task.go:925] ["session variable 'time_zone' is overwritten by default UTC timezone."] [time_zone=+00:00] | |
| [2021/08/26 09:06:08.620 +08:00] [WARN] [task.go:925] ["session variable 'time_zone' is overwritten by default UTC timezone."] [time_zone=+00:00] | |
| [2021/08/26 09:06:08.631 +08:00] [INFO] [worker.go:694] ["start to create subtask"] [sourceID=mysql-replica-01] [task=test] | |
| [2021/08/26 09:06:08.634 +08:00] [INFO] [worker.go:426] ["subtask created"] [component="worker controller"] [config="{\"is-sharding\":true,\"shard-mode\":\"optimistic\",\"online-ddl\":false,\"online-ddl-scheme\":\"\",\"case-sensitive\":false,\"name\":\"test\",\"mode\":\"all\",\"ignore-checking-items\":null,\"source-id\":\"mysql-replica-01\",\"server-id\":429553783,\"flavor\":\"mysql\",\"meta-schema\":\"dm_meta\",\"heartbeat-update-interval\":1,\"heartbeat-report-interval\":10,\"enable-heartbeat\":false,\"meta\":null,\"timezone\":\"\",\"relay-dir\":\"relay-dir\",\"use-relay\":false,\"from\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"max-allowed-packet\":null,\"session\":{\"time_zone\":\"+00:00\"},\"security\":null},\"to\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"test\",\"max-allowed-packet\":null,\"session\":{\"tidb_txn_mode\":\"optimistic\",\"time_zone\":\"+00:00\"},\"security\":null},\"route-rules\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"tb*\",\"target-schema\":\"shardddl\",\"target-table\":\"tb\"},{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"\",\"target-schema\":\"shardddl\",\"target-table\":\"\"}],\"filter-rules\":[],\"mapping-rule\":[],\"expression-filter\":[],\"black-white-list\":null,\"block-allow-list\":{\"do-tables\":null,\"do-dbs\":[\"shardddl1\",\"shardddl2\"],\"ignore-tables\":null,\"ignore-dbs\":null},\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\",\"pool-size\":16,\"dir\":\"./dumped_data.test\",\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false,\"log-level\":\"\",\"log-file\":\"\",\"log-format\":\"\",\"log-rotate\":\"\",\"pprof-addr\":\"\",\"status-addr\":\"\",\"config-file\":\"\",\"clean-dump-file\":true,\"ansi-quotes\":false}"] | |
| [2021/08/26 09:06:08.636 +08:00] [INFO] [syncer.go:3403] ["use timezone"] [task=test] [unit="binlog replication"] [location=UTC] | |
| [2021/08/26 09:06:08.647 +08:00] [INFO] [dumpling.go:304] ["found upstream SQL mode"] [task=test] [unit=dump] ["SQL mode"=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION] | |
| [2021/08/26 09:06:08.651 +08:00] [INFO] [dumpling.go:67] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\"\",\"CsvDelimiter\":\"\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"HasTiKV\":false,\"ServerType\":0,\"ServerVersion\":null},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":67108864,\"StatementSize\":1000000,\"SessionParams\":{\"time_zone\":\"+00:00\"},\"Tables\":null}"] | |
| [2021/08/26 09:06:08.670 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE SCHEMA IF NOT EXISTS `dm_meta`"][2021/08/26 09:06:08.680 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE TABLE IF NOT EXISTS `dm_meta`.`test_loader_checkpoint` (\n\t\tid char(32) NOT NULL,\n\t\tfilename varchar(255) NOT NULL,\n\t\tcp_schema varchar(128) NOT NULL,\n\t\tcp_table varchar(128) NOT NULL,\n\t\toffset bigint NOT NULL,\n\t\tend_pos bigint NOT NULL,\n\t\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n\t\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n\t\tUNIQUE KEY uk_id_f (id,filename)\n\t);\n"][2021/08/26 09:06:08.690 +08:00] [DEBUG] [loader.go:911] ["all route rules"] [task=test] [unit=load] ["schema route rules"="{\"shardddl*\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"\",\"target-schema\":\"shardddl\",\"target-table\":\"\"}]}"] ["table route rules"="{\"shardddl*\":{\"tb*\":[{\"schema-pattern\":\"shardddl*\",\"table-pattern\":\"tb*\",\"target-schema\":\"shardddl\",\"target-table\":\"tb\"}]}}"] | |
| [2021/08/26 09:06:08.694 +08:00] [INFO] [loader.go:540] ["loader's sql_mode is"] [task=test] [unit=load] [sqlmode=IGNORE_SPACE,ALLOW_INVALID_DATES,ONLY_FULL_GROUP_BY,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO] | |
| [2021/08/26 09:06:08.744 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit="binlog replication"] [scope=upstream] ["server info string"=5.7.34-log] | |
| [2021/08/26 09:06:08.744 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit="binlog replication"] [scope=upstream] [type=MySQL] | |
| [2021/08/26 09:06:08.744 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit="binlog replication"] [scope=upstream] [version=5.7.34-log] | |
| [2021/08/26 09:06:08.746 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit="binlog replication"] [scope=downstream] ["server info string"=5.7.25-TiDB-v5.0.1] | |
| [2021/08/26 09:06:08.746 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit="binlog replication"] [scope=downstream] [type=TiDB] | |
| [2021/08/26 09:06:08.746 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit="binlog replication"] [scope=downstream] [version=5.0.1] | |
| [2021/08/26 09:06:08.746 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [component=schema-tracker] [task=test] [query="SHOW VARIABLES LIKE 'tidb_skip_utf8_check'"][2021/08/26 09:06:09.618 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:09.672 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:10.039 +08:00] [INFO] [common.go:110] ["no tables need to sync"] [schema=shardddl2] | |
| [2021/08/26 09:06:10.049 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="CREATE SCHEMA IF NOT EXISTS `dm_meta`"][2021/08/26 09:06:10.050 +08:00] [INFO] [checkpoint.go:741] ["create checkpoint schema"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [statement="CREATE SCHEMA IF NOT EXISTS `dm_meta`"] | |
| [2021/08/26 09:06:10.050 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="CREATE TABLE IF NOT EXISTS `dm_meta`.`test_syncer_checkpoint` (\n\t\t\tid VARCHAR(32) NOT NULL,\n\t\t\tcp_schema VARCHAR(128) NOT NULL,\n\t\t\tcp_table VARCHAR(128) NOT NULL,\n\t\t\tbinlog_name VARCHAR(128),\n\t\t\tbinlog_pos INT UNSIGNED,\n\t\t\tbinlog_gtid TEXT,\n\t\t\texit_safe_binlog_name VARCHAR(128) DEFAULT '',\n\t\t\texit_safe_binlog_pos INT UNSIGNED DEFAULT 0,\n\t\t\texit_safe_binlog_gtid TEXT,\n\t\t\ttable_info JSON NOT NULL,\n\t\t\tis_global BOOLEAN,\n\t\t\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n\t\t\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n\t\t\tUNIQUE KEY uk_id_schema_table (id, cp_schema, cp_table)\n\t\t)"][2021/08/26 09:06:10.062 +08:00] [INFO] [checkpoint.go:765] ["create checkpoint table"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [statements="[\"CREATE TABLE IF NOT EXISTS `dm_meta`.`test_syncer_checkpoint` (\\n\\t\\t\\tid VARCHAR(32) NOT NULL,\\n\\t\\t\\tcp_schema VARCHAR(128) NOT NULL,\\n\\t\\t\\tcp_table VARCHAR(128) NOT NULL,\\n\\t\\t\\tbinlog_name VARCHAR(128),\\n\\t\\t\\tbinlog_pos INT UNSIGNED,\\n\\t\\t\\tbinlog_gtid TEXT,\\n\\t\\t\\texit_safe_binlog_name VARCHAR(128) DEFAULT '',\\n\\t\\t\\texit_safe_binlog_pos INT UNSIGNED DEFAULT 0,\\n\\t\\t\\texit_safe_binlog_gtid TEXT,\\n\\t\\t\\ttable_info JSON NOT NULL,\\n\\t\\t\\tis_global BOOLEAN,\\n\\t\\t\\tcreate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\\n\\t\\t\\tupdate_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\\n\\t\\t\\tUNIQUE KEY uk_id_schema_table (id, cp_schema, cp_table)\\n\\t\\t)\"]"] | |
| [2021/08/26 09:06:10.062 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit="binlog replication"] [query="SELECT cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global FROM `dm_meta`.`test_syncer_checkpoint` WHERE id = ?"][2021/08/26 09:06:10.083 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit=load] [query="SELECT COUNT(id) FROM `dm_meta`.`test_loader_checkpoint` WHERE `id` = ?"][2021/08/26 09:06:10.084 +08:00] [DEBUG] [checkpoint.go:446] ["checkpoint record"] [task=test] [unit=load] [component="remote checkpoint"] [count=0] | |
| [2021/08/26 09:06:10.085 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Dump] | |
| [2021/08/26 09:06:10.100 +08:00] [DEBUG] [config.go:594] ["parse server info"] [task=test] [unit=dump] ["server info string"=5.7.34-log] | |
| [2021/08/26 09:06:10.100 +08:00] [INFO] [config.go:608] ["detect server type"] [task=test] [unit=dump] [type=MySQL] | |
| [2021/08/26 09:06:10.101 +08:00] [INFO] [config.go:627] ["detect server version"] [task=test] [unit=dump] [version=5.7.34-log] | |
| [2021/08/26 09:06:10.103 +08:00] [INFO] [dump.go:93] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\"\",\"CsvDelimiter\":\"\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"HasTiKV\":false,\"ServerType\":1,\"ServerVersion\":\"5.7.34-log\"},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":67108864,\"StatementSize\":1000000,\"SessionParams\":{\"time_zone\":\"+00:00\"},\"Tables\":null}"] | |
| [2021/08/26 09:06:10.230 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:10.573 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n {\n \"name\": \"test\",\n \"stage\": \"Running\",\n \"unit\": \"Dump\",\n \"result\": null,\n \"unresolvedDDLLockID\": \"\",\n \"dump\": {\n\n }\n }\n ]\n}"] | |
| [2021/08/26 09:06:11.267 +08:00] [DEBUG] [block_allow_list.go:16] ["start to filter tables"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.267 +08:00] [DEBUG] [block_allow_list.go:38] ["ignore table"] [task=test] [unit=dump] [tables="tables list\n\nschema adjust_gtid :[t1, ]schema dm_syncer_ignore_db :[dm_syncer_ignore_db_1, dm_syncer_ignore_db_2, ]schema metrics :[t1, ]schema mianbaoduo_dev :[affiliates, agreeProduct, apiLog, ban_user, banner, chat, chat_record, chat_report, checkCode, comments, copyright, copyright_pro, discounts, ditaner, download_handle, fans, files, geo, geo_china, groupbuy, hermit, hermit_result, inBucket, json_set, landingShow, mentions, mianbaoTag, orders, partner, partner_category, partner_invite, partner_manage, partner_product, partner_product_tag, payCut, payLog, pay_complaint, pixabay, prizes, productComplaint, productDraft, productView, product_complaint, product_diff_price, product_setting, products, productsLog, promotion, protected_pdf, qrcodeLog, qy_wx_live, recommend, reuse_discount, rewards, searchTag, search_word_filter, seckill_products, sendRecords, service_service, service_serviceorder, tags, tags_link, traffic_detail, updateLog, user, user_financial, wx_reply, ]schema Upper_Db_IGNORE :[ignore_table, ]schema dm_syncer :[dm_syncer_1, t1, ]schema downstream_more_column :[t1, ]schema only_dml :[t1, ]schema filters :[t1, ]schema full/mode :[t1, tb\\/1, ]schema openapi :[t, ]schema gtid :[t1, ]schema test_tcp_reader_db :[test_tcp_reader_table, ]schema dmctl_command :[t1, ]schema dm_syncer_do_db :[dm_syncer_do_db_1, dm_syncer_do_db_2, dm_syncer_do_db_3, ]schema sharding_seq_tmp :[t1, ]schema expr_filter :[t1, ]schema sys :[sys_config, ]schema information_schema :[CHARACTER_SETS, COLLATIONS, COLLATION_CHARACTER_SET_APPLICABILITY, COLUMNS, COLUMN_PRIVILEGES, ENGINES, EVENTS, FILES, GLOBAL_STATUS, GLOBAL_VARIABLES, KEY_COLUMN_USAGE, OPTIMIZER_TRACE, PARAMETERS, PARTITIONS, PLUGINS, PROCESSLIST, PROFILING, REFERENTIAL_CONSTRAINTS, ROUTINES, SCHEMATA, SCHEMA_PRIVILEGES, SESSION_STATUS, SESSION_VARIABLES, STATISTICS, TABLES, TABLESPACES, TABLE_CONSTRAINTS, TABLE_PRIVILEGES, TRIGGERS, USER_PRIVILEGES, VIEWS, INNODB_LOCKS, INNODB_TRX, INNODB_SYS_DATAFILES, INNODB_FT_CONFIG, INNODB_SYS_VIRTUAL, INNODB_CMP, INNODB_FT_BEING_DELETED, INNODB_CMP_RESET, INNODB_CMP_PER_INDEX, INNODB_CMPMEM_RESET, INNODB_FT_DELETED, INNODB_BUFFER_PAGE_LRU, INNODB_LOCK_WAITS, INNODB_TEMP_TABLE_INFO, INNODB_SYS_INDEXES, INNODB_SYS_TABLES, INNODB_SYS_FIELDS, INNODB_CMP_PER_INDEX_RESET, INNODB_BUFFER_PAGE, INNODB_FT_DEFAULT_STOPWORD, INNODB_FT_INDEX_TABLE, INNODB_FT_INDEX_CACHE, INNODB_SYS_TABLESPACES, INNODB_METRICS, INNODB_SYS_FOREIGN_COLS, INNODB_CMPMEM, INNODB_BUFFER_POOL_STATS, INNODB_SYS_COLUMNS, INNODB_SYS_FOREIGN, INNODB_SYS_TABLESTATS, ]schema drop_column_with_index :[t1, ]schema full_mode :[t1, ]schema load_task2 :[t1, ]schema sharding_seq :[t1, t2, ]schema load_interrupt :[t1, ]schema all_mode :[t1, ]schema mysql :[columns_priv, db, engine_cost, event, func, general_log, gtid_executed, help_category, help_keyword, help_relation, help_topic, innodb_index_stats, innodb_table_stats, ndb_binlog_index, plugin, proc, procs_priv, proxies_priv, server_cost, servers, slave_master_info, slave_relay_log_info, slave_worker_info, slow_log, tables_priv, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type, user, ]schema ignore_db :[ignore_table, ]schema dm_benchmark :[sbtest1, ]schema performance_schema :[accounts, cond_instances, events_stages_current, events_stages_history, events_stages_history_long, events_stages_summary_by_account_by_event_name, events_stages_summary_by_host_by_event_name, events_stages_summary_by_thread_by_event_name, events_stages_summary_by_user_by_event_name, events_stages_summary_global_by_event_name, events_statements_current, events_statements_history, events_statements_history_long, events_statements_summary_by_account_by_event_name, events_statements_summary_by_digest, events_statements_summary_by_host_by_event_name, events_statements_summary_by_program, events_statements_summary_by_thread_by_event_name, events_statements_summary_by_user_by_event_name, events_statements_summary_global_by_event_name, events_transactions_current, events_transactions_history, events_transactions_history_long, events_transactions_summary_by_account_by_event_name, events_transactions_summary_by_host_by_event_name, events_transactions_summary_by_thread_by_event_name, events_transactions_summary_by_user_by_event_name, events_transactions_summary_global_by_event_name, events_waits_current, events_waits_history, events_waits_history_long, events_waits_summary_by_account_by_event_name, events_waits_summary_by_host_by_event_name, events_waits_summary_by_instance, events_waits_summary_by_thread_by_event_name, events_waits_summary_by_user_by_event_name, events_waits_summary_global_by_event_name, file_instances, file_summary_by_event_name, file_summary_by_instance, global_status, global_variables, host_cache, hosts, memory_summary_by_account_by_event_name, memory_summary_by_host_by_event_name, memory_summary_by_thread_by_event_name, memory_summary_by_user_by_event_name, memory_summary_global_by_event_name, metadata_locks, mutex_instances, objects_summary_global_by_type, performance_timers, prepared_statements_instances, replication_applier_configuration, replication_applier_status, replication_applier_status_by_coordinator, replication_applier_status_by_worker, replication_connection_configuration, replication_connection_status, replication_group_member_stats, replication_group_members, rwlock_instances, session_account_connect_attrs, session_connect_attrs, session_status, session_variables, setup_actors, setup_consumers, setup_instruments, setup_objects, setup_timers, socket_instances, socket_summary_by_event_name, socket_summary_by_instance, status_by_account, status_by_host, status_by_thread, status_by_user, table_handles, table_io_waits_summary_by_index_usage, table_io_waits_summary_by_table, table_lock_waits_summary_by_table, threads, user_variables_by_thread, users, variables_by_thread, ]schema sharding_seq_opt :[t1, t2, ]schema DM_HEARTBEAT :[HEARTBEAT, ]schema fake_rotate_event :[t1, ]schema load_task4 :[t1, ]schema online_ddl :[_ignore_old, ignore, t1, t2, ]schema Upper_DB1 :[Do_Table1, ]schema test :[_loader_checkpoint, syncer_ut_onlineddl, ]schema tls :[t, ]schema load_task1 :[t1, ]schema Upper_DB :[Do_Table, Do_table_ignore, lower_Table1, lower_table, ]schema dmctl :[t_1, t_2, tb_1, tb_2, ]"] | |
| [2021/08/26 09:06:11.268 +08:00] [DEBUG] [dump.go:1207] ["no need to build region info because database is not TiDB 3.x"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.272 +08:00] [INFO] [dump.go:191] ["All the dumping transactions have started. Start to unlock tables"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.272 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of dababase 'shardddl2'"] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of dababase 'shardddl1'"] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:310] ["start dumping table..."] [task=test] [unit=dump] [database=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl2] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl2] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.273 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="meta of table 'shardddl1'.'tb1'"] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:119] ["start dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.274 +08:00] [DEBUG] [writer_util.go:132] ["finish dumping meta data"] [task=test] [unit=dump] [target=shardddl1] | |
| [2021/08/26 09:06:11.275 +08:00] [INFO] [dump.go:490] ["get estimated rows count"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [estimateCount=1] | |
| [2021/08/26 09:06:11.275 +08:00] [WARN] [dump.go:496] ["skip concurrent dump due to estimate count < rows"] [task=test] [unit=dump] ["estimate count"=1] [conf.rows=200000] [database=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [dump.go:553] ["send task to writer"] [task=test] [unit=dump] [task="data of table 'shardddl1'.'tb1'(0/1)"] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [writer.go:168] ["trying to dump table chunk"] [task=test] [unit=dump] [retryTime=1] [db=shardddl1] [table=tb1] [chunkIndex=0] [] | |
| [2021/08/26 09:06:11.275 +08:00] [DEBUG] [ir_impl.go:207] ["try to start tableData"] [task=test] [unit=dump] [query="SELECT `a`,`b` FROM `shardddl1`.`tb1` ORDER BY `a`"] | |
| [2021/08/26 09:06:11.276 +08:00] [WARN] [writer.go:230] ["no data written in table chunk"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [chunkIdx=0] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [dump.go:274] ["finish dumping table data task"] [task=test] [unit=dump] [database=shardddl1] [table=tb1] [chunkIdx=0] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [status.go:27] ["stopping log progress"] [task=test] [unit=dump] | |
| [2021/08/26 09:06:11.276 +08:00] [INFO] [collector.go:232] ["backup success summary"] [task=test] [unit=dump] [total-ranges=4] [ranges-succeed=4] [ranges-failed=0] [total-take=3.797736ms] | |
| [2021/08/26 09:06:11.276 +08:00] [DEBUG] [writer_util.go:438] ["opened file"] [task=test] [unit=dump] [path=file:/tmp/dm_test/shardddl1/worker1/dumped_data.test/metadata] | |
| [2021/08/26 09:06:11.276 +08:00] [INFO] [dumpling.go:140] ["dump data finished"] [task=test] [unit=dump] ["cost time"=1.187529872s] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Dump] [stage=Finished] [status={}] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:303] ["switching to next unit"] [subtask=test] [unit=Dump] | |
| [2021/08/26 09:06:11.277 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Load] | |
| [2021/08/26 09:06:11.279 +08:00] [INFO] [loader.go:1551] ["put load worker in etcd"] [task=test] [unit=load] [task=test] [source=mysql-replica-01] [worker=worker1] | |
| [2021/08/26 09:06:11.279 +08:00] [DEBUG] [loader.go:1073] ["collected files"] [task=test] [unit=load] [files="{\"metadata\":{},\"shardddl1-schema-create.sql\":{},\"shardddl1.tb1-schema.sql\":{},\"shardddl2-schema-create.sql\":{}}"] | |
| [2021/08/26 09:06:11.279 +08:00] [INFO] [loader.go:1046] ["prepare loading"] [task=test] [unit=load] ["cost time"=477.757µs] | |
| [2021/08/26 09:06:11.280 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit=load] [query="SELECT `filename`,`cp_schema`,`cp_table`,`offset`,`end_pos` from `dm_meta`.`test_loader_checkpoint` where `id`=?"][2021/08/26 09:06:11.281 +08:00] [INFO] [checkpoint.go:168] ["load checkpoint"] [task=test] [unit=load] [component="remote checkpoint"] ["cost time"=1.052436ms] | |
| [2021/08/26 09:06:11.281 +08:00] [INFO] [checkpoint.go:301] ["calculate checkpoint finished."] [task=test] [unit=load] [component="remote checkpoint"] ["finished tables"={}] | |
| [2021/08/26 09:06:11.288 +08:00] [INFO] [loader.go:1330] ["start to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.288 +08:00] [INFO] [loader.go:1330] ["start to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.288 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] | |
| [2021/08/26 09:06:11.288 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] | |
| [2021/08/26 09:06:11.289 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"][2021/08/26 09:06:11.289 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"][2021/08/26 09:06:11.290 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [error="Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;"] [error="Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;, RawCause: Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.290 +08:00] [INFO] [loader.go:1104] ["database already exists, skip it"] [task=test] [unit=load] ["db schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.290 +08:00] [INFO] [loader.go:1335] ["finish to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl1-schema-create.sql] | |
| [2021/08/26 09:06:11.291 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE DATABASE `shardddl` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */;, RawCause: Error 1007: Can't create database 'shardddl'; database exists"] | |
| [2021/08/26 09:06:11.291 +08:00] [INFO] [loader.go:1104] ["database already exists, skip it"] [task=test] [unit=load] ["db schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.291 +08:00] [INFO] [loader.go:1335] ["finish to create schema"] [task=test] [unit=load] ["schema file"=./dumped_data.test/shardddl2-schema-create.sql] | |
| [2021/08/26 09:06:11.296 +08:00] [INFO] [loader.go:1370] ["start to create table"] [task=test] [unit=load] ["table file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.296 +08:00] [DEBUG] [loader.go:1171] ["schema create statement"] [task=test] [unit=load] [sql="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"] | |
| [2021/08/26 09:06:11.296 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="USE `shardddl`;"][2021/08/26 09:06:11.297 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit=load] [query="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"][2021/08/26 09:06:11.297 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit=load] [query="CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;"] [error="Error 1050: Table 'shardddl.tb' already exists"] | |
| [2021/08/26 09:06:11.298 +08:00] [ERROR] [db.go:177] ["execute statements failed after retry"] [task=test] [unit=load] [queries="[USE `shardddl`; CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: CREATE TABLE `tb` (`a` int(11) NOT NULL,`b` varchar(10) DEFAULT NULL,PRIMARY KEY (`a`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;, RawCause: Error 1050: Table 'shardddl.tb' already exists"] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1121] ["table already exists, skip it"] [task=test] [unit=load] ["table schema file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1375] ["finish to create table"] [task=test] [unit=load] ["table file"=./dumped_data.test/shardddl1.tb1-schema.sql] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1422] ["finish to create tables"] [task=test] [unit=load] ["cost time"=15.998743ms] | |
| [2021/08/26 09:06:11.298 +08:00] [DEBUG] [loader.go:1429] ["restoring table data"] [task=test] [unit=load] [schema=shardddl1] [table=tb1] ["data files"={}] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:1467] ["all data files have been dispatched, waiting for them finished"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=0] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=15] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=11] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=2] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=9] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=1] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=7] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=5] | |
| [2021/08/26 09:06:11.298 +08:00] [INFO] [loader.go:236] ["file queue was closed, main routine exit."] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=8] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=4] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=3] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=10] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:135] ["start to close..."] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=6] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=12] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=14] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:138] ["closed !!!"] [task=test] [unit=load] ["worker ID"=13] | |
| [2021/08/26 09:06:11.299 +08:00] [INFO] [loader.go:739] ["all data files have been finished"] [task=test] [unit=load] ["cost time"=17.58735ms] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [loader.go:1517] ["clean dump files"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [status.go:69] ["progress status of load"] [task=test] [unit=load] [finished_bytes=0] [total_bytes=0] [total_file_count=3] [progress="100.00 %"] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Load] [stage=Finished] [status="{\"progress\":\"100.00 %\",\"metaBinlog\":\"(mysql-bin.000001, 1094)\",\"metaBinlogGTID\":\"b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5\"}"] | |
| [2021/08/26 09:06:11.300 +08:00] [INFO] [loader.go:787] ["stop importing data process"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [DEBUG] [loader.go:791] ["all workers have been closed"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.300 +08:00] [DEBUG] [loader.go:794] ["all loader's go-routines have been closed"] [task=test] [unit=load] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:303] ["switching to next unit"] [subtask=test] [unit=Load] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:11.302 +08:00] [INFO] [subtask.go:226] ["start to run"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:11.303 +08:00] [INFO] [checkpoint.go:960] ["loaded checkpoints from meta"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] ["global checkpoint"="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5(flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:11.304 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1094 b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5 0 null true]"] | |
| [2021/08/26 09:06:11.305 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5(flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:11.306 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:11.308 +08:00] [INFO] [syncer.go:3548] ["delete load worker in etcd for all mode"] [task=test] [unit="binlog replication"] [task=test] [source=mysql-replica-01] | |
| [2021/08/26 09:06:11.308 +08:00] [INFO] [syncer.go:1530] ["try to remove all dump files"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:11.310 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5"] | |
| [2021/08/26 09:06:11.318 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=161] | |
| [2021/08/26 09:06:11.319 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:11.319 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:11.320 +08:00] [INFO] [mode.go:57] ["enable safe-mode because of task initialization"] [task=test] [unit="binlog replication"] ["duration in seconds"=60] | |
| [2021/08/26 09:06:11.362 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:11.363 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.352 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:12.481 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1159,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.482 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1227,\"Flags\":8}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1281,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1325,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.488 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1325), gtid-set: "] ["raw event data"="[[1,\"aaa\"]]"] | |
| [2021/08/26 09:06:12.502 +08:00] [INFO] [optimist.go:175] ["no init schema exists"] [task=test] [unit="binlog replication"] [component="shard DDL optimist"] [schema=shardddl] [table=tb] [revision=19] | |
| [2021/08/26 09:06:12.504 +08:00] [DEBUG] [baseconn.go:99] ["query statement"] [task=test] [unit="binlog replication"] [query="SHOW CREATE TABLE `shardddl`.`tb`"][2021/08/26 09:06:12.507 +08:00] [DEBUG] [syncer.go:809] ["reverse-synchronized table schema"] [task=test] [unit="binlog replication"] [origSchema=shardddl1] [origTable=tb1] [renamedSchema=shardddl] [renamedTable=tb] [sql="CREATE TABLE IF NOT EXISTS `shardddl1`.`tb1` (`a` INT(11) NOT NULL,`b` VARCHAR(10) DEFAULT NULL,PRIMARY KEY(`a`) CLUSTERED) ENGINE = InnoDB DEFAULT CHARACTER SET = LATIN1 DEFAULT COLLATE = LATIN1_BIN"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=1`shardddl`.`tb`] [keys="[1`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=3] [key=1`shardddl`.`tb`] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1325), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1356,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.525 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1227), gtid-set: "] [location="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:12.526 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:12.536 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] aaa]"] | |
| [2021/08/26 09:06:12.586 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1421,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.586 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":115,\"LogPos\":1536,\"Flags\":0}"] | |
| [2021/08/26 09:06:12.593 +08:00] [INFO] [syncer.go:2314] [task=test] [unit="binlog replication"] [event=query] [statement="alter table shardddl1.tb1 change a c int"] [schema=] ["last location"="position: (mysql-bin.000001, 1356), gtid-set: "] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.594 +08:00] [INFO] [syncer.go:2326] ["resolve sql"] [task=test] [unit="binlog replication"] [event=query] ["raw statement"="alter table shardddl1.tb1 change a c int"] [statements="[\"ALTER TABLE `shardddl1`.`tb1` CHANGE COLUMN `a` `c` INT\"]"] [schema=] ["last location"="position: (mysql-bin.000001, 1536), gtid-set: "] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [INFO] [syncer.go:2425] ["prepare to handle ddls"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT\"]"] ["raw statement"="alter table shardddl1.tb1 change a c int"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:12.595 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1094), gtid-set: b8ba86a4-a3d5-11eb-95ce-1add06868d30:1-5)"] | |
| [2021/08/26 09:06:12.597 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1356 0 null true]"] | |
| [2021/08/26 09:06:12.598 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] shardddl1 tb1 mysql-bin.000001 1325 0 {\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handl..."] | |
| [2021/08/26 09:06:12.599 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:12.600 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:12.619 +08:00] [INFO] [optimist.go:182] ["putted a shard DDL info into etcd"] [task=test] [unit="binlog replication"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT\"],\"table-info-before\":{\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_cst_id\":0,\"update_timestamp\":427278984154185728,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0},\"table-info-after\":[{\"id\":56,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_cst_id\":0,\"update_timestamp\":427278984178302976,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0}],\"ignore-conflict\":false}"] | |
| [2021/08/26 09:06:12.643 +08:00] [INFO] [optimist.go:187] ["got a shard DDL lock operation"] [task=test] [unit="binlog replication"] [operation="{\"id\":\"test-`shardddl`.`tb`\",\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"ddls\":[],\"conflict-stage\":\"detected\",\"conflict-message\":\"[code=11111:class=functional:scope=internal:level=medium], Message: fail to try sync the optimistic shard ddl lock test-`shardddl`.`tb`: there will be conflicts if DDLs [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] are applied to the downstream. old table info: CREATE TABLE `tbl`(`a` INT(11) NOT NULL, `b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, PRIMARY KEY (`a`)) CHARSET LATIN1 COLLATE LATIN1_BIN, new table info: CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN, RawCause: at tuple index 2: at map key \\\"a\\\": column with no default value cannot be missing, Workaround: Please use `show-ddl-locks` command for more details.\",\"done\":false,\"cols\":[]}"] | |
| [2021/08/26 09:06:12.645 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1356 mysql-bin.000001 1536 null true]"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:12.646 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:12.655 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 1325), gtid-set: (flushed position: (mysql-bin.000001, 1325), gtid-set: )"] | |
| [2021/08/26 09:06:12.655 +08:00] [INFO] [checkpoint.go:694] ["rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [from="position: (mysql-bin.000001, 1325), gtid-set: "] [to="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:12.674 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:12.707 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Sync] [stage=Paused] [status="{\"totalEvents\":1,\"syncerBinlog\":\"(mysql-bin.000001, 1356)\",\"binlogType\":\"remote\"}"] | |
| [2021/08/26 09:06:12.707 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":36062,\"ErrClass\":\"sync-unit\",\"ErrScope\":\"internal\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (mysql-bin.000001, 1421), gtid-set: ], endLocation: [position: (mysql-bin.000001, 1536), gtid-set: ], origin SQL: [alter table shardddl1.tb1 change a c int]: fail to handle shard ddl [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] in optimistic mode, because schema conflict detected, conflict error: [code=11111:class=functional:scope=internal:level=medium], Message: fail to try sync the optimistic shard ddl lock test-`shardddl`.`tb`: there will be conflicts if DDLs [ALTER TABLE `shardddl`.`tb` CHANGE COLUMN `a` `c` INT] are applied to the downstream. old table info: CREATE TABLE `tbl`(`a` INT(11) NOT NULL, `b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, PRIMARY KEY (`a`)) CHARSET LATIN1 COLLATE LATIN1_BIN, new table info: CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN, RawCause: at tuple index 2: at map key \\\"a\\\": column with no default value cannot be missing, Workaround: Please use `show-ddl-locks` command for more details.\",\"Workaround\":\"Please use show-ddl-locks command for more details.\"}"] | |
| [2021/08/26 09:06:13.346 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:13.501 +08:00] [INFO] [server.go:917] [request=HandleError] [payload="op:Skip task:\"test\" "] | |
| [2021/08/26 09:06:13.502 +08:00] [INFO] [operator.go:93] ["set a new operator"] [task=test] [unit="binlog replication"] [component="error operator holder"] [position="(mysql-bin.000001, 1421)"] ["new operator"="uuid: e7119583-0894-4a95-9ce6-4cf90d2e8eb7, op: Skip, events: "] | |
| [2021/08/26 09:06:13.502 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:13.503 +08:00] [INFO] [subtask.go:525] ["resume with unit"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:13.537 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1356), gtid-set: "] | |
| [2021/08/26 09:06:13.540 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=170] | |
| [2021/08/26 09:06:13.541 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:13.542 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:13.542 +08:00] [INFO] [mode.go:37] ["enable safe-mode for safe mode exit point, will exit at"] [task=test] [unit="binlog replication"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.542 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1421,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.546 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":115,\"LogPos\":1536,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.546 +08:00] [INFO] [operator.go:167] ["match and apply a operator"] [task=test] [unit="binlog replication"] [component="error operator holder"] [startlocation="position: (mysql-bin.000001, 1421), gtid-set: "] [endlocation="position: (mysql-bin.000001, 1536), gtid-set: "] [operator="uuid: e7119583-0894-4a95-9ce6-4cf90d2e8eb7, op: Skip, events: "] | |
| [2021/08/26 09:06:13.546 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.546 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 1536), gtid-set: (flushed position: (mysql-bin.000001, 1356), gtid-set: )"] | |
| [2021/08/26 09:06:13.548 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1536 mysql-bin.000001 1536 null true]"] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1536), gtid-set: (flushed position: (mysql-bin.000001, 1536), gtid-set: )"] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:13.549 +08:00] [INFO] [syncer.go:1872] ["flush jobs when handle-error skip"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.550 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1601,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.554 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1669,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.554 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1536 0 null true]"] | |
| [2021/08/26 09:06:13.555 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1723,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1767,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["raw event data"="[[4,\"ddd\"]]"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=4`shardddl`.`tb`] [keys="[4`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.556 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=5] [key=4`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1798,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1669), gtid-set: "] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:13.557 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1863,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.558 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1931,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.558 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1985,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.560 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2029,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1767), gtid-set: "] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["raw event data"="[[7,\"ggg\"]]"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=7`shardddl`.`tb`] [keys="[7`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=7] [key=7`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.561 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2060,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1931), gtid-set: "] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2125,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2193,\"Flags\":8}"] | |
| [2021/08/26 09:06:13.562 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":2247,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2291,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2029), gtid-set: "] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["raw event data"="[[10,\"jjj\"]]"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=10`shardddl`.`tb`] [keys="[10`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=14] [key=10`shardddl`.`tb`] | |
| [2021/08/26 09:06:13.563 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2322,\"Flags\":0}"] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2193), gtid-set: "] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.564 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.567 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] ddd]"] | |
| [2021/08/26 09:06:13.568 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] ddd]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.568 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] ddd]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.568 +08:00] [INFO] [syncer.go:1464] ["received subtask's done"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.572 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] ggg]"] | |
| [2021/08/26 09:06:13.573 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] ggg]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.573 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] ggg]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.574 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] jjj]"] | |
| [2021/08/26 09:06:13.574 +08:00] [ERROR] [baseconn.go:184] ["execute statement failed"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)"] jjj]"] [error="Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.575 +08:00] [ERROR] [db.go:201] ["execute statements failed after retry"] [task=test] [unit="binlog replication"] [queries="[INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)]"] jjj]]"] [error="[code=10006:class=database:scope=not-set:level=high], Message: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?), RawCause: Error 1054: Unknown column 'a' in 'field list'"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:1470] ["the last job is transaction end, done directly"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:13.575 +08:00] [INFO] [syncer.go:1762] ["binlog replication main routine quit(context canceled)!"] [task=test] [unit="binlog replication"] ["last location"="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:13.576 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1536 mysql-bin.000001 2322 null true]"] | |
| [2021/08/26 09:06:13.577 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:13.579 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 2291), gtid-set: (flushed position: (mysql-bin.000001, 1325), gtid-set: )"] | |
| [2021/08/26 09:06:13.579 +08:00] [INFO] [checkpoint.go:694] ["rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [from="position: (mysql-bin.000001, 2291), gtid-set: "] [to="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:13.614 +08:00] [WARN] [syncer.go:699] ["something wrong with rollback global checkpoint"] [task=test] [unit="binlog replication"] ["previous position"="position: (mysql-bin.000001, 2322), gtid-set: "] ["current position"="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:13.615 +08:00] [INFO] [subtask.go:292] ["unit process returned"] [subtask=test] [unit=Sync] [stage=Paused] [status="{\"totalEvents\":1,\"syncerBinlog\":\"(mysql-bin.000001, 1536)\",\"binlogType\":\"remote\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 1767), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 2029), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.615 +08:00] [ERROR] [subtask.go:311] ["unit process error"] [subtask=test] [unit=Sync] ["error information"="{\"ErrCode\":10006,\"ErrClass\":\"database\",\"ErrScope\":\"not-set\",\"ErrLevel\":\"high\",\"Message\":\"startLocation: [position: (, 0), gtid-set: ], endLocation: [position: (mysql-bin.000001, 2291), gtid-set: ]: execute statement failed: INSERT INTO `shardddl`.`tb` (`a`,`b`) VALUES (?,?)\",\"RawCause\":\"Error 1054: Unknown column 'a' in 'field list'\"}"] | |
| [2021/08/26 09:06:13.632 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:13.747 +08:00] [INFO] [server.go:799] [request=OperateSchema] [payload="op:SetSchema task:\"test\" source:\"mysql-replica-01\" database:\"shardddl1\" table:\"tb1\" schema:\"CREATE TABLE `tb1` ( `c` int NOT NULL, `b` varchar(10) DEFAULT NULL, PRIMARY KEY (`c`)) ENGINE=InnoDB DEFAULT CHARSET=latin1\\n\" flush:true sync:true "] | |
| [2021/08/26 09:06:13.808 +08:00] [INFO] [schema.go:94] ["flush table info"] ["table info"="CREATE TABLE `shardddl1`.`tb1` (`c` INT NOT NULL,`b` VARCHAR(10) DEFAULT NULL,PRIMARY KEY(`c`)) ENGINE = InnoDB DEFAULT CHARACTER SET = LATIN1"] | |
| [2021/08/26 09:06:13.809 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] shardddl1 tb1 mysql-bin.000001 1325 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\"..."] | |
| [2021/08/26 09:06:13.815 +08:00] [INFO] [schema.go:110] ["sync info with operate-schema"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"\"],\"table-before\":\"CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN\",\"table-after\":\"CREATE TABLE `tbl`(`b` VARCHAR(10) CHARACTER SET LATIN1 COLLATE latin1_bin, `c` INT(11) NOT NULL, PRIMARY KEY (`c`)) CHARSET LATIN1 COLLATE LATIN1_BIN\",\"is-deleted\":false,\"version\":0,\"revision\":0,\"ignore-conflict\":true}"] | |
| [2021/08/26 09:06:14.108 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":2,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=false] | |
| [2021/08/26 09:06:14.109 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [worker.go:473] ["resume sub task"] [component="worker controller"] [task=test] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [subtask.go:626] ["wait condition between two units"] [subtask=test] ["previous unit"=Load] [unit=Sync] | |
| [2021/08/26 09:06:14.112 +08:00] [INFO] [subtask.go:525] ["resume with unit"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:14.147 +08:00] [INFO] [syncer.go:1585] ["replicate binlog from checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 1536), gtid-set: "] | |
| [2021/08/26 09:06:14.149 +08:00] [INFO] [streamer_controller.go:76] ["last slave connection"] [task=test] [unit="binlog replication"] ["connection ID"=174] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=0] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=0] ["new count"=1] | |
| [2021/08/26 09:06:14.150 +08:00] [INFO] [mode.go:37] ["enable safe-mode for safe mode exit point, will exit at"] [task=test] [unit="binlog replication"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":0,\"EventType\":4,\"ServerID\":1,\"EventSize\":43,\"LogPos\":0,\"Flags\":32}"] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939959,\"EventType\":15,\"ServerID\":1,\"EventSize\":119,\"LogPos\":0,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.151 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1601,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1669,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1723,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":1767,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.152 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1325), gtid-set: "] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 1767), gtid-set: "] ["raw event data"="[[4,\"ddd\"]]"] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=4`shardddl`.`tb`] [keys="[4`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=5] [key=4`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1767), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.153 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":1798,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1669), gtid-set: "] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 1798), gtid-set: "] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":1863,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":1931,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":1985,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.154 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2029,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["old location"="position: (mysql-bin.000001, 1767), gtid-set: "] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2029), gtid-set: "] ["raw event data"="[[7,\"ggg\"]]"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=7`shardddl`.`tb`] [keys="[7`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=7] [key=7`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2029), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939972,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2060,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.155 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 1931), gtid-set: "] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2060), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2125,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2193,\"Flags\":8}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":19,\"ServerID\":1,\"EventSize\":54,\"LogPos\":2247,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":30,\"ServerID\":1,\"EventSize\":44,\"LogPos\":2291,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2029), gtid-set: "] | |
| [2021/08/26 09:06:14.156 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2291), gtid-set: "] ["raw event data"="[[10,\"jjj\"]]"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=10`shardddl`.`tb`] [keys="[10`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=14] [key=10`shardddl`.`tb`] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2291), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939973,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2322,\"Flags\":0}"] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2193), gtid-set: "] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.157 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2322), gtid-set: "] | |
| [2021/08/26 09:06:14.164 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] ddd]"] | |
| [2021/08/26 09:06:14.166 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] ggg]"] | |
| [2021/08/26 09:06:14.168 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="REPLACE INTO `shardddl`.`tb` (`c`,`b`) VALUES (?,?)"] jjj]"] | |
| [2021/08/26 09:06:14.220 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:14.603 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:15.308 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:15.548 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2387,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.549 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":117,\"LogPos\":2504,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.550 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 1536 0 null true]"] | |
| [2021/08/26 09:06:15.553 +08:00] [INFO] [mode.go:100] ["change count"] [task=test] [unit="binlog replication"] ["previous count"=1] ["new count"=0] | |
| [2021/08/26 09:06:15.554 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n {\n \"name\": \"test\",\n \"stage\": \"Running\",\n \"unit\": \"Sync\",\n \"result\": null,\n \"unresolvedDDLLockID\": \"\",\n \"sync\": {\n \"totalEvents\": \"4\",\n \"totalTps\": \"0\",\n \"recentTps\": \"0\",\n \"masterBinlog\": \"\",\n \"masterBinlogGtid\": \"\",\n \"syncerBinlog\": \"(mysql-bin.000001, 1536)\",\n \"syncerBinlogGtid\": \"\",\n \"blockingDDLs\": [\n ],\n \"unresolvedGroups\": [\n ],\n \"synced\": false,\n \"binlogType\": \"remote\",\n \"secondsBehindMaster\": \"0\"\n }\n }\n ]\n}"] | |
| [2021/08/26 09:06:15.555 +08:00] [INFO] [syncer.go:2314] [task=test] [unit="binlog replication"] [event=query] [statement="alter table shardddl1.tb1 add column d int"] [schema=] ["last location"="position: (mysql-bin.000001, 2322), gtid-set: "] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.556 +08:00] [INFO] [syncer.go:2326] ["resolve sql"] [task=test] [unit="binlog replication"] [event=query] ["raw statement"="alter table shardddl1.tb1 add column d int"] [statements="[\"ALTER TABLE `shardddl1`.`tb1` ADD COLUMN `d` INT\"]"] [schema=] ["last location"="position: (mysql-bin.000001, 2504), gtid-set: "] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2291), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [INFO] [syncer.go:2425] ["prepare to handle ddls"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.557 +08:00] [INFO] [syncer.go:3126] ["flush all jobs"] [task=test] [unit="binlog replication"] ["global checkpoint"="position: (mysql-bin.000001, 2322), gtid-set: (flushed position: (mysql-bin.000001, 1536), gtid-set: )"] | |
| [2021/08/26 09:06:15.559 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 2322 0 null true]"] | |
| [2021/08/26 09:06:15.560 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] shardddl1 tb1 mysql-bin.000001 2291 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\"..."] | |
| [2021/08/26 09:06:15.561 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 2322), gtid-set: (flushed position: (mysql-bin.000001, 2322), gtid-set: )"] | |
| [2021/08/26 09:06:15.561 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:15.596 +08:00] [INFO] [optimist.go:182] ["putted a shard DDL info into etcd"] [task=test] [unit="binlog replication"] [info="{\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"down-schema\":\"shardddl\",\"down-table\":\"tb\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"],\"table-info-before\":{\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\":\"\",\"L\":\"\"},\"idx_cols\":[{\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"length\":-1}],\"state\":5,\"comment\":\"\",\"index_type\":1,\"is_unique\":true,\"is_primary\":true,\"is_invisible\":false,\"is_global\":false}],\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":false,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":1,\"max_cst_id\":0,\"update_timestamp\":427278984489992196,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0},\"table-info-after\":[{\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[{\"id\":1,\"idx_name\":{\"O\":\"PRIMARY\",\"L\":\"primary\"},\"tbl_name\":{\"O\":\"\",\"L\":\"\"},\"idx_cols\":[{\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"length\":-1}],\"state\":5,\"comment\":\"\",\"index_type\":1,\"is_unique\":true,\"is_primary\":true,\"is_invisible\":false,\"is_global\":false}],\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":false,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":3,\"max_idx_id\":1,\"max_cst_id\":0,\"update_timestamp\":427278984958443520,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":4,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0}],\"ignore-conflict\":false}"] | |
| [2021/08/26 09:06:15.616 +08:00] [INFO] [optimist.go:187] ["got a shard DDL lock operation"] [task=test] [unit="binlog replication"] [operation="{\"id\":\"test-`shardddl`.`tb`\",\"task\":\"test\",\"source\":\"mysql-replica-01\",\"up-schema\":\"shardddl1\",\"up-table\":\"tb1\",\"ddls\":[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"],\"conflict-stage\":\"none\",\"conflict-message\":\"\",\"done\":false,\"cols\":[]}"] | |
| [2021/08/26 09:06:15.619 +08:00] [INFO] [optimist.go:197] ["start to handle ddls in optimistic shard mode"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.620 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT"][2021/08/26 09:06:15.650 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.650 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.651 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 2504 0 null true]"] | |
| [2021/08/26 09:06:15.651 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] shardddl1 tb1 mysql-bin.000001 2504 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_defau..."] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 2504), gtid-set: (flushed position: (mysql-bin.000001, 2504), gtid-set: )"] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:15.653 +08:00] [INFO] [optimist.go:236] ["finish to handle ddls in optimistic shard mode"] [task=test] [unit="binlog replication"] [event=query] [ddls="[\"ALTER TABLE `shardddl`.`tb` ADD COLUMN `d` INT\"]"] ["raw statement"="alter table shardddl1.tb1 add column d int"] [location="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.665 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2569,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2637,\"Flags\":8}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":2692,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.666 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":2740,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2740), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2504), gtid-set: "] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 2740), gtid-set: "] ["raw event data"="[[13,\"mmm\",13]]"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=13`shardddl`.`tb`] [keys="[13`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=12] [key=13`shardddl`.`tb`] | |
| [2021/08/26 09:06:15.667 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2740), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":2771,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2637), gtid-set: "] [location="position: (mysql-bin.000001, 2771), gtid-set: "] | |
| [2021/08/26 09:06:15.668 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 2771), gtid-set: "] | |
| [2021/08/26 09:06:15.677 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:15.678 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] mmm 13]"] | |
| [2021/08/26 09:06:15.837 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":2836,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":2904,\"Flags\":8}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":2959,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":3007,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3007), gtid-set: "] ["old location"="position: (mysql-bin.000001, 2740), gtid-set: "] | |
| [2021/08/26 09:06:15.838 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 3007), gtid-set: "] ["raw event data"="[[16,\"ppp\",16]]"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=16`shardddl`.`tb`] [keys="[16`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=10] [key=16`shardddl`.`tb`] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3007), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939975,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":3038,\"Flags\":0}"] | |
| [2021/08/26 09:06:15.839 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 2904), gtid-set: "] [location="position: (mysql-bin.000001, 3038), gtid-set: "] | |
| [2021/08/26 09:06:15.840 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3038), gtid-set: "] | |
| [2021/08/26 09:06:15.850 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] ppp 16]"] | |
| [2021/08/26 09:06:16.089 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":33,\"ServerID\":1,\"EventSize\":65,\"LogPos\":3103,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":2,\"ServerID\":1,\"EventSize\":68,\"LogPos\":3171,\"Flags\":8}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":19,\"ServerID\":1,\"EventSize\":55,\"LogPos\":3226,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":30,\"ServerID\":1,\"EventSize\":48,\"LogPos\":3274,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.090 +08:00] [DEBUG] [checkpoint.go:478] ["compare table location whether is newer"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3274), gtid-set: "] ["old location"="position: (mysql-bin.000001, 3007), gtid-set: "] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:2109] [task=test] [unit="binlog replication"] [event=row] ["origin schema"=shardddl1] ["origin table"=tb1] ["target schema"=shardddl] ["target table"=tb] [location="position: (mysql-bin.000001, 3274), gtid-set: "] ["raw event data"="[[19,\"sss\",19]]"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:2828] ["key for keys"] [task=test] [unit="binlog replication"] [key=19`shardddl`.`tb`] [keys="[19`shardddl`.`tb`]"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1014] ["queue for key"] [task=test] [unit="binlog replication"] [queue=0] [key=19`shardddl`.`tb`] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [checkpoint.go:375] ["save table checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3274), gtid-set: "] [schema=shardddl1] [table=tb1] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1825] ["receive binlog event"] [task=test] [unit="binlog replication"] [header="{\"Timestamp\":1629939976,\"EventType\":16,\"ServerID\":1,\"EventSize\":31,\"LogPos\":3305,\"Flags\":0}"] | |
| [2021/08/26 09:06:16.091 +08:00] [DEBUG] [syncer.go:1964] [task=test] [unit="binlog replication"] [event=XID] ["last location"="position: (mysql-bin.000001, 3171), gtid-set: "] [location="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:16.092 +08:00] [DEBUG] [checkpoint.go:491] ["save global checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [location="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:16.102 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `shardddl`.`tb` (`c`,`b`,`d`) VALUES (?,?,?)"] sss 19]"] | |
| [2021/08/26 09:06:16.517 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:17.116 +08:00] [INFO] [worker.go:660] ["receive subtask stage change"] [stage="{\"expect\":0,\"source\":\"mysql-replica-01\",\"task\":\"test\"}"] ["is deleted"=true] | |
| [2021/08/26 09:06:17.116 +08:00] [INFO] [worker.go:466] ["stop sub task"] [component="worker controller"] [task=test] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [subtask.go:463] [closing] [subtask=test] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [subtask.go:360] ["closing unit process"] [subtask=test] [unit=Sync] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1464] ["received subtask's done"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1470] ["the last job is transaction end, done directly"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:2962] ["print status routine exits"] [task=test] [unit="binlog replication"] [error="context canceled"] | |
| [2021/08/26 09:06:17.117 +08:00] [INFO] [syncer.go:1762] ["binlog replication main routine quit(context canceled)!"] [task=test] [unit="binlog replication"] ["last location"="position: (mysql-bin.000001, 3305), gtid-set: "] | |
| [2021/08/26 09:06:17.118 +08:00] [INFO] [server.go:753] [request=QueryStatus] [payload="name:\"test\" "] | |
| [2021/08/26 09:06:17.119 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] mysql-bin.000001 3305 mysql-bin.000001 3305 null true]"] | |
| [2021/08/26 09:06:17.120 +08:00] [DEBUG] [baseconn.go:164] ["execute statement"] [task=test] [unit="binlog replication"] [query="INSERT INTO `dm_meta`.`test_syncer_checkpoint`\n\t\t(id, cp_schema, cp_table, binlog_name, binlog_pos, binlog_gtid, exit_safe_binlog_name, exit_safe_binlog_pos, exit_safe_binlog_gtid, table_info, is_global) VALUES\n\t\t(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n\t\tON DUPLICATE KEY UPDATE\n\t\t\tbinlog_name = VALUES(binlog_name),\n\t\t\tbinlog_pos = VALUES(binlog_pos),\n\t\t\tbinlog_gtid = VALUES(binlog_gtid),\n\t\t\texit_safe_binlog_name = VALUES(exit_safe_binlog_name),\n\t\t\texit_safe_binlog_pos = VALUES(exit_safe_binlog_pos),\n\t\t\texit_safe_binlog_gtid = VALUES(exit_safe_binlog_gtid),\n\t\t\ttable_info = VALUES(table_info),\n\t\t\tis_global = VALUES(is_global);\n\t"] shardddl1 tb1 mysql-bin.000001 3274 0 {\"id\":66,\"name\":{\"O\":\"tb1\",\"L\":\"tb1\"},\"charset\":\"latin1\",\"collate\":\"latin1_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"c\",\"L\":\"c\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":10,\"Decimal\":0,\"Charset\":\"latin1\",\"Collate\":\"latin1_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":3,\"name\":{\"O\":\"d\",\"L\":\"d\"},\"offset\":2,\"origin_default\":null,\"origin_defau..."] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1158] ["flushed checkpoint"] [task=test] [unit="binlog replication"] [checkpoint="position: (mysql-bin.000001, 3305), gtid-set: (flushed position: (mysql-bin.000001, 3305), gtid-set: )"] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1166] ["after last flushing checkpoint, DM has ignored row changes by expression filter"] [task=test] [unit="binlog replication"] ["number of filtered insert"=0] ["number of filtered update"=0] ["number of filtered delete"=0] | |
| [2021/08/26 09:06:17.123 +08:00] [INFO] [syncer.go:1668] ["flush checkpoints when exit task"] [task=test] [unit="binlog replication"] | |
| [2021/08/26 09:06:17.123 +08:00] [DEBUG] [checkpoint.go:691] ["try to rollback checkpoint"] [task=test] [unit="binlog replication"] [component="remote checkpoint"] [schema=shardddl1] [table=tb1] [checkpoint="position: (mysql-bin.000001, 3274), gtid-set: (flushed position: (mysql-bin.000001, 3274), gtid-set: )"] | |
| [2021/08/26 09:06:18.687 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:20.554 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] | |
| [2021/08/26 09:06:21.693 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:24.695 +08:00] [DEBUG] [server.go:284] ["etcd member list doesn't change"] ["client URLs"="[http://127.0.0.1:8261]"] | |
| [2021/08/26 09:06:25.559 +08:00] [DEBUG] [worker.go:146] ["runtime status"] [component="worker controller"] [status="{\n \"status\": [\n ]\n}"] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment