Skip to content

Instantly share code, notes, and snippets.

@rcoup

rcoup/initial.json Secret

Last active November 13, 2017 14:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rcoup/2ed1601577bb42d47db5f595f8d2ff46 to your computer and use it in GitHub Desktop.
Save rcoup/2ed1601577bb42d47db5f595f8d2ff46 to your computer and use it in GitHub Desktop.
Download ZIP
Debezium Postgresql wal2json drops unknown type values.
Raw
initial.json
{
"schema": {
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
}
],
"optional": false,
"name": "pgwal_test.public.nz_titles.Key"
},
"payload": {
"gid": 54914
}
}
{
"schema": {
"type": "struct",
"fields": [
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
},
{
"type": "int32",
"optional": true,
"field": "id"
},
{
"type": "string",
"optional": true,
"field": "title_no"
},
{
"type": "string",
"optional": true,
"field": "status"
},
{
"type": "string",
"optional": true,
"field": "type"
},
{
"type": "string",
"optional": true,
"field": "land_distr"
},
{
"type": "int32",
"optional": true,
"name": "io.debezium.time.Date",
"version": 1,
"field": "issue_date"
},
{
"type": "string",
"optional": true,
"field": "guarantee_"
},
{
"type": "string",
"optional": true,
"field": "estate_des"
},
{
"type": "string",
"optional": true,
"field": "number_own"
},
{
"type": "string",
"optional": true,
"field": "spatial_ex"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Json",
"version": 1,
"field": "geom"
}
],
"optional": true,
"name": "pgwal_test.public.nz_titles.Value",
"field": "before"
},
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
},
{
"type": "int32",
"optional": true,
"field": "id"
},
{
"type": "string",
"optional": true,
"field": "title_no"
},
{
"type": "string",
"optional": true,
"field": "status"
},
{
"type": "string",
"optional": true,
"field": "type"
},
{
"type": "string",
"optional": true,
"field": "land_distr"
},
{
"type": "int32",
"optional": true,
"name": "io.debezium.time.Date",
"version": 1,
"field": "issue_date"
},
{
"type": "string",
"optional": true,
"field": "guarantee_"
},
{
"type": "string",
"optional": true,
"field": "estate_des"
},
{
"type": "string",
"optional": true,
"field": "number_own"
},
{
"type": "string",
"optional": true,
"field": "spatial_ex"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Json",
"version": 1,
"field": "geom"
}
],
"optional": true,
"name": "pgwal_test.public.nz_titles.Value",
"field": "after"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "name"
},
{
"type": "int64",
"optional": true,
"field": "ts_usec"
},
{
"type": "int32",
"optional": true,
"field": "txId"
},
{
"type": "int64",
"optional": true,
"field": "lsn"
},
{
"type": "boolean",
"optional": true,
"field": "snapshot"
},
{
"type": "boolean",
"optional": true,
"field": "last_snapshot_record"
}
],
"optional": false,
"name": "io.debezium.connector.postgresql.Source",
"field": "source"
},
{
"type": "string",
"optional": false,
"field": "op"
},
{
"type": "int64",
"optional": true,
"field": "ts_ms"
}
],
"optional": false,
"name": "pgwal_test.public.nz_titles.Envelope",
"version": 1
},
"payload": {
"before": null,
"after": {
"gid": 54914,
"id": 976948,
"title_no": "NL10B/847",
"status": "LIVE",
"type": "Cross Lease",
"land_distr": "Nelson",
"issue_date": 8378,
"guarantee_": "Guarantee",
"estate_des": "Fee Simple, 1/2, Lot 27 Deposited Plan 15779, 597 m2\r\nLeasehold, 1/1, Flat 1 Deposited Plan 16024",
"number_own": "1",
"spatial_ex": "T",
"geom": "0106000020910800000100000001030000000100000008000000B728E3570FB2384102B5DDC1B3B15441E5B042C820B23841CB8D2260B7B154418FC1C5C834B238416547CD59B1B1544195C1EFBA2FB23841C1DF574DB0B154413FB1D5E524B2384167A515CDAFB15441101CA89C1AB23841FF681BCBB0B154418AD7713413B23841E8954106B3B15441B728E3570FB2384102B5DDC1B3B15441"
},
"source": {
"name": "pgwal_test",
"ts_usec": 1510583956596000,
"txId": 580,
"lsn": 141303808,
"snapshot": true,
"last_snapshot_record": false
},
"op": "r",
"ts_ms": 1510583956596
}
}
Raw
log.txt
connect_1 | 2017-11-13 14:43:04,666 DEBUG Postgres|pgwal_test|records-stream-producer Message arrived for decoding {
connect_1 | "xid" : 583,
connect_1 | "timestamp" : "2017-11-13 14:43:04.594286+00",
connect_1 | "change" : [ {
connect_1 | "kind" : "update",
connect_1 | "schema" : "public",
connect_1 | "table" : "nz_titles",
connect_1 | "columnnames" : [ "gid", "id", "title_no", "status", "type", "land_distr", "issue_date", "guarantee_", "estate_des", "number_own", "spatial_ex", "geom" ],
connect_1 | "columntypes" : [ "int4", "int4", "varchar", "varchar", "varchar", "varchar", "date", "varchar", "varchar", "varchar", "varchar", "geometry" ],
connect_1 | "columnvalues" : [ 54914, 976948, "NL10B/847", "LIVE", "Cross Lease", "Nelson", "1992-12-09", "Guarantee", "Fee Simple, 1/2, Lot 27 Deposited Plan 15779, 597 m2\r\nLeasehold, 1/1, Flat 1 Deposited Plan 16024", "1", "T", "0106000020910800000100000001030000000100000008000000B728E3570FB2384102B5DDC1B3B15441E5B042C820B23841CB8D2260B7B154418FC1C5C834B238416547CD59B1B1544195C1EFBA2FB23841C1DF574DB0B154413FB1D5E524B2384167A515CDAFB15441101CA89C1AB23841FF681BCBB0B154418AD7713413B23841E8954106B3B15441B728E3570FB2384102B5DDC1B3B15441" ],
connect_1 | "oldkeys" : {
connect_1 | "keynames" : [ "gid" ],
connect_1 | "keytypes" : [ "int4" ],
connect_1 | "keyvalues" : [ 54914 ]
connect_1 | }
connect_1 | } ]
connect_1 | } [io.debezium.connector.postgresql.connection.wal2json.Wal2JsonMessageDecoder]
connect_1 | 2017-11-13 14:43:04,706 DEBUG Postgres|pgwal_test|records-stream-producer received new message at position 0/86D44E9
connect_1 | io.debezium.connector.postgresql.connection.wal2json.Wal2JsonReplicationMessage@7f3cffc5 [io.debezium.connector.postgresql.RecordsStreamProducer]
connect_1 | 2017-11-13 14:43:04,726 WARN Postgres|pgwal_test|records-stream-producer processing column 'geom' with unknown data type 'geometry' as byte array [io.debezium.connector.postgresql.connection.wal2json.Wal2JsonReplicationMessage]
connect_1 | 2017-11-13 14:43:04,735 DEBUG Postgres|pgwal_test|postgres-connector-task checking for more records... [io.debezium.connector.postgresql.PostgresConnectorTask]
connect_1 | 2017-11-13 14:43:04,735 DEBUG Postgres|pgwal_test|postgres-connector-task no records available yet, sleeping a bit... [io.debezium.connector.postgresql.PostgresConnectorTask]
connect_1 | 2017-11-13 14:43:04,736 DEBUG Postgres|pgwal_test|records-stream-producer Connected to jdbc:postgresql://postgres:5432/postgres with {server.name=pgwal_test, history.kafka.topic=schema-changes.pgwal_test, user=postgres, assumeMinServerVersion=9.4, history.kafka.bootstrap.servers=kafka:9092, password=postgres} [io.debezium.jdbc.JdbcConnection]
connect_1 | 2017-11-13 14:43:04,804 DEBUG Postgres|pgwal_test|records-stream-producer refreshing DB schema for table 'public.nz_titles' [io.debezium.connector.postgresql.PostgresSchema]
connect_1 | 2017-11-13 14:43:04,804 DEBUG Postgres|pgwal_test|records-stream-producer Mapping table 'public.nz_titles' to schemas under 'pgwal_test.public.nz_titles' [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,805 DEBUG Postgres|pgwal_test|records-stream-producer - field 'gid' (INT32) from column gid serial(10,0) NOT NULL AUTO_INCREMENTED [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,806 DEBUG Postgres|pgwal_test|records-stream-producer - field 'gid' (INT32) from column gid serial(10,0) NOT NULL AUTO_INCREMENTED [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,807 DEBUG Postgres|pgwal_test|records-stream-producer - field 'id' (INT32) from column id int4(10,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,807 DEBUG Postgres|pgwal_test|records-stream-producer - field 'title_no' (STRING) from column title_no varchar(20,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,807 DEBUG Postgres|pgwal_test|records-stream-producer - field 'status' (STRING) from column status varchar(4,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,808 DEBUG Postgres|pgwal_test|records-stream-producer - field 'type' (STRING) from column type varchar(100,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,808 DEBUG Postgres|pgwal_test|records-stream-producer - field 'land_distr' (STRING) from column land_distr varchar(100,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,809 DEBUG Postgres|pgwal_test|records-stream-producer - field 'issue_date' (INT32) from column issue_date date(13,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,809 DEBUG Postgres|pgwal_test|records-stream-producer - field 'guarantee_' (STRING) from column guarantee_ varchar(100,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,809 DEBUG Postgres|pgwal_test|records-stream-producer - field 'estate_des' (STRING) from column estate_des varchar(254,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,810 DEBUG Postgres|pgwal_test|records-stream-producer - field 'number_own' (STRING) from column number_own varchar(20,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,810 DEBUG Postgres|pgwal_test|records-stream-producer - field 'spatial_ex' (STRING) from column spatial_ex varchar(2,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,811 DEBUG Postgres|pgwal_test|records-stream-producer - field 'geom' (STRING) from column geom geometry(2147483647,0) [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,812 DEBUG Postgres|pgwal_test|records-stream-producer Mapped primary key for table 'public.nz_titles' to schema: {"name" : "pgwal_test.public.nz_titles.Key", "type" : "STRUCT", "optional" : "false", "fields" : [{"name" : "gid", "index" : "0", "schema" : {"type" : "INT32", "optional" : "false"}}]} [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,813 DEBUG Postgres|pgwal_test|records-stream-producer Mapped columns for table 'public.nz_titles' to schema: {"name" : "pgwal_test.public.nz_titles.Value", "type" : "STRUCT", "optional" : "true", "fields" : [{"name" : "gid", "index" : "0", "schema" : {"type" : "INT32", "optional" : "false"}}, {"name" : "id", "index" : "1", "schema" : {"type" : "INT32", "optional" : "true"}}, {"name" : "title_no", "index" : "2", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "status", "index" : "3", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "type", "index" : "4", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "land_distr", "index" : "5", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "issue_date", "index" : "6", "schema" : {"name" : "io.debezium.time.Date", "type" : "INT32", "optional" : "true", "version" : "1"}}, {"name" : "guarantee_", "index" : "7", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "estate_des", "index" : "8", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "number_own", "index" : "9", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "spatial_ex", "index" : "10", "schema" : {"type" : "STRING", "optional" : "true"}}, {"name" : "geom", "index" : "11", "schema" : {"name" : "io.debezium.data.Json", "type" : "STRING", "optional" : "true", "version" : "1"}}]} [io.debezium.relational.TableSchemaBuilder]
connect_1 | 2017-11-13 14:43:04,828 DEBUG Postgres|pgwal_test|postgres-connector-task Placed source record 'SourceRecord{sourcePartition={server=pgwal_test}, sourceOffset={lsn=141378793, txId=583, ts_usec=1510584184594286000}} ConnectRecord{topic='pgwal_test.public.nz_titles', kafkaPartition=null, key=Struct{gid=54914}, value=Struct{before=Struct{gid=54914},after=Struct{gid=54914,id=976948,title_no=NL10B/847,status=LIVE,type=Cross Lease,land_distr=Nelson,issue_date=8378,guarantee_=Guarantee,estate_des=Fee Simple, 1/2, Lot 27 Deposited Plan 15779, 597 m2
connect_1 | Leasehold, 1/1, Flat 1 Deposited Plan 16024,number_own=1,spatial_ex=T},source=Struct{name=pgwal_test,ts_usec=1510584184594286000,txId=583,lsn=141378793},op=u,ts_ms=1510584184823}, timestamp=null}' into queue [io.debezium.connector.postgresql.PostgresConnectorTask]
Raw
update.json
{
"schema": {
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
}
],
"optional": false,
"name": "pgwal_test.public.nz_titles.Key"
},
"payload": {
"gid": 54914
}
}
{
"schema": {
"type": "struct",
"fields": [
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
},
{
"type": "int32",
"optional": true,
"field": "id"
},
{
"type": "string",
"optional": true,
"field": "title_no"
},
{
"type": "string",
"optional": true,
"field": "status"
},
{
"type": "string",
"optional": true,
"field": "type"
},
{
"type": "string",
"optional": true,
"field": "land_distr"
},
{
"type": "int32",
"optional": true,
"name": "io.debezium.time.Date",
"version": 1,
"field": "issue_date"
},
{
"type": "string",
"optional": true,
"field": "guarantee_"
},
{
"type": "string",
"optional": true,
"field": "estate_des"
},
{
"type": "string",
"optional": true,
"field": "number_own"
},
{
"type": "string",
"optional": true,
"field": "spatial_ex"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Json",
"version": 1,
"field": "geom"
}
],
"optional": true,
"name": "pgwal_test.public.nz_titles.Value",
"field": "before"
},
{
"type": "struct",
"fields": [
{
"type": "int32",
"optional": false,
"field": "gid"
},
{
"type": "int32",
"optional": true,
"field": "id"
},
{
"type": "string",
"optional": true,
"field": "title_no"
},
{
"type": "string",
"optional": true,
"field": "status"
},
{
"type": "string",
"optional": true,
"field": "type"
},
{
"type": "string",
"optional": true,
"field": "land_distr"
},
{
"type": "int32",
"optional": true,
"name": "io.debezium.time.Date",
"version": 1,
"field": "issue_date"
},
{
"type": "string",
"optional": true,
"field": "guarantee_"
},
{
"type": "string",
"optional": true,
"field": "estate_des"
},
{
"type": "string",
"optional": true,
"field": "number_own"
},
{
"type": "string",
"optional": true,
"field": "spatial_ex"
},
{
"type": "string",
"optional": true,
"name": "io.debezium.data.Json",
"version": 1,
"field": "geom"
}
],
"optional": true,
"name": "pgwal_test.public.nz_titles.Value",
"field": "after"
},
{
"type": "struct",
"fields": [
{
"type": "string",
"optional": false,
"field": "name"
},
{
"type": "int64",
"optional": true,
"field": "ts_usec"
},
{
"type": "int32",
"optional": true,
"field": "txId"
},
{
"type": "int64",
"optional": true,
"field": "lsn"
},
{
"type": "boolean",
"optional": true,
"field": "snapshot"
},
{
"type": "boolean",
"optional": true,
"field": "last_snapshot_record"
}
],
"optional": false,
"name": "io.debezium.connector.postgresql.Source",
"field": "source"
},
{
"type": "string",
"optional": false,
"field": "op"
},
{
"type": "int64",
"optional": true,
"field": "ts_ms"
}
],
"optional": false,
"name": "pgwal_test.public.nz_titles.Envelope",
"version": 1
},
"payload": {
"before": {
"gid": 54914,
"id": null,
"title_no": null,
"status": null,
"type": null,
"land_distr": null,
"issue_date": null,
"guarantee_": null,
"estate_des": null,
"number_own": null,
"spatial_ex": null,
"geom": null
},
"after": {
"gid": 54914,
"id": 976948,
"title_no": "NL10B/847",
"status": "LIVE",
"type": "Cross Lease",
"land_distr": "Nelson",
"issue_date": 8378,
"guarantee_": "Guarantee",
"estate_des": "Fee Simple, 1/2, Lot 27 Deposited Plan 15779, 597 m2\r\nLeasehold, 1/1, Flat 1 Deposited Plan 16024",
"number_own": "1",
"spatial_ex": "T",
"geom": null
},
"source": {
"name": "pgwal_test",
"ts_usec": 1510584063551842000,
"txId": 582,
"lsn": 141341025,
"snapshot": null,
"last_snapshot_record": null
},
"op": "u",
"ts_ms": 1510584063702
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment