This is a series of gists documenting testing done with the numeric.mapping
option in Kafka Connect.
- Oracle
- MS SQL Server
- Postgres
- MySQL - n/a because of #563
—@rmoff January 9, 2019
builder | |
.table("users") | |
.suppress(Suppressed.untilTimeLimit( | |
BufferConfig.maxBytes(myConfig.getUsersBufferSize()) | |
)) | |
... |
driver.pipeInput(recordFactory.create( | |
/* topic */ "input", | |
/* key */ "A", | |
/* value */ "v1", | |
/* timestamp */ 10L | |
)); | |
// Stream time is now 10L | |
driver.pipeInput(recordFactory.create("input", "A", "v2", 11L)); | |
// Stream time is now 11L |
events | |
.groupByKey() | |
.windowedBy( | |
TimeWindows.of(Duration.ofMinutes(2).withGrace(Duration.ofMinutes(2)) | |
) | |
.count(Materialized.as("count-metric")) | |
.suppress(Suppressed.untilWindowClose(BufferConfig.unbounded())) | |
.filter( _ < 4 ) | |
.toStream() | |
.foreach( /* Send that email! */) |
events | |
.groupByKey() | |
.windowedBy(TimeWindows.of(Duration.ofMinutes(2))) | |
.count(Materialized.as("count-metric")) | |
.filter( _ < 4 ) | |
.toStream() | |
.foreach( /* Send that email! */) | |
// graph servlet queries "count-metric" |
events | |
.groupByKey() | |
.windowedBy(TimeWindows.of(Duration.ofMinutes(2))) | |
.count(Materialized.as("count-metric")) | |
// graph servlet queries "count-metric" |
This is a series of gists documenting testing done with the numeric.mapping
option in Kafka Connect.
—@rmoff January 9, 2019
CREATE STREAM creditcardfraud_per_user WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_enahnced c INNER JOIN USERS u on c.userid = u.userid WHERE V1 > 5 AND V2 IS NOT NULL AND u.CITY LIKE 'Premium%'; |
SELECT Id, IFNULL(Class, -1) FROM creditcardfraud_source; |
SELECT Id, MASK_LEFT(User, 2) FROM creditcardfraud_source; |
CREATE STREAM creditcardfraud_preprocessed_avro WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_source WHERE Class IS NOT NULL; |