Skip to content

Instantly share code, notes, and snippets.

Avatar

Kai Waehner kaiwaehner

View GitHub Profile
View gist:209280e2d69e93c3b1bd5f9d5fc00b0c
java.lang.NoClassDefFoundError: Could not initialize class com.fasterxml.jackson.databind.SerializationConfig
at com.fasterxml.jackson.databind.ObjectMapper.<init>(ObjectMapper.java:558)
at com.fasterxml.jackson.databind.ObjectMapper.<init>(ObjectMapper.java:474)
at io.confluent.rest.Application.getJsonMapper(Application.java:407)
at io.confluent.rest.Application.configureBaseApplication(Application.java:380)
at io.confluent.rest.Application.createServer(Application.java:148)
at io.confluent.kafka.schemaregistry.RestApp.start(RestApp.java:59)
at com.github.megachucky.kafka.streams.machinelearning.test.utils.EmbeddedSingleNodeKafkaCluster.start(EmbeddedSingleNodeKafkaCluster.java:68)
at com.github.megachucky.kafka.streams.machinelearning.test.utils.EmbeddedSingleNodeKafkaCluster.before(EmbeddedSingleNodeKafkaCluster.java:83)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:46)
View KSQL UDF Query
SELECT STRINGTOTIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss.SSS') FROM TimestampStream;
View KSQL UDF Method Interface
@Udf(description = "apply analytic model to sensor input")
public String anomaly(String sensorinput){ "YOUR LOGIC" }
View KSQL Stream with UDF for Anomaly Detection
create stream AnomalyDetectionWithFilter as select rowtime, eventid, anomaly(sensorinput) as Anomaly from carsensor where anomaly(sensorinput) > 5;
View Preprocessing with KSQL
CREATE STREAM creditcardfraud_preprocessed_avro WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_source WHERE Class IS NOT NULL;
@kaiwaehner
kaiwaehner / gist:46ec08597794bb827162751365f52695
Created January 9, 2019 08:55
Data Anonymisation in KSQL
View gist:46ec08597794bb827162751365f52695
SELECT Id, MASK_LEFT(User, 2) FROM creditcardfraud_source;
@kaiwaehner
kaiwaehner / gist:128053794064d7709028449ca42b8c21
Created January 9, 2019 08:56
Data Augmentation with KSQL
View gist:128053794064d7709028449ca42b8c21
SELECT Id, IFNULL(Class, -1) FROM creditcardfraud_source;
View gist:e79749a8232a1d7bc3268ca71b8fa6f6
CREATE STREAM creditcardfraud_per_user WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_enahnced c INNER JOIN USERS u on c.userid = u.userid WHERE V1 > 5 AND V2 IS NOT NULL AND u.CITY LIKE 'Premium%';
@kaiwaehner
kaiwaehner / gist:a5d6710c2d038bada2d3dfd7a66d6450
Created January 9, 2019 09:01
Create KSQL STREAM for Preprocessing with Python
View gist:a5d6710c2d038bada2d3dfd7a66d6450
client.create_stream_as(table_name='creditcardfraud_preprocessed_avro',
select_columns=['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'Amount', 'Class'],
src_table='creditcardfraud_source',
conditions='Class IS NOT NULL',
kafka_topic='creditcardfraud_preprocessed_avro',
value_format='AVRO')