Skip to content

Instantly share code, notes, and snippets.

Avatar

Kai Waehner kaiwaehner

View GitHub Profile
View gist:209280e2d69e93c3b1bd5f9d5fc00b0c
java.lang.NoClassDefFoundError: Could not initialize class com.fasterxml.jackson.databind.SerializationConfig
at com.fasterxml.jackson.databind.ObjectMapper.<init>(ObjectMapper.java:558)
at com.fasterxml.jackson.databind.ObjectMapper.<init>(ObjectMapper.java:474)
at io.confluent.rest.Application.getJsonMapper(Application.java:407)
at io.confluent.rest.Application.configureBaseApplication(Application.java:380)
at io.confluent.rest.Application.createServer(Application.java:148)
at io.confluent.kafka.schemaregistry.RestApp.start(RestApp.java:59)
at com.github.megachucky.kafka.streams.machinelearning.test.utils.EmbeddedSingleNodeKafkaCluster.start(EmbeddedSingleNodeKafkaCluster.java:68)
at com.github.megachucky.kafka.streams.machinelearning.test.utils.EmbeddedSingleNodeKafkaCluster.before(EmbeddedSingleNodeKafkaCluster.java:83)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:46)
View KSQL UDF Query
SELECT STRINGTOTIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss.SSS') FROM TimestampStream;
View KSQL UDF Method Interface
@Udf(description = "apply analytic model to sensor input")
public String anomaly(String sensorinput){ "YOUR LOGIC" }
View KSQL UDF Java Implementation (Deep Learning for Anomaly Detection)
package com.github.megachucky.kafka.streams.machinelearning;
import java.util.Arrays;
import hex.genmodel.GenModel;
import hex.genmodel.easy.EasyPredictModelWrapper;
import hex.genmodel.easy.RowData;
import hex.genmodel.easy.exception.PredictException;
import hex.genmodel.easy.prediction.AutoEncoderModelPrediction;
import io.confluent.ksql.function.udf.Udf;
View KSQL Stream with UDF for Anomaly Detection
create stream AnomalyDetectionWithFilter as select rowtime, eventid, anomaly(sensorinput) as Anomaly from carsensor where anomaly(sensorinput) > 5;
View Preprocessing with KSQL
CREATE STREAM creditcardfraud_preprocessed_avro WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_source WHERE Class IS NOT NULL;
@kaiwaehner
kaiwaehner / gist:46ec08597794bb827162751365f52695
Created January 9, 2019 08:55
Data Anonymisation in KSQL
View gist:46ec08597794bb827162751365f52695
SELECT Id, MASK_LEFT(User, 2) FROM creditcardfraud_source;
@kaiwaehner
kaiwaehner / gist:128053794064d7709028449ca42b8c21
Created January 9, 2019 08:56
Data Augmentation with KSQL
View gist:128053794064d7709028449ca42b8c21
SELECT Id, IFNULL(Class, -1) FROM creditcardfraud_source;
View gist:e79749a8232a1d7bc3268ca71b8fa6f6
CREATE STREAM creditcardfraud_per_user WITH (VALUE_FORMAT='AVRO', KAFKA_TOPIC='creditcardfraud_preprocessed_avro') AS SELECT Time, V1 , V2 , V3 , V4 , V5 , V6 , V7 , V8 , V9 , V10 , V11 , V12 , V13 , V14 , V15 , V16 , V17 , V18 , V19 , V20 , V21 , V22 , V23 , V24 , V25 , V26 , V27 , V28 , Amount , Class FROM creditcardfraud_enahnced c INNER JOIN USERS u on c.userid = u.userid WHERE V1 > 5 AND V2 IS NOT NULL AND u.CITY LIKE 'Premium%';