Skip to content

Instantly share code, notes, and snippets.

@ariens
ariens / Korpse.scala
Last active February 22, 2017 04:40
Kafka Offset Range Provider/Storage Engine
package com.blackberry.bdp.korpse
import scala.collection.parallel.ParIterable
import scala.collection.mutable
import scala.collection.mutable.{ Map => MutableMap }
import org.apache.spark.streaming.kafka.OffsetRange
import kafka.api._
import kafka.api.OffsetResponse
/**
* Iterates over json1 which is intended to be a full representation of a complete JSON
* structure. It compares nodes on json1 against nodes on json2 which should contain
* either the same identical structure of json1 or a subset of JSON structure contained
* in json1.
*
* If identically named nodes on json1 and json2 vary in type (ObjectNode vs ArrayNode
* for example) then an exception is thrown since json2 must not contain any additional
* structure than what is found in json1.
*
package com.blackberry.bdp.kafka.sctest;
import java.util.ArrayList;
import java.util.List;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.TopicMetadataResponse;
import kafka.javaapi.consumer.SimpleConsumer;
import org.slf4j.Logger;
@ariens
ariens / VersionedAnnotationIntrospector.java
Created July 29, 2015 13:21
An extension of com.fasterxml.jackson.databind.introspect.NopAnnotationIntrospector
/*
* Copyright 2015 dariens.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@ariens
ariens / main.scala
Created June 26, 2015 15:24
Spark-Boom-Demo
package com.blackberry.bdp.sparkMesosDemo
import com.blackberry.bdp.containers.boom._
import com.twitter.chill.avro.AvroSerializer
import com.twitter.chill.avro.AvroSerializer.SpecificRecordBinarySerializer
import com.esotericsoftware.kryo.Kryo
import org.apache.spark.serializer.{KryoSerializer, KryoRegistrator}
import org.apache.hadoop.io.NullWritable
import com.twitter.chill.avro.AvroSerializer
@ariens
ariens / auto-mesos-config.py
Created June 17, 2015 19:59
Auto build and configure a multi-master mesos cluster
#!/usr/bin/python3.4
from datetime import datetime
from shutil import copy2
from os import makedirs
from os.path import exists
import socket
import fileinput
import re
import math
@ariens
ariens / StringMatcher.java
Created April 2, 2015 18:43
An Experiment in Fast Text Matching
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.blackberry.bdp.samples;
import java.nio.charset.Charset;
import java.util.ArrayList;
import static java.util.Arrays.sort;
@ariens
ariens / gist:78504d5e23e33b6aaae4
Created March 20, 2015 15:29
InstrumentedLoggerSingleton
public class InstrumentedLoggerSingleton implements ConfigurationListener
{
private static final Logger LOG = LoggerFactory.getLogger(InstrumentedLoggerSingleton.class);
private static InstrumentedLoggerSingleton instance = null;
private final InstrumentedAppender appender;
private final Filter filter = null;
private final PatternLayout layout = null;
public static synchronized InstrumentedLoggerSingleton getInstance()