jayswan

Putting Apache Kafka to Use for Event Streams, Jay Kreps 20150323

"Apache Kafka and the Next 700 Stream Processing Systems" by Jay Kreps

I ♥️ Logs: Apache Kafka and Real-Time Data Integration

["Turning the database inside out with Apache Samza" by Martin Kleppmann] (https://www.youtube.com/watch?v=fU9hR3kiOK0)

[Building real-time data products at LinkedIn with Apache Samza]

jayswan

>>> import itertools
>>> import string
>>> from elasticsearch import Elasticsearch,helpers
es = Elasticsearch()
>>> # k is a generator expression that produces
... # a series of dictionaries containing test data.
... # The test data are just letter permutations
... # created with itertools.permutations.
... #
... # We then reference k as the iterator that's

jayswan

In [144]: tt = Search(using=es,index=i)\
              .filter('term',TargetUserName.raw='Domain Admins')\
              .filter('term',EventID=4728)
  File "<ipython-input-144-1b746eb83e6f>", line 1
    tt = Search(using=es,index=i)\
    .filter('term',TargetUserName.raw='Domain Admins')\
    .filter('term',EventID=4728)
SyntaxError: keyword can't be an expression

jayswan

In [142]: d
Out[142]: {'TargetUserName.raw': 'Domain Admins'}

In [143]: tt = Search(using=es,index=i)\
              .filter('term',**d).filter('term',EventID=4728)

jayswan

{
    "query": {
        "filtered": {
            "filter": {
                "bool": {
                    "must": [
                        {
                            "term": {
                                "EventID": 4728
                            }

jayswan

syntax enable
set ruler
set nobackup
set nocompatible
set encoding=utf-8
set showcmd
set number
set background=dark

"" Indentation

jayswan

from collections import defaultdict
from operator import itemgetter
import sys

FILENAME = sys.argv[1]

class SimpleCounter(defaultdict):
    """ Scrutinizer ships with Python 2.6 and doesn't have the Counter object
        from collections. This is a simple version of it.
    """

jayswan

import hashlib

def hash(s,a='md5'):
    """ One-stop hex-digest of a string. Allows any algorithm supported by hashlib. """
    f = getattr(hashlib,a)
    return f(s).hexdigest()

def fhash(fn,a='md5'):
    """ Hash a file as a string. Not memory considerate. """
    with open(fn) as f:

jayswan

from collections import namedtuple

def d2n(name,d):
    """ convert dict to namedtuple """
    NewClass = namedtuple(name,d.keys())
    return NewClass(*d.values())

jayswan

from collections import Counter
from csv import DictReader
import gzip
from pprint import pprint
from sys import argv

FIELDNAMES = ['ts', 'uid', 'id.orig_h', 'id.orig_p', 'id.resp_h', 'id.resp_p', 'proto', 'trans_id', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answersTTLs', 'rejected']


def ingest(files, delim='\t', qchar='"'):