ioc32

Problem statement

It's your first day at HorribleLegacy, Inc. You've been told to investigate a performance issue within a system written by your predecessor. This sytem has no documentation. All we know is that it relies on the following program running inside a Docker image:

docker run gi11es/interview-exercise /usr/bin/helloworld

Ignoring the amount of time it takes to download the docker image for the first time, running the above command outputs "hello" after a while:

~ $ docker run gi11es/interview-exercise /usr/bin/helloworld
hello

ioc32

1. General Background and Overview

• Probabilistic Data Structures for Web Analytics and Data Mining : A great overview of the space of probabilistic data structures and how they are used in approximation algorithm implementation.
• Models and Issues in Data Stream Systems
• Philippe Flajolet’s contribution to streaming algorithms : A presentation by Jérémie Lumbroso that visits some of the hostorical perspectives and how it all began with Flajolet
• Approximate Frequency Counts over Data Streams by Gurmeet Singh Manku & Rajeev Motwani : One of the early papers on the subject.
• [Methods for Finding Frequent Items in Data Streams](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.187.9800&rep

ioc32

root@esnog19:/etc/logstash# cat conf.d/inputs-intermediate.conf
input {
    file {
        path => "/home/malibu/data/nokia.log"
        start_position => "beginning"
        type => "nokia"
        codec => multiline {
            pattern => "^\d+\s+\d{4}"
            what => "previous"

ioc32

PORT [0-9]+
BIND_TIME %{MONTHDAY}-%{MONTH}-%{YEAR} %{TIME}
ZONE [0-9a-zA-Z.-]+
DNS_ZONE %{ZONE:zone}/%{DATA:class}/%{DATA:view}
STR [a-zA-Z\-]+

XFR_START1 zone %{DNS_ZONE}: %{DATA:daemon_status}\.
XFR_START2 client %{IP:client_ip}[@#]+%{PORT:client_port} \(%{ZONE}\): view %{DATA:view}: transfer of \'%{DATA:zone}/%{DATA:class}\': %{DATA:daemon_status} (?:\(serial %{PORT:serial}\)|\(serial %{PORT:serial_old} -> %{PORT:serial}\))
XFR_START (?:%{XFR_START1}|%{XFR_START2})
XFR_SERIAL zone %{DNS_ZONE}: %{DATA:daemon_status} serial %{POSINT:serial}

ioc32

[root@localhost build]# make clean-heka
Scanning dependencies of target clean-heka
[100%] Resynchronizing the Go workspace with the Heka repository
-- GeoIP.h was not found, GeoIP functionality will not be included in this build.
-- Docker plugins enabled.
-- sphinx-build was not found, the documentation will not be generated.
-- Configuring done
-- Generating done
-- Build files have been written to: /root/heka/build
[100%] Built target clean-heka

ioc32

[root@localhost build]# ctest
Test project /root/heka/build
      Start  1: cmd/hekad
 1/26 Test  #1: cmd/hekad ........................   Passed    3.22 sec
      Start  2: message
 2/26 Test  #2: message ..........................   Passed    1.52 sec
      Start  3: pipeline
 3/26 Test  #3: pipeline .........................   Passed    3.90 sec
      Start  4: plugins
 4/26 Test  #4: plugins ..........................   Passed    2.62 sec

ioc32

AS0 -Reserved AS-,ZZ
AS1 LVLT-1 - Level 3 Communications, Inc.,US
AS2 UDEL-DCN - University of Delaware,US
AS3 MIT-GATEWAYS - Massachusetts Institute of Technology,US
AS4 ISI-AS - University of Southern California,US
AS5 SYMBOLICS - Symbolics, Inc.,US
AS6 BULL-HN - Bull HN Information Systems Inc.,US
AS7 UK Defence Research Agency,GB
AS8 RICE-AS - Rice University,US
AS9 CMU-ROUTER - Carnegie Mellon University,US

ioc32

# cat get_apu_temp.sh                                                                                                   
#!/bin/sh

date=$(date "+%Y-%m-%dT%H:%M:%S")
temp=$(sysctl hw.sensors.km0.temp0 | sed 's/.*=\([0-9\.]*\) degC/\1/')
ints=$(vmstat -w 1 -c 2 | awk 'END {print $14}')
echo $date,$temp,$ints >> /root/apu-temperatures.csv

# crontab -l | grep get_apu_temp
*     *       *       *       *       /bin/sh /root/get_apu_temp.sh

ioc32

iortiz@monet:~/Code/git/cifu-archive
$ ls -1 /mnt/storage/cifu/atj-*
/mnt/storage/cifu/atj-2008-08-28-A todo jazz - 280808.mp3
/mnt/storage/cifu/atj-2008-09-06-Primera hora - 060908.mp3
/mnt/storage/cifu/atj-2008-09-06-Segunda hora - 060908.mp3
/mnt/storage/cifu/atj-2008-09-13-Primera hora - 130908.mp3
/mnt/storage/cifu/atj-2008-09-13-Segunda hora - 130908.mp3
/mnt/storage/cifu/atj-2008-09-20-Primera hora - 200908.mp3
/mnt/storage/cifu/atj-2008-09-20-Segunda hora - 200908.mp3
/mnt/storage/cifu/atj-2008-09-27-Primera hora - 270908.mp3

ioc32

[hekad]
maxprocs = 2
base_dir = "/tmp/cache"
pid_file = "/tmp/hekad.pid"
share_dir = "/usr/share/heka"

[log1]
type = "LogstreamerInput"
log_directory = "/var/log/"
file_match = 'yum\.log'