oleiade

#!/usr/bin/python
# -*- coding : utf-8 -*-

from __future__ import with_statement

import os

from fabric.api import *
from fabric.contrib.files import exists

oleiade

; <<>> DiG 9.7.0-P1 <<>> telecomix.org
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 49117
;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 0

;; QUESTION SECTION:
;telecomix.org.			IN	A

;; Query time: 188 msec

oleiade

from Queue import Queue
from threading import Thread

class Worker(Thread):
    """Thread executing tasks from a given tasks queue"""
    def __init__(self, tasks):
        Thread.__init__(self)
        self.tasks = tasks
        self.daemon = True
        self.start()

oleiade

import time
import random
import itertools

from fastset.bitvector import bitvector
from Queue import Queue
from threading import Thread

from multiprocessing import Process

oleiade

def MGet(self, db, keys, fill_cache=True, *args, **kwargs):
        def get_or_none(key, context):
            try:
                res = db.Get(key, fill_cache=fill_cache)
            except KeyError:
                warning_msg = "Key {0} does not exist".format(key)
                context.update({'status': WARNING_STATUS})
                self.errors_logger.warning(warning_msg)
                res = None
            return res

oleiade

import tempfile
import hurdles
import leveldb
import shutil

from hurdles.tools import extra_setup

common_setup = "import random\n"

oleiade

First Article (Adressing the problem)

Special needs

Here at Botify

• Server logs data analytics storage (TeraBytes order)
• Bulk write/read GigaBytes datas loads that would not suit in server's memory
• Need for persistence

oleiade

Spark

Rdd

• Spark manipulates input datas as RDD, which basically are distributed datasets.
• RDD transformations (map) are lazy. It's like a roadmap of transformations to operate over dataset. But lazy, still.
• RDD actions evaluates transformations and reduces in order to generate and return the result.
• RDD transformations are re-evaluated on each actions by default unless you cache them

tips

oleiade

#!/usr/bin/env python

import sys
import os
import gzip

fpath = sys.argv[1]
output = sys.argv[2]
parts_size = int(sys.argv[3])

oleiade

#!/bin/sh

SANDBOX_DIR=/tmp/leveldb_install

## Bootstrap a sandbox
create_sandbox() {
  if [ ! -d $SANDBOX_DIR ]
	then
		mkdir -p $SANDBOX_DIR;
	fi