Mike Sukmanowsky msukmanowsky

## storm_transactional_mongodb.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                msukmanowsky
                / storm_transactional_mongodb.md
            
            
              Last active
              August 29, 2015 13:56
            
              
                Thoughts on how to achieve strict and opaque transactional Trident topologies with a hypothetical Mongo Trident state.
              
          
    Storm Transactional Topologies with Mongo DB State

Strict Transactional State


Batches for a given txid are always the same. Replays of batches for a txid will exact same set of tuples as the first time that batch was emitted for that txid.
There's no overlap between batches of tuples (tuples are in one batch or another, never multiple).
Every tuple is in a batch (no tuples are skipped)

Current DB document State


## save_dict_list.py
from functools import wraps
import csv
import os.path


def save_dict_list(filename, **kwargs):
    """Decorator to take the results of a function call (assumed to be a
    ``list`` of ``dicts``) and cache them in a local file via
    csv.DictWriter and serialize them with csv.DictReader"""
    def decorator(f):

## cbloomfilter.pyx
# ported from https://github.com/jvirkki/libbloom
from cpython cimport bool
from libc.stdlib cimport malloc, calloc, free
from libc.string cimport memset
from libc.stdio cimport printf
from libc.math cimport log, ceil
from cpython.mem cimport PyMem_Malloc, PyMem_Free


DEF LN2_SQUARED = 0.480453013918201  # ln(2)^2

## gist:2905f50540eb0463e50f
{
 "metadata": {
  "name": "",
  "signature": "sha256:d0e242ec0ee3bf0798a38aba54eda99ab710de1890ddab0f0b6fd91939170314"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [

## url_monkey.py
import urlparse
import urllib
import urllib2


def patch_unquote():
    urllib.unquote = unquote
    urllib2.unquote = unquote
    urlparse.unquote = unquote

## python_urlparse_unicode_woes.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                msukmanowsky
                / python_urlparse_unicode_woes.md
            
            
              Last active
              August 29, 2015 14:02
            
          
    Python 2.7 contains a bug when dealing with percent-encoded Unicode strings
such as:
>>> import urlparse
>>> url = u"http%3A%2F%2F%C5%A1%C4%BC%C5%AB%C4%8D.org%2F"
>>> print "{!r}".format(urlparse.unquote(url))
u'http://\xc5\xa1\xc4\xbc\xc5\xab\xc4\x8d.org/'
>>> print urlparse.unquote(url)
http://Å¡Ä¼Å«Ä�.org/


## hello.html
<!DOCTYPE html>
<html lang="en">
    <head>
        <title>TODO</title>
        <!-- CSS -->
        <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css">
        <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
        <style>
            .done {
                text-decoration: line-through;

## custom_code_bolt.py
import logging

from streamparse.bolt import Bolt


log = logging.getLogger("custom_code_bolt")


class CustomCodeBolt(Bolt):

## storm_version.py
import re


class InvalidVersionException(Exception): pass

class StormVersion(object):

    VERSION_RE = re.compile(r"(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)"
                             "(?P<older_patch>\.\d+)?(?P<other>.*)")
    RC_RE = re.compile(r"-rc(?P<release_candidate>\d+)", re.IGNORECASE)

## CassandraConverters.scala
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
	from functools import wraps
	import csv
	import os.path


	def save_dict_list(filename, **kwargs):
	"""Decorator to take the results of a function call (assumed to be a
	``list`` of ``dicts``) and cache them in a local file via
	csv.DictWriter and serialize them with csv.DictReader"""
	def decorator(f):
	# ported from https://github.com/jvirkki/libbloom
	from cpython cimport bool
	from libc.stdlib cimport malloc, calloc, free
	from libc.string cimport memset
	from libc.stdio cimport printf
	from libc.math cimport log, ceil
	from cpython.mem cimport PyMem_Malloc, PyMem_Free


	DEF LN2_SQUARED = 0.480453013918201 # ln(2)^2
	{
	"metadata": {
	"name": "",
	"signature": "sha256:d0e242ec0ee3bf0798a38aba54eda99ab710de1890ddab0f0b6fd91939170314"
	},
	"nbformat": 3,
	"nbformat_minor": 0,
	"worksheets": [
	{
	"cells": [
	import urlparse
	import urllib
	import urllib2


	def patch_unquote():
	urllib.unquote = unquote
	urllib2.unquote = unquote
	urlparse.unquote = unquote
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<title>TODO</title>
	<!-- CSS -->
	<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css">
	<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
	<style>
	.done {
	text-decoration: line-through;
	import logging

	from streamparse.bolt import Bolt


	log = logging.getLogger("custom_code_bolt")


	class CustomCodeBolt(Bolt):
	import re


	class InvalidVersionException(Exception): pass

	class StormVersion(object):

	VERSION_RE = re.compile(r"(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)"
	"(?P<older_patch>\.\d+)?(?P<other>.*)")
	RC_RE = re.compile(r"-rc(?P<release_candidate>\d+)", re.IGNORECASE)
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*