Louis Amon LouisAmon

## serialize-numpy-array.py

import time

import numpy
import redis

# a 2D array to serialize
A = 10 * numpy.random.randn(10000).reshape(1000, 10)

# flatten the 2D NumPy array and save it as a binary string

## shovel.py
(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

## shovel.py
(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

## pandas_to_elasticsearch.py
def pandas_to_elasticsearch(df):
    """
    Generator that converts `pandas.DataFrame` rows into ElasticSearch actions
    """
    # ElasticSearch will raise an exception on `NaN` values
    df = df.dropna()
    for i, row in df.iterrows():
        action = {
            '_op_type': 'index',
            '_index': 'my_index',

## avro_to_dataframe.py
import pandas
import fastavro

def avro_df(filepath, encoding):
    # Open file stream
    with open(filepath, encoding) as fp:
        # Configure Avro reader
        reader = fastavro.reader(fp)
        # Load records in memory
        records = [r for r in reader]

	import time

	import numpy
	import redis

	# a 2D array to serialize
	A = 10 * numpy.random.randn(10000).reshape(1000, 10)

	# flatten the 2D NumPy array and save it as a binary string
	(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
	got 901 messages

	real 0m1.617s
	user 0m0.558s
	sys 0m0.044s
	def pandas_to_elasticsearch(df):
	"""
	Generator that converts `pandas.DataFrame` rows into ElasticSearch actions
	"""
	# ElasticSearch will raise an exception on `NaN` values
	df = df.dropna()
	for i, row in df.iterrows():
	action = {
	'_op_type': 'index',
	'_index': 'my_index',
	import pandas
	import fastavro

	def avro_df(filepath, encoding):
	# Open file stream
	with open(filepath, encoding) as fp:
	# Configure Avro reader
	reader = fastavro.reader(fp)
	# Load records in memory
	records = [r for r in reader]