LouisAmon

import pandas
import fastavro

def avro_df(filepath, encoding):
    # Open file stream
    with open(filepath, encoding) as fp:
        # Configure Avro reader
        reader = fastavro.reader(fp)
        # Load records in memory
        records = [r for r in reader]

LouisAmon

def pandas_to_elasticsearch(df):
    """
    Generator that converts `pandas.DataFrame` rows into ElasticSearch actions
    """
    # ElasticSearch will raise an exception on `NaN` values
    df = df.dropna()
    for i, row in df.iterrows():
        action = {
            '_op_type': 'index',
            '_index': 'my_index',

LouisAmon

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

LouisAmon

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

LouisAmon

import time

import numpy
import redis

# a 2D array to serialize
A = 10 * numpy.random.randn(10000).reshape(1000, 10)

# flatten the 2D NumPy array and save it as a binary string

LouisAmon

'use strict';
const aws = require('aws-sdk');

const lambda = new aws.Lambda({region: 'eu-west-1'});
const async = require('async');

exports.handler = (event, context, callback) => {
    /*
    Invokes a given Lambda function `event.function` in parallel for each given
    argument `event.events`

LouisAmon

# System dependencies
yum update -y
yum install -y \
  binutils \
  findutils \
  python36
# Create directory for our lambda
mkdir build
pip install -t build numpy pandas scipy
# Check uncompressed file size

LouisAmon

def trigo():
    product = 1.0
    for counter in range(1, 1000, 1):
        for dex in list(range(1, 360, 1)):
            angle = radians(dex)
            product *= sin(angle)**2 + cos(angle)**2
    return product
    
def sha():
    h = hashlib.sha1('secret')

LouisAmon

import json
import gzip

def compress_data(data):
    # Convert to JSON
    json_data = json.dumps(data, indent=2)
    # Convert to bytes
    encoded = json_data.encode('utf-8')
    # Compress
    compressed = gzip.compress(encoded)

LouisAmon

import subprocess
import logging

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    """
    Runs a shell command and returns the output code
    STDOUT & STDERR are logged into Cloudwatch