LouisAmon

import json
import gzip

def compress_data(data):
    # Convert to JSON
    json_data = json.dumps(data, indent=2)
    # Convert to bytes
    encoded = json_data.encode('utf-8')
    # Compress
    compressed = gzip.compress(encoded)

LouisAmon

def pandas_to_elasticsearch(df):
    """
    Generator that converts `pandas.DataFrame` rows into ElasticSearch actions
    """
    # ElasticSearch will raise an exception on `NaN` values
    df = df.dropna()
    for i, row in df.iterrows():
        action = {
            '_op_type': 'index',
            '_index': 'my_index',

LouisAmon

import pandas
import fastavro

def avro_df(filepath, encoding):
    # Open file stream
    with open(filepath, encoding) as fp:
        # Configure Avro reader
        reader = fastavro.reader(fp)
        # Load records in memory
        records = [r for r in reader]

LouisAmon

version: '3'
services:
  opensearch-node1:
    image: opensearchproject/opensearch:1.0.0-rc1
    container_name: opensearch-node1
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node1
      - discovery.seed_hosts=opensearch-node1
      - cluster.initial_master_nodes=opensearch-node1

LouisAmon

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

LouisAmon

def mod10(number):
	digits = []
	even = False
	for digit in reversed(number):
		digit = ord(digit) - ord('0')
		if even:
			digit = digit * 2
			if digit >= 10:
				digit = digit % 10 + digit / 10
		digits.append(digit)

LouisAmon

# This simple example illustrates how Nuage recommends you should build
# a Dynamic Provider in Pulumi
#
# The example is based on the official documentation:
# https://www.pulumi.com/docs/intro/concepts/programming-model/#example-github-labels-rest-api
#
# It defines a Github Label programmatically via Pulumi.
#
# The recommended directory structure is the following:
#

LouisAmon

import base64
import hashlib


def sha256sum(filename):
    """
    Helper function that calculates the hash of a file
    using the SHA256 algorithm
    Inspiration:
    https://stackoverflow.com/a/44873382

LouisAmon

#python3
def hook(t):
  def inner(bytes_amount):
    t.update(bytes_amount)
  return inner

BUCKET_NAME = 'your_s3_bucket_name'
FILE_NAME = 'your_s3_file_name'

s3 = boto3.resource('s3')

LouisAmon

import subprocess
import logging

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    """
    Runs a shell command and returns the output code
    STDOUT & STDERR are logged into Cloudwatch