uhho

def s3_to_pandas(client, bucket, key, header=None):

    # get key using boto3 client
    obj = client.get_object(Bucket=bucket, Key=key)
    gz = gzip.GzipFile(fileobj=obj['Body'])
    
    # load stream directly to DF
    return pd.read_csv(gz, header=header, dtype=str)

def s3_to_pandas_with_processing(client, bucket, key, header=None):

philippkeller

Getting Started With Superset: Airbnb’s data exploration platform

These instructions are for Amazon Linux Version 2

Update Python and PIP versions on EC2 (Amazon AMI)

sudo yum update -y
sudo yum install python3 -y

mrshu

awscli==1.14.45
boto3==1.5.35
botocore==1.8.49
colorama==0.3.7
docutils==0.14
futures==3.2.0
jmespath==0.9.3
numpy==1.14.0
pandas==0.22.0
pip==9.0.1

mrshu

import pkg_resources
installed_packages = pkg_resources.working_set
for package in sorted(installed_packages, lambda x: x.key):
    print "{}=={}".format(package.key, package.version)

christopherhan

AWSTemplateFormatVersion: "2010-09-09"
Description: Put a SecureString parameter in SSM Parameter Store
Parameters:
  KmsKeyId:
    Type: String
    Description: The KMS Key you want to use to encrypt the string.
  SSMParameterKey:
    Type: String
    Description: The Parameter Store Key
  SSMParameterValue:

snixon

#!/usr/bin/env python

import os
import boto3
import argparse
import json
# `pip install -U PTable` will get you the right fork of PrettyTable
from prettytable import PrettyTable

from botocore.exceptions import ClientError

m4tthumphrey

<?php
/*
 * Plugin:        StreamlineFoundation
 *
 * Class:        Schedule
 *
 * Description:    Provides scheduling mechanics including creating a schedule, testing if a specific moment is part of the schedule, moving back
 *                and forth between scheduled moments in time and translating the created schedule back to a human readable form.
 *
 * Usage:        ::fromCronString() creates a new Schedule class and requires a string in the cron ('* * * * *', $language) format.

pofider

# .ebextensions/01-ebs.config
commands:
  01clear-if-unmounted:
    command: if ! mount | grep /media/ebs_volume > /dev/nul; then rm -rf /media/ebs_volume; fi
  02attach-volume:
    command: aws ec2 attach-volume --region eu-central-1 --volume-id vol-ddb08e34 --instance-id $(curl -s http://169.254.169.254/latest/meta-data/instance-id) --device /dev/sdh
    ignoreErrors: true
  03wait:
    command: sleep 10
  04trymount:

simecek

ozanturksever

    def get_uncompressed_size(self, file):
        fileobj = open(file, 'r')
        fileobj.seek(-8, 2)
        crc32 = gzip.read32(fileobj)
        isize = gzip.read32(fileobj)  # may exceed 2GB
        fileobj.close()
        return isize