Immersion Day

emr.md

Labs

• Lab 1 - https://s3.amazonaws.com/dreis-id/emr/lab_01.pdf
• Lab 2 - https://s3.amazonaws.com/dreis-id/emr/lab_02.pdf
• Lab Kinesis - https://s3-us-west-2.amazonaws.com/abd321/ABD321.html

id_cleanup.py

import boto3
import json
from datetime import datetime, timedelta, date
import traceback

date_ref = datetime.today() - timedelta(days=3)

REGIONS = ['us-east-1', 'us-west-2', 'us-east-2', 'eu-west-1', 'eu-west-2']
PROFILES = ['ws01', 'ws02', 'ws03', 'ws04', 'ws05', 'ws06']
#PROFILES = ['ws01', ]

iam = boto3.client('iam')

def json_serial(obj):
    """JSON serializer for objects not serializable by default json code"""

    if isinstance(obj, (datetime, date)):
        return obj.isoformat()
    raise TypeError ("Type %s not serializable" % type(obj))


def delete_user(user):
    print("\n>delete_user()")

    try:
        r = iam.list_access_keys(
           UserName=user,
        )

        for key in r['AccessKeyMetadata']:
            print("Removing key {}...".format(key['AccessKeyId']))
            r = iam.delete_access_key(
                UserName=user,
                AccessKeyId=key['AccessKeyId']
            )
    except Exception as e:
        print(e)

    try:
        r = iam.remove_user_from_group(
            GroupName='workshop',
            UserName=user
        )
    except Exception as e:
        print(e)

    try:
        r = iam.delete_login_profile(
            UserName=user
        )
    except Exception as e:
        print(e)

    try:
        r = iam.delete_user(
            UserName=user
        )
    except Exception as e:
        print(e)


def clean_cfn():
    for region in REGIONS:

        print("------")
        cfn = boto3.client('cloudformation', region_name=region)
        r = cfn.list_stacks()
        waiter = cfn.get_waiter('stack_delete_complete')

        for stacks_summary in r['StackSummaries']:
            print(json.dumps(stacks_summary, default=json_serial))
            if stacks_summary['StackStatus'] == 'CREATE_COMPLETE' and 'SAMPLE' not in stacks_summary['StackName'].upper():
                print("CFN - {} - Removing stack '{}'".format(region, stacks_summary['StackId']))
                cfn.delete_stack(StackName=stacks_summary['StackName'])
                print("CFN - {} - Waiting stack '{}'".format(region, stacks_summary['StackId']))
                waiter.wait(StackName=stacks_summary['StackName'])
                print("CFN - {} - Deleted stack '{}'".format(region, stacks_summary['StackId']))


def remove_ddb_tables():
    print("\n>remove_ddb_tables()")
    for region in REGIONS:
        print("------")
        ddb = boto3.client('dynamodb', region_name=region)

        r = ddb.list_tables()

        for table in r['TableNames']:
            print("DDB - {} - Removing table '{}'".format(region, table))
            ddb.delete_table(TableName=table)


def remove_buckets():
    for region in REGIONS:
        s3 = boto3.client('s3', region_name=region)

        r = s3.list_buckets()

        for bucket in r['Buckets']:
            bucket_name=bucket['Name']

            if bucket['CreationDate'].replace(tzinfo=None) > date_ref:
                if 'do-not-delete' not in bucket_name.lower():
                    try:
                        print("S3 - {} - DELETING bucket '{}'".format(region, bucket['Name']))

                        s3_res = boto3.resource('s3')
                        b = s3_res.Bucket(bucket['Name'])
                        b.objects.all().delete()

                        r = s3.delete_bucket(Bucket=bucket['Name'])
                    except Exception as e:
                        print(e)
            else:
                print("S3 - {} - Skipping bucket '{}'".format(region, bucket['Name']))


def cleanup_kinesis_analytics():
    print("\n>cleanup_kinesis_analytics")

    for region in REGIONS:
        print("------")
        try:
            kin = boto3.client('kinesisanalyticsv2', region_name=region)
            streams = kin.list_applications()

            for app in streams['ApplicationSummaries']:
                app_name = app['ApplicationName']

                print("KIN: Deleting app {}".format(app_name))

                r = kin.describe_application(
                    ApplicationName=app_name,
                    IncludeAdditionalDetails=True
                )

                create_timestamp = r['ApplicationDetail']['CreateTimestamp']

                response = kin.delete_application(
                    ApplicationName=app_name,
                    CreateTimestamp=create_timestamp
                )

                print("Response: {}".format(response))

        except Exception as e:
            print("ERRO: cleanup_kinesis_analytics")
            traceback.print_exc()
            print(e)


def cleanup_kinesis():
    print("\n>cleanup_kinesis")

    for region in REGIONS:
        print("------")
        try:
            kin = boto3.client('kinesis', region_name=region)
            streams = kin.list_streams()

            for stream in streams['StreamNames']:
                print("KIN: Deleting stream {}".format(stream))
                response = kin.delete_stream(StreamName=stream)

        except Exception as e:
            print(e)


def cleanup_lambda():
    print("\n>cleanup_lambda")

    for region in REGIONS:
        print("------")
        try:
            lbd = boto3.client('lambda', region_name=region)
            funcs = lbd.list_functions(FunctionVersion='ALL')

            for f in funcs['Functions']:
                print("LBD: Deleting function {}".format(f['FunctionName']))
                response = lbd.delete_function(FunctionName=f['FunctionName'])

        except Exception as e:
            print(e)


def remove_collections():
    print("\n>remove_collections")
    for region in REGIONS:
        print("------")
        try:
            rek = boto3.client('rekognition', region_name=region)
            collections = rek.list_collections()

            for col in collections['CollectionIds']:
                print("REK: Deleting collection {}".format(col))
                rek.delete_collection(
                    CollectionId=col
                )

        except Exception as e:
            print(e)


def cleanup_api_gateway():
    print("\n>cleanup_api_gateway")

    for region in REGIONS:
        print("------")
        try:
            api_cli = boto3.client('apigateway', region_name=region)
            apis = api_cli.get_rest_apis()

            for api in apis['items']:
                print("API - {} - Deleting API {}".format(region, api['id']))
                response = api_cli.delete_rest_api(
                    restApiId=api['id']
                )

        except Exception as e:
            print(e)


if __name__ == "__main__":

    for profile in PROFILES:
        boto3.setup_default_session(profile_name=profile)
        clean_cfn()
        cleanup_kinesis_analytics()
        delete_user('virginia')
        delete_user('ohio')
        delete_user('oregon')
        delete_user('ireland')
        delete_user('london')
        remove_ddb_tables()
        remove_buckets()
        remove_collections()
        cleanup_kinesis()
        cleanup_lambda()
        cleanup_api_gateway()

        #remove api gtw

prep_env.sh

#!/bin/bash

echo Script name: $0 $1

echo $# arguments 

if [ $# -ne 1 ]; 
    then echo "Usage: $0 <password>"
    exit 0
fi

function prep_users {
    aws iam create-group \
    --group-name workshop \
    --profile $1

    aws iam attach-group-policy \
    --group-name workshop \
    --policy-arn arn:aws:iam::aws:policy/AdministratorAccess \
    --profile $1

    aws iam create-user --user-name ireland --profile $1
    aws iam create-user --user-name ohio --profile $1
    aws iam create-user --user-name oregon --profile $1
    aws iam create-user --user-name virginia --profile $1
    aws iam create-user --user-name london --profile $1

    aws iam create-login-profile \
    --user-name ireland --password $2 \
    --profile $1

    aws iam create-login-profile \
    --user-name ohio --password $2 \
    --profile $1

    aws iam create-login-profile \
    --user-name oregon --password $2 \
    --profile $1

    aws iam create-login-profile \
    --user-name virginia --password $2 \
    --profile $1

    aws iam create-login-profile \
    --user-name london --password $2 \
    --profile $1

    aws iam add-user-to-group \
    --user-name ireland \
    --group-name workshop \
    --profile $1

    aws iam add-user-to-group \
    --user-name ohio \
    --group-name workshop \
    --profile $1

    aws iam add-user-to-group \
    --user-name oregon \
    --group-name workshop \
    --profile $1

    aws iam add-user-to-group \
    --user-name virginia \
    --group-name workshop \
    --profile $1

    aws iam add-user-to-group \
    --user-name london \
    --group-name workshop \
    --profile $1
}

prep_users ws01 $1
prep_users ws02 $1
prep_users ws03 $1
prep_users ws04 $1
prep_users ws05 $1