boto2 forum example for boto3

forum.py

import boto3, hashlib, json, datetime, time, sys, getopt, threading, logging
import botocore
from boto3.dynamodb.conditions import Key, Attr

logging.getLogger('botocore').setLevel(logging.CRITICAL)
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.basicConfig(filename='forum.log',level=logging.INFO, format="%(threadName)s:%(message)s")

class APIerror(Exception):
    pass
class NotFound(Exception):
    pass
class ResourceInUse(Exception):
    pass
class ConnectionError(Exception):
    pass

_localdb = True

dynamodb = None
client = None

date_encoder = lambda obj: (
    obj.isoformat()
    if isinstance(obj, datetime.datetime)
    or isinstance(obj, datetime.date)
    else None
)

######

def createBotoConnection(localdb=_localdb):
    try:
        print("using dynammodb in default profile defined in ~/.aws/credentials")
        global dynamodb, client
        if localdb:
            logging.info(time.strftime("%H:%M:%S") + " - createBotoConnection - starting dynamodb connections - locally")
            dynamodb = boto3.resource('dynamodb', endpoint_url='http://localhost:9000')
            client = boto3.client('dynamodb', endpoint_url='http://localhost:9000')
        else:
            logging.info(time.strftime("%H:%M:%S") + " - createBotoConnection - starting dynamodb connections")
            dynamodb = boto3.resource('dynamodb', region_name='eu-west-1')
            client = boto3.client('dynamodb', region_name='eu-west-1')
    except Exception as e:
        raise botocore.exceptions.ConnectionError(str(e))


#def create_messages
createBotoConnection()

def table_p(dynamodb, tableName):
    return tableName in [table.name for table in list(dynamodb.tables.all())]

def create_forum(dynamodb, client):
    try:
        tableName = 'Forum'
        dynamodb.create_table(
            TableName=tableName,
            KeySchema=[
                { 'AttributeName': 'Name', 'KeyType': 'HASH' },
            ],
            AttributeDefinitions=[
                { 'AttributeName': 'Name', 'AttributeType': 'S' },
            ],
            ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 }    
        )
        waiter = client.get_waiter('table_exists')
        waiter.wait(TableName=tableName)
    except botocore.exceptions.ClientError as e:
        if e.args[0].find("Cannot create preexisting table") > 0:
            print("table already exists")
            

def create_thread(dynamodb, client):
    try:
        tableName = 'Thread'
        thread = dynamodb.create_table(
            TableName=tableName,
            KeySchema=[
                { 'AttributeName': 'ForumName', 'KeyType': 'HASH' },
                { 'AttributeName': 'Subject', 'KeyType': 'RANGE' },
            ],
            AttributeDefinitions=[
                { 'AttributeName': 'ForumName', 'AttributeType': 'S' },
                { 'AttributeName': 'Subject', 'AttributeType': 'S' },
            ],
            ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 }
        )
        waiter = client.get_waiter('table_exists')
        waiter.wait(TableName=tableName)
    except botocore.exceptions.ClientError as e:
        if e.args[0].find("Cannot create preexisting table") > 0:
            print("table already exists")


def create_reply(dynamodb, client):
    try:
        tableName='Reply'
        reply  = dynamodb.create_table(
            TableName=tableName,
            KeySchema=[
                { 'AttributeName': 'Id', 'KeyType': 'HASH' },
                { 'AttributeName': 'ReplyDateTime', 'KeyType': 'RANGE' },
            ],
            AttributeDefinitions=[
                { 'AttributeName': 'Id', 'AttributeType': 'S' },
                { 'AttributeName': 'ReplyDateTime', 'AttributeType': 'S' },
                { 'AttributeName': 'PostedBy', 'AttributeType': 'S' },
            ],
            LocalSecondaryIndexes=[
                {
                    'IndexName': 'PostedBy-Index',
                    'KeySchema': [
                        { 'AttributeName': 'Id', 'KeyType': 'HASH' },
                        { 'AttributeName': 'PostedBy', 'KeyType': 'RANGE' },
                    ],
                    'Projection': {
                        'ProjectionType': 'KEYS_ONLY',
                    }
                },
            ],
            ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 }
        )
        waiter = client.get_waiter('table_exists')
        waiter.wait(TableName=tableName)
    except botocore.exceptions.ClientError as e:
        if e.args[0].find("Cannot create preexisting table") > 0:
            print("table already exists")


def deleteTable(client, tableName):
    try:
        client.delete_table(TableName=tableName)
        waiter = client.get_waiter('table_does_not_exist')
        print("waiting for dynamodb table " + tableName + " to die")
        waiter.wait(TableName=tableName)
    except:
        print("dynamodb table " + tableName + " does not exist")
        pass


def loadSamplesForums(tableName):
    table = dynamodb.Table(tableName)
    try:
        table.put_item(
            Item={
                'Name': 'Amazon DynamoDB',
                'Category': 'Amazon Web Services',
                'Threads': 2,
                'Messages':4,
                'Views': 1000,
            }
        )
        table.put_item(
            Item={
                'Name': 'Amazon S3',
                'Category': 'Amazon Web Services',
                'Threads': 0,
            }
        )
        print("data loaded for Forum")
    except Exception as e:
        print("Failed to create item in " + tableName)
        print(str(e))


time.strftime("%Y-%m-%d:%H:%M:%S%z")

def postThread1(tableName):
    table = dynamodb.Table(tableName)
    try:
        date2 = datetime.datetime.fromtimestamp(time.time() - (14*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
        table.put_item(
            Item={
                'ForumName': 'Amazon DynamoDB',
                'Subject': "DynamoDB Thread 1",
                'Message': "DynamoDB thread 1 message",
                'LastPostedBy': "User A",
                'LastPostedDateTime': date2,
                'Views': 0,
                'Replies': 0,
                'Answered': 0,
                'Tags': {"index", "primarykey", "table"}         
            }
        )
    except Exception as e:
        print("Failed to create item in " + tableName)
        print(str(e))
    
def postThread2(tableName):
    table = dynamodb.Table(tableName)
    try:
        date3 = datetime.datetime.fromtimestamp(time.time() - (21*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
        table.put_item(
            Item={
                'ForumName': 'Amazon DynamoDB',
                'Subject': "DynamoDB Thread 2",
                'Message': "DynamoDB thread 2 message",
                'LastPostedBy': "User A",
                'LastPostedDateTime': date3,
                'Views': 0,
                'Replies': 0,
                'Answered': 0,
                'Tags': {"index", "primarykey", "table"}         
            }
        )
    except Exception as e:
        print("Failed to create item in " + tableName)
        print(str(e))
    
def postThread3(tableName):
    table = dynamodb.Table(tableName)
    try:
        date1 = datetime.datetime.fromtimestamp(time.time() - (7*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
        table.put_item(
            Item={
                'ForumName': 'Amazon S3',
                'Subject': "DynamoDB Thread 1",
                'Message': "DynamoDB thread 3 message",
                'LastPostedBy': "User A",
                'LastPostedDateTime': date1,
                'Views': 0,
                'Replies': 0,
                'Answered': 0,
                'Tags': {"largeobjects", "multipart upload"}         
            }
        )
    except Exception as e:
        print("Failed to create item in " + tableName)
        print(str(e))
    
    
def loadSamplesThreads(tableName):
    postThread1(tableName)
    postThread2(tableName)
    postThread3(tableName)
    print("data loaded for Threads")

#print(client.describe_table(TableName="Reply"))

def loadSamplesReplies(tableName):
    table = dynamodb.Table(tableName)
    date0 = datetime.datetime.fromtimestamp(time.time() - (1*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
    date1 = datetime.datetime.fromtimestamp(time.time() - (7*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
    date2 = datetime.datetime.fromtimestamp(time.time() - (14*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
    date3 = datetime.datetime.fromtimestamp(time.time() - (21*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z")
    try:
        table.put_item(
            Item={
                "Id": "Amazon DynamoDB#DynamoDB Thread 1",
                "ReplyDateTime": date3,
                "Message": "DynamoDB Thread 1 Reply 1 text",
                "PostedBy": "User A"
             }
        )
        table.put_item(
            Item={
                "Id": "Amazon DynamoDB#DynamoDB Thread 1",
                "ReplyDateTime": date2,
                "Message": "DynamoDB Thread 1 Reply 2 text",
                "PostedBy": "User A"
             }
        )
        table.put_item(
            Item={
                "Id": "Amazon DynamoDB#DynamoDB Thread 2",
                "ReplyDateTime": date1,
                "Message": "DynamoDB Thread 2 Reply 1 text",
                "PostedBy": "User A"
             }
        )
        table.put_item(
            Item={
                "Id": "Amazon DynamoDB#DynamoDB Thread 2",
                "ReplyDateTime": date0,
                "Message": "DynamoDB Thread 2 Reply 2 text",
                "PostedBy": "User A"
             }
        )
        print("data loaded for Reply")
    except Exception as e:
        print("Failed to create item in " + tableName)
        print(str(e))

def createForum(name, category):
    table = dynamodb.Table('Forum')
    try:
        table.put_item(
            Item={
                'Name': name,
                'Category': category,
                'Threads': 0,
            }
        )
    except Excpetion as e:
        print(str(e))


def createThread(forumName, subject, message, user, tags):
    table = dynamodb.Table('Thread')
    try:
        table.put_item(
            Item={
                'ForumName': forumName,
                'Subject': subject,
                'Message': message,
                'LastPostedBy': user,
                'LastPostedDateTime': time.strftime("%Y-%m-%d:%H:%M:%S%z"),
                'Views': 0,
                'Replies': 0,
                'Answered': 0,
                'Tags': tags         
            }
        )
    except Exception as e:
        print(str(e))


def postReply(forum, thread, user, message):
    table = dynamodb.Table('Reply')
    try:
        table.put_item(
            Item={
                'Id': forum + '#' + thread,
                'ReplyDateTime': time.strftime("%Y-%m-%d:%H:%M:%S%z"),
                'PostedBy': user,
                'Message': message,
             }
        )
    except Exception as e:
        print(str(e))


### forum
def getFora():
    table = dynamodb.Table('Forum')
    return table.scan()

def getForumNames():
    items = getFora()
    return  [item['Name'] for item in items['Items']]
        
def pretty(response):
    for item in response['Items']:
        print(item)

### thread

def getForumSubjects(forumName):
    table = dynamodb.Table('Thread')
    items = table.query( KeyConditionExpression=Key('ForumName').eq(forumName))
    return [item['Subject'] for item in items['Items']]

# getForumSubjects('Amazon DynamoDB')

### replies
def getReplies(forum, thread):
    response = None
    table = dynamodb.Table('Reply')
    try:
        response = table.query(IndexName='PostedBy-Index', TableName='Reply',
                               KeyConditionExpression=Key('Id').eq(forum+'#'+thread))
        response = response['Items']
    except Exception as e:
        print(str(e))
    return response

def getRepliesBy(forum, thread, user):
    response = None
    table = dynamodb.Table('Reply')
    try:
        response = table.query(IndexName='PostedBy-Index', TableName='Reply',
                               KeyConditionExpression=Key('Id').eq(forum+'#'+thread) &
                               Key('PostedBy').eq(user))
    except Exception as e:
        print(str(e))
    return response

def getReplyByDate(forum, thread, exact_date):
    response = None
    table = dynamodb.Table('Reply')
    try:
        response = table.query(TableName='Reply',
                               KeyConditionExpression=Key('Id').eq(forum+'#'+thread)
                               & Key('ReplyDateTime').eq(exact_date))
    except Exception as e:
        print(str(e))
    return response

def getReplyBetweenDates(forum, thread, start_date, end_date):
    response = None
    table = dynamodb.Table('Reply')
    try:
        response = table.query(TableName='Reply',
                               KeyConditionExpression=Key('Id').eq(forum+'#'+thread)
                               & Key('ReplyDateTime').between(start_date, end_date))
    except Exception as e:
        print(str(e))
    return response

# getReplies("Amazon DynamoDB", "DynamoDB Thread 1")
# getRepliesBy("Amazon DynamoDB", "DynamoDB Thread 1", "User A")
# getReplyByDate("Amazon DynamoDB", "DynamoDB Thread 1",'2016-02-05:10:58:13')
# getReplyBetweenDates("Amazon DynamoDB", "DynamoDB Thread 1",'2016-02-05:10:58:12','2016-02-05:10:58:15')

### Here be Dragons ###


create_forum(dynamodb, client)
create_thread(dynamodb, client)
create_reply(dynamodb, client)

#client.delete_table(TableName='Reply')
#client.delete_table(TableName='Forum')
#client.delete_table(TableName='Thread')

#print(list(dynamodb.tables.all()))

#print(client.describe_table(TableName="Forum"))

loadSamplesForums("Forum")
loadSamplesThreads("Thread")
loadSamplesReplies("Reply")


#postReply("Amazon DynamoDB", "DynamoDB Thread 2", "User B", "DynamoDB Thread 2 Reply 2 text")
createForum('Amazon EC2', 'Amazon Web Services')
createThread('Amazon EC2', 'Instance types', 'There are several..', 'User B', tags = {"index", "primarykey", "table"})
postReply("Amazon EC2", "Instance types", "User B", "first post")

createThread('Amazon S3', 'S3 Thread 1', 'S3 the storage for everybody ', 'User B', tags = {"index", "primarykey", "table"})

postReply("Amazon S3", "S3 Thread 1", "User B", "S3 Thread 1 Reply 1 text")
postReply("Amazon S3", "DynamoDB Thread 1", "User B", "this again?")

fora = getForumNames()
print("created fora " + str(fora))
print("created subjects")
for forum in fora:
    subjects = getForumSubjects(forum)
    print(forum + " --> " + str(subjects))
    for subject in subjects:
        replies = getReplies(forum, subject)
        for reply in replies:
            print(subject + " --> " + str(reply))