Created
February 29, 2016 10:12
-
-
Save klang/3aaed43e595521e6b8d2 to your computer and use it in GitHub Desktop.
boto2 forum example for boto3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3, hashlib, json, datetime, time, sys, getopt, threading, logging | |
import botocore | |
from boto3.dynamodb.conditions import Key, Attr | |
logging.getLogger('botocore').setLevel(logging.CRITICAL) | |
logging.getLogger('boto3').setLevel(logging.CRITICAL) | |
logging.basicConfig(filename='forum.log',level=logging.INFO, format="%(threadName)s:%(message)s") | |
class APIerror(Exception): | |
pass | |
class NotFound(Exception): | |
pass | |
class ResourceInUse(Exception): | |
pass | |
class ConnectionError(Exception): | |
pass | |
_localdb = True | |
dynamodb = None | |
client = None | |
date_encoder = lambda obj: ( | |
obj.isoformat() | |
if isinstance(obj, datetime.datetime) | |
or isinstance(obj, datetime.date) | |
else None | |
) | |
###### | |
def createBotoConnection(localdb=_localdb): | |
try: | |
print("using dynammodb in default profile defined in ~/.aws/credentials") | |
global dynamodb, client | |
if localdb: | |
logging.info(time.strftime("%H:%M:%S") + " - createBotoConnection - starting dynamodb connections - locally") | |
dynamodb = boto3.resource('dynamodb', endpoint_url='http://localhost:9000') | |
client = boto3.client('dynamodb', endpoint_url='http://localhost:9000') | |
else: | |
logging.info(time.strftime("%H:%M:%S") + " - createBotoConnection - starting dynamodb connections") | |
dynamodb = boto3.resource('dynamodb', region_name='eu-west-1') | |
client = boto3.client('dynamodb', region_name='eu-west-1') | |
except Exception as e: | |
raise botocore.exceptions.ConnectionError(str(e)) | |
#def create_messages | |
createBotoConnection() | |
def table_p(dynamodb, tableName): | |
return tableName in [table.name for table in list(dynamodb.tables.all())] | |
def create_forum(dynamodb, client): | |
try: | |
tableName = 'Forum' | |
dynamodb.create_table( | |
TableName=tableName, | |
KeySchema=[ | |
{ 'AttributeName': 'Name', 'KeyType': 'HASH' }, | |
], | |
AttributeDefinitions=[ | |
{ 'AttributeName': 'Name', 'AttributeType': 'S' }, | |
], | |
ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } | |
) | |
waiter = client.get_waiter('table_exists') | |
waiter.wait(TableName=tableName) | |
except botocore.exceptions.ClientError as e: | |
if e.args[0].find("Cannot create preexisting table") > 0: | |
print("table already exists") | |
def create_thread(dynamodb, client): | |
try: | |
tableName = 'Thread' | |
thread = dynamodb.create_table( | |
TableName=tableName, | |
KeySchema=[ | |
{ 'AttributeName': 'ForumName', 'KeyType': 'HASH' }, | |
{ 'AttributeName': 'Subject', 'KeyType': 'RANGE' }, | |
], | |
AttributeDefinitions=[ | |
{ 'AttributeName': 'ForumName', 'AttributeType': 'S' }, | |
{ 'AttributeName': 'Subject', 'AttributeType': 'S' }, | |
], | |
ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } | |
) | |
waiter = client.get_waiter('table_exists') | |
waiter.wait(TableName=tableName) | |
except botocore.exceptions.ClientError as e: | |
if e.args[0].find("Cannot create preexisting table") > 0: | |
print("table already exists") | |
def create_reply(dynamodb, client): | |
try: | |
tableName='Reply' | |
reply = dynamodb.create_table( | |
TableName=tableName, | |
KeySchema=[ | |
{ 'AttributeName': 'Id', 'KeyType': 'HASH' }, | |
{ 'AttributeName': 'ReplyDateTime', 'KeyType': 'RANGE' }, | |
], | |
AttributeDefinitions=[ | |
{ 'AttributeName': 'Id', 'AttributeType': 'S' }, | |
{ 'AttributeName': 'ReplyDateTime', 'AttributeType': 'S' }, | |
{ 'AttributeName': 'PostedBy', 'AttributeType': 'S' }, | |
], | |
LocalSecondaryIndexes=[ | |
{ | |
'IndexName': 'PostedBy-Index', | |
'KeySchema': [ | |
{ 'AttributeName': 'Id', 'KeyType': 'HASH' }, | |
{ 'AttributeName': 'PostedBy', 'KeyType': 'RANGE' }, | |
], | |
'Projection': { | |
'ProjectionType': 'KEYS_ONLY', | |
} | |
}, | |
], | |
ProvisionedThroughput={ 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } | |
) | |
waiter = client.get_waiter('table_exists') | |
waiter.wait(TableName=tableName) | |
except botocore.exceptions.ClientError as e: | |
if e.args[0].find("Cannot create preexisting table") > 0: | |
print("table already exists") | |
def deleteTable(client, tableName): | |
try: | |
client.delete_table(TableName=tableName) | |
waiter = client.get_waiter('table_does_not_exist') | |
print("waiting for dynamodb table " + tableName + " to die") | |
waiter.wait(TableName=tableName) | |
except: | |
print("dynamodb table " + tableName + " does not exist") | |
pass | |
def loadSamplesForums(tableName): | |
table = dynamodb.Table(tableName) | |
try: | |
table.put_item( | |
Item={ | |
'Name': 'Amazon DynamoDB', | |
'Category': 'Amazon Web Services', | |
'Threads': 2, | |
'Messages':4, | |
'Views': 1000, | |
} | |
) | |
table.put_item( | |
Item={ | |
'Name': 'Amazon S3', | |
'Category': 'Amazon Web Services', | |
'Threads': 0, | |
} | |
) | |
print("data loaded for Forum") | |
except Exception as e: | |
print("Failed to create item in " + tableName) | |
print(str(e)) | |
time.strftime("%Y-%m-%d:%H:%M:%S%z") | |
def postThread1(tableName): | |
table = dynamodb.Table(tableName) | |
try: | |
date2 = datetime.datetime.fromtimestamp(time.time() - (14*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
table.put_item( | |
Item={ | |
'ForumName': 'Amazon DynamoDB', | |
'Subject': "DynamoDB Thread 1", | |
'Message': "DynamoDB thread 1 message", | |
'LastPostedBy': "User A", | |
'LastPostedDateTime': date2, | |
'Views': 0, | |
'Replies': 0, | |
'Answered': 0, | |
'Tags': {"index", "primarykey", "table"} | |
} | |
) | |
except Exception as e: | |
print("Failed to create item in " + tableName) | |
print(str(e)) | |
def postThread2(tableName): | |
table = dynamodb.Table(tableName) | |
try: | |
date3 = datetime.datetime.fromtimestamp(time.time() - (21*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
table.put_item( | |
Item={ | |
'ForumName': 'Amazon DynamoDB', | |
'Subject': "DynamoDB Thread 2", | |
'Message': "DynamoDB thread 2 message", | |
'LastPostedBy': "User A", | |
'LastPostedDateTime': date3, | |
'Views': 0, | |
'Replies': 0, | |
'Answered': 0, | |
'Tags': {"index", "primarykey", "table"} | |
} | |
) | |
except Exception as e: | |
print("Failed to create item in " + tableName) | |
print(str(e)) | |
def postThread3(tableName): | |
table = dynamodb.Table(tableName) | |
try: | |
date1 = datetime.datetime.fromtimestamp(time.time() - (7*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
table.put_item( | |
Item={ | |
'ForumName': 'Amazon S3', | |
'Subject': "DynamoDB Thread 1", | |
'Message': "DynamoDB thread 3 message", | |
'LastPostedBy': "User A", | |
'LastPostedDateTime': date1, | |
'Views': 0, | |
'Replies': 0, | |
'Answered': 0, | |
'Tags': {"largeobjects", "multipart upload"} | |
} | |
) | |
except Exception as e: | |
print("Failed to create item in " + tableName) | |
print(str(e)) | |
def loadSamplesThreads(tableName): | |
postThread1(tableName) | |
postThread2(tableName) | |
postThread3(tableName) | |
print("data loaded for Threads") | |
#print(client.describe_table(TableName="Reply")) | |
def loadSamplesReplies(tableName): | |
table = dynamodb.Table(tableName) | |
date0 = datetime.datetime.fromtimestamp(time.time() - (1*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
date1 = datetime.datetime.fromtimestamp(time.time() - (7*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
date2 = datetime.datetime.fromtimestamp(time.time() - (14*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
date3 = datetime.datetime.fromtimestamp(time.time() - (21*24*60*60)).strftime("%Y-%m-%d:%H:%M:%S%z") | |
try: | |
table.put_item( | |
Item={ | |
"Id": "Amazon DynamoDB#DynamoDB Thread 1", | |
"ReplyDateTime": date3, | |
"Message": "DynamoDB Thread 1 Reply 1 text", | |
"PostedBy": "User A" | |
} | |
) | |
table.put_item( | |
Item={ | |
"Id": "Amazon DynamoDB#DynamoDB Thread 1", | |
"ReplyDateTime": date2, | |
"Message": "DynamoDB Thread 1 Reply 2 text", | |
"PostedBy": "User A" | |
} | |
) | |
table.put_item( | |
Item={ | |
"Id": "Amazon DynamoDB#DynamoDB Thread 2", | |
"ReplyDateTime": date1, | |
"Message": "DynamoDB Thread 2 Reply 1 text", | |
"PostedBy": "User A" | |
} | |
) | |
table.put_item( | |
Item={ | |
"Id": "Amazon DynamoDB#DynamoDB Thread 2", | |
"ReplyDateTime": date0, | |
"Message": "DynamoDB Thread 2 Reply 2 text", | |
"PostedBy": "User A" | |
} | |
) | |
print("data loaded for Reply") | |
except Exception as e: | |
print("Failed to create item in " + tableName) | |
print(str(e)) | |
def createForum(name, category): | |
table = dynamodb.Table('Forum') | |
try: | |
table.put_item( | |
Item={ | |
'Name': name, | |
'Category': category, | |
'Threads': 0, | |
} | |
) | |
except Excpetion as e: | |
print(str(e)) | |
def createThread(forumName, subject, message, user, tags): | |
table = dynamodb.Table('Thread') | |
try: | |
table.put_item( | |
Item={ | |
'ForumName': forumName, | |
'Subject': subject, | |
'Message': message, | |
'LastPostedBy': user, | |
'LastPostedDateTime': time.strftime("%Y-%m-%d:%H:%M:%S%z"), | |
'Views': 0, | |
'Replies': 0, | |
'Answered': 0, | |
'Tags': tags | |
} | |
) | |
except Exception as e: | |
print(str(e)) | |
def postReply(forum, thread, user, message): | |
table = dynamodb.Table('Reply') | |
try: | |
table.put_item( | |
Item={ | |
'Id': forum + '#' + thread, | |
'ReplyDateTime': time.strftime("%Y-%m-%d:%H:%M:%S%z"), | |
'PostedBy': user, | |
'Message': message, | |
} | |
) | |
except Exception as e: | |
print(str(e)) | |
### forum | |
def getFora(): | |
table = dynamodb.Table('Forum') | |
return table.scan() | |
def getForumNames(): | |
items = getFora() | |
return [item['Name'] for item in items['Items']] | |
def pretty(response): | |
for item in response['Items']: | |
print(item) | |
### thread | |
def getForumSubjects(forumName): | |
table = dynamodb.Table('Thread') | |
items = table.query( KeyConditionExpression=Key('ForumName').eq(forumName)) | |
return [item['Subject'] for item in items['Items']] | |
# getForumSubjects('Amazon DynamoDB') | |
### replies | |
def getReplies(forum, thread): | |
response = None | |
table = dynamodb.Table('Reply') | |
try: | |
response = table.query(IndexName='PostedBy-Index', TableName='Reply', | |
KeyConditionExpression=Key('Id').eq(forum+'#'+thread)) | |
response = response['Items'] | |
except Exception as e: | |
print(str(e)) | |
return response | |
def getRepliesBy(forum, thread, user): | |
response = None | |
table = dynamodb.Table('Reply') | |
try: | |
response = table.query(IndexName='PostedBy-Index', TableName='Reply', | |
KeyConditionExpression=Key('Id').eq(forum+'#'+thread) & | |
Key('PostedBy').eq(user)) | |
except Exception as e: | |
print(str(e)) | |
return response | |
def getReplyByDate(forum, thread, exact_date): | |
response = None | |
table = dynamodb.Table('Reply') | |
try: | |
response = table.query(TableName='Reply', | |
KeyConditionExpression=Key('Id').eq(forum+'#'+thread) | |
& Key('ReplyDateTime').eq(exact_date)) | |
except Exception as e: | |
print(str(e)) | |
return response | |
def getReplyBetweenDates(forum, thread, start_date, end_date): | |
response = None | |
table = dynamodb.Table('Reply') | |
try: | |
response = table.query(TableName='Reply', | |
KeyConditionExpression=Key('Id').eq(forum+'#'+thread) | |
& Key('ReplyDateTime').between(start_date, end_date)) | |
except Exception as e: | |
print(str(e)) | |
return response | |
# getReplies("Amazon DynamoDB", "DynamoDB Thread 1") | |
# getRepliesBy("Amazon DynamoDB", "DynamoDB Thread 1", "User A") | |
# getReplyByDate("Amazon DynamoDB", "DynamoDB Thread 1",'2016-02-05:10:58:13') | |
# getReplyBetweenDates("Amazon DynamoDB", "DynamoDB Thread 1",'2016-02-05:10:58:12','2016-02-05:10:58:15') | |
### Here be Dragons ### | |
create_forum(dynamodb, client) | |
create_thread(dynamodb, client) | |
create_reply(dynamodb, client) | |
#client.delete_table(TableName='Reply') | |
#client.delete_table(TableName='Forum') | |
#client.delete_table(TableName='Thread') | |
#print(list(dynamodb.tables.all())) | |
#print(client.describe_table(TableName="Forum")) | |
loadSamplesForums("Forum") | |
loadSamplesThreads("Thread") | |
loadSamplesReplies("Reply") | |
#postReply("Amazon DynamoDB", "DynamoDB Thread 2", "User B", "DynamoDB Thread 2 Reply 2 text") | |
createForum('Amazon EC2', 'Amazon Web Services') | |
createThread('Amazon EC2', 'Instance types', 'There are several..', 'User B', tags = {"index", "primarykey", "table"}) | |
postReply("Amazon EC2", "Instance types", "User B", "first post") | |
createThread('Amazon S3', 'S3 Thread 1', 'S3 the storage for everybody ', 'User B', tags = {"index", "primarykey", "table"}) | |
postReply("Amazon S3", "S3 Thread 1", "User B", "S3 Thread 1 Reply 1 text") | |
postReply("Amazon S3", "DynamoDB Thread 1", "User B", "this again?") | |
fora = getForumNames() | |
print("created fora " + str(fora)) | |
print("created subjects") | |
for forum in fora: | |
subjects = getForumSubjects(forum) | |
print(forum + " --> " + str(subjects)) | |
for subject in subjects: | |
replies = getReplies(forum, subject) | |
for reply in replies: | |
print(subject + " --> " + str(reply)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment