Created
October 2, 2016 00:04
-
-
Save tajchert/f23c855e4f3c27aa33aed3e283630ab3 to your computer and use it in GitHub Desktop.
Saving multiple values to DynamoDb at Amazon AWS - splitting list of items to be save so it doesn't exceed limits of one request (28, 25 is just for safety), as well with checking if there are no items left to saved - leftovers due to exceeding throughput limits. Might be useful for people working with Amazon DynamoDb and Java SDK.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
private static void writeDynamoMultipleItems(ArrayList<Item> itemsBatchWrite, String tableName, DynamoDB dynamoDB) { | |
System.out.println("Write to DynamoDB with " + itemsBatchWrite.size() + " items"); | |
if (itemsBatchWrite.size() > 25) { | |
System.out.println("Splitting table"); | |
ArrayList<Item> writeItems = new ArrayList<>(); | |
for (Item item : itemsBatchWrite) { | |
if (writeItems.size() < 25) { | |
writeItems.add(item); | |
} else { | |
writeDynamoMultipleItems(writeItems, tableName, dynamoDB); | |
writeItems.clear(); | |
} | |
} | |
if (writeItems.size() > 0) { | |
writeDynamoMultipleItems(writeItems, tableName, dynamoDB); | |
writeItems.clear(); | |
} | |
return; | |
} | |
TableWriteItems steamPricesTableWriteItems = new TableWriteItems(tableName); | |
steamPricesTableWriteItems.withItemsToPut(itemsBatchWrite); | |
BatchWriteItemOutcome outcome = dynamoDB.batchWriteItem(steamPricesTableWriteItems); | |
itemsBatchWrite.clear(); | |
do {// Check for unprocessed keys which could happen if you exceed provisioned throughput | |
Map<String, List<WriteRequest>> unprocessedItems = outcome.getUnprocessedItems(); | |
if (outcome.getUnprocessedItems().size() == 0) { | |
System.out.println("No unprocessed items found"); | |
} else { | |
System.out.println("Retrieving the unprocessed items"); | |
outcome = dynamoDB.batchWriteItemUnprocessed(unprocessedItems); | |
} | |
} while (outcome.getUnprocessedItems().size() > 0); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Append this line of code after line number # 11.
writeItems.add(item);