Skip to content

Instantly share code, notes, and snippets.

@markymarkus
Last active March 18, 2022 09:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save markymarkus/ab8dd0511374eb84d0efd872dc2c0291 to your computer and use it in GitHub Desktop.
Save markymarkus/ab8dd0511374eb84d0efd872dc2c0291 to your computer and use it in GitHub Desktop.
Step Functions with paginate
---
Parameters:
pBucketName:
Type: String
Default: ''
Description: Input S3 Bucket
Resources:
StateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
DefinitionString: |-
{
"Comment": "List S3 objects.",
"StartAt": "list_s3",
"States": {
"list_s3": {
"Comment": "Get first batch of objects.",
"Type": "Task",
"Resource": "arn:aws:states:::aws-sdk:s3:listObjectsV2",
"ResultPath": "$.s3_objects",
"Parameters": {
"Bucket": "${BucketName}",
"MaxKeys": ${BatchSize}
},
"Next": "process_s3_objects"
},
"process_s3_objects": {
"Comment": "Processing logic. Now we just wait.",
"Type": "Wait",
"Seconds": 2,
"Next": "check_if_all_listed"
},
"check_if_all_listed": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.s3_objects.IsTruncated",
"BooleanEquals": false,
"Next": "success_state"
}
],
"Default": "list_s3_with_continuation_token"
},
"list_s3_with_continuation_token": {
"Comment": "Get next batch of objects. Provide ContinuationToken in the request.",
"Type": "Task",
"Resource": "arn:aws:states:::aws-sdk:s3:listObjectsV2",
"ResultPath": "$.s3_objects",
"Parameters": {
"Bucket": "${BucketName}",
"MaxKeys": ${BatchSize},
"ContinuationToken.$": "$.s3_objects.NextContinuationToken"
},
"Next": "process_s3_objects"
},
"success_state": {
"Type": "Succeed"
}
}
}
DefinitionSubstitutions:
BucketName: !Ref pBucketName
BatchSize: 10
RoleArn: !GetAtt StatesExecutionRole.Arn
StatesExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: "Allow"
Principal:
Service:
- !Sub states.${AWS::Region}.amazonaws.com
Action: "sts:AssumeRole"
Path: "/"
Policies:
- PolicyName: StatesExecutionPolicy
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- 's3:ListBucket'
Resource: '*'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment