Skip to content

Instantly share code, notes, and snippets.

@anna-geller
Last active February 3, 2023 14:52
Show Gist options
  • Save anna-geller/45c99852c22cc44fa260156b47339c0f to your computer and use it in GitHub Desktop.
Save anna-geller/45c99852c22cc44fa260156b47339c0f to your computer and use it in GitHub Desktop.
# search replace AWS_ACCOUNT_ID with your AWS account ID and adjust the variables below (line 3-7), especially your API key
# if your flow needs access to other AWS resources other than S3, add those in the task role policy: line 96-108
export AWS_REGION=us-east-1
export ECS_CLUSTER_NAME=prefectEcsCluster
export ECS_LOG_GROUP_NAME=/ecs/prefectEcsAgent
export ECS_SERVICE_NAME=prefectECSAgent
export PREFECT_API_KEY=your_Prefect_Cloud_API_key
export AWS_PAGER=""
aws ssm put-parameter --type SecureString --name PREFECT__CLOUD__API_KEY --value $PREFECT_API_KEY --region $AWS_REGION --overwrite
# adjust capacity providers to your needs
aws ecs create-cluster --cluster-name $ECS_CLUSTER_NAME --region $AWS_REGION
cat <<EOF >ecs_tasks_trust_policy.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ecs-tasks.amazonaws.com"
]
},
"Action": "sts:AssumeRole"
}
]
}
EOF
aws iam create-role --role-name prefectECSAgentTaskExecutionRole \
--assume-role-policy-document file://ecs_tasks_trust_policy.json --region $AWS_REGION
aws iam attach-role-policy --role-name prefectECSAgentTaskExecutionRole \
--policy-arn "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
cat <<EOF >ecs_tasks_execution_role.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ssm:GetParameters"
],
"Resource": "*"
}
]
}
EOF
aws iam put-role-policy --role-name prefectECSAgentTaskExecutionRole --policy-name prefectECSAgentTaskExecutionRolePolicy --policy-document file://ecs_tasks_execution_role.json
# permissions needed by Prefect to register new task definitions, deregister old ones, and create new flow runs as ECS tasks
cat <<EOF >ecs_task_role.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"ec2:DescribeVpcs",
"ec2:DeleteSecurityGroup",
"ecs:CreateCluster",
"ecs:DeleteCluster",
"ecs:DeregisterTaskDefinition",
"ecs:DescribeClusters",
"ecs:DescribeTaskDefinition",
"ecs:DescribeTasks",
"ecs:ListAccountSettings",
"ecs:ListClusters",
"ecs:ListTaskDefinitions",
"ecs:RegisterTaskDefinition",
"ecs:RunTask",
"ecs:StopTask",
"iam:PassRole",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogGroups",
"logs:GetLogEvents"
],
"Resource": "*"
}
]
}
EOF
# adjust it to include permissions needed by your flows
cat <<EOF >ecs_task_role_s3.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": "arn:aws:s3:::*prefect*"
}
]
}
EOF
aws iam create-role --role-name prefectTaskRole --assume-role-policy-document file://ecs_tasks_trust_policy.json --region $AWS_REGION
aws iam put-role-policy --role-name prefectTaskRole --policy-name prefectTaskRolePolicy --policy-document file://ecs_task_role.json
aws iam put-role-policy --role-name prefectTaskRole --policy-name prefectTaskRoleS3Policy --policy-document file://ecs_task_role_s3.json
aws logs create-log-group --log-group-name $ECS_LOG_GROUP_NAME --region $AWS_REGION
# search replace the "AWS_ACCOUNT_ID" below with your AWS account ID. Also, replace or add ECS Agent labels on line 140
cat <<EOF >prefect_ecs_agent_task_definition.json
{
"family": "$ECS_SERVICE_NAME",
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"cpu": "512",
"memory": "1024",
"taskRoleArn": "arn:aws:iam::AWS_ACCOUNT_ID:role/prefectTaskRole",
"executionRoleArn": "arn:aws:iam::AWS_ACCOUNT_ID:role/prefectECSAgentTaskExecutionRole",
"containerDefinitions": [
{
"name": "$ECS_SERVICE_NAME",
"image": "prefecthq/prefect:latest-python3.8",
"essential": true,
"command": [
"prefect",
"agent",
"ecs",
"start"
],
"environment": [
{
"name": "PREFECT__CLOUD__AGENT__LABELS",
"value": "['prod']"
},
{
"name": "PREFECT__CLOUD__AGENT__LEVEL",
"value": "INFO"
},
{
"name": "PREFECT__CLOUD__API",
"value": "https://api.prefect.io"
}
],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "$ECS_LOG_GROUP_NAME",
"awslogs-region": "$AWS_REGION",
"awslogs-stream-prefix": "ecs",
"awslogs-create-group": "true"
}
},
"secrets": [
{
"name": "PREFECT__CLOUD__API_KEY",
"valueFrom": "arn:aws:ssm:$AWS_REGION:AWS_ACCOUNT_ID:parameter/PREFECT__CLOUD__API_KEY"
}
]
}
]
}
EOF
aws ecs register-task-definition --cli-input-json file://prefect_ecs_agent_task_definition.json --region $AWS_REGION
export VPC=$(aws ec2 describe-vpcs --filters Name=is-default,Values=true)
export VPC_ID=$(echo $VPC | jq -r '.Vpcs | .[0].VpcId')
SUBNETS=$(aws ec2 describe-subnets --filters Name=vpc-id,Values=$VPC_ID --region $AWS_REGION)
export SUBNET1=$(echo $SUBNETS | jq -r '.Subnets | .[0].SubnetId')
export SUBNET2=$(echo $SUBNETS | jq -r '.Subnets | .[1].SubnetId')
export SUBNET3=$(echo $SUBNETS | jq -r '.Subnets | .[2].SubnetId')
aws ecs create-service \
--service-name $ECS_SERVICE_NAME\
--task-definition $ECS_SERVICE_NAME:1 \
--desired-count 1 \
--launch-type FARGATE \
--platform-version LATEST \
--cluster $ECS_CLUSTER_NAME \
--network-configuration awsvpcConfiguration="{subnets=[$SUBNET1, $SUBNET2, $SUBNET3],assignPublicIp=ENABLED}" --region $AWS_REGION
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment