Run 6 ECS tasks with lambda triggered by Eventbridge scheduler

ecs_run_tasks.MD

Here is the AWS CLI command to create an ECS task definition:

cat <<EOF> task-definition.json
{
  "family": "eb-start-ecs-stask",
    "taskRoleArn": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ecsTaskExecutionRole",
    "executionRoleArn": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ecsTaskExecutionRole",
    "networkMode": "bridge",
    "containerDefinitions": [
      {
        "name": "eb-start-ecs-stask",
        "image": "httpd:2.4",
        "logConfiguration": {
          "logDriver": "awslogs",
          "options": {
            "awslogs-group": "/ecs/eb-start-ecs-stask",
            "awslogs-region": "us-east-1",
            "awslogs-create-group": "true",
            "awslogs-stream-prefix": "ecs"
          }
        },
        "portMappings": [
          {
            "containerPort": 80,
            "hostPort": 80
          }
        ],
        "essential": true,
        "memory": 256,
        "entryPoint": ["sh", "-c"],
        "command": ["echo 'Hello'; sleep 5"]
      }
    ],
    "cpu": "256",
    "memory": "512",
    "requiresCompatibilities": [ 
            "EC2"
          ]
}
EOF

aws ecs register-task-definition --cli-input-json file://task-definition.json

To create an IAM role that will be associated with your Lambda function, you can use the following AWS CLI command:

• https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html

export ROLE_NAME=A_eb-ecs-start-task-lambda-role

cat <<EOF> assume-role-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Principal": {
          "Service": "lambda.amazonaws.com"
        },
        "Action": "sts:AssumeRole"
      }
    ]
  }
EOF

ROLE_ARN=$(aws iam create-role \
  --role-name ${ROLE_NAME} \
  --assume-role-policy-document file://assume-role-policy.json \
  --query 'Role.Arn' --output text)

aws iam attach-role-policy \
  --role-name ${ROLE_NAME} \
  --policy-arn arn:aws:iam::aws:policy/AWSLambdaExecute

Here is an example policy document that grants the ecs:RunTask permission to the lambda-role IAM role:

cat <<EOF> ecs-run-task-policy.json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": "ecs:RunTask",
      "Resource": "arn:aws:ecs:${AWS_REGION}:${AWS_ACCOUNT_ID}:task-definition/*"
    }
  ]
}
EOF

aws iam put-role-policy \
  --role-name ${ROLE_NAME} \
  --policy-name ecs-run-task-policy \
  --policy-document file://ecs-run-task-policy.json

  cat <<EOF> passrole.json  
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": "iam:PassRole",
      "Resource": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ecsTaskExecutionRole"
    }
  ]
}
EOF

aws iam put-role-policy \
  --role-name  ${ROLE_NAME} \
  --policy-name A_eb-PassRolePolicy \
  --policy-document file://passrole.json

Here is some sample code that demonstrates how to start an ECS task from a Lambda function using the boto3 library:

export CLUSTER_NAME=<CLUSTER_NAME>

cat <<EOF> start_ecs_tasks.py
import boto3
import random
import string
import time

def start_ecs_tasks(event, context):
        # Create an ECS client
    ecs_client = boto3.client('ecs')
        # Generate a random string to use as the container name
    def generate_random_name():
        return ''.join(random.choices(string.ascii_lowercase, k=3))
        
    random_name = generate_random_name()
    for i in range(1,7):    
        # Start the ECS tasks
        response = ecs_client.run_task(
            startedBy='eventbridge_' + random_name + '_' + str(i),
            cluster='${CLUSTER_NAME}',
            taskDefinition='eb-start-ecs-stask',
            launchType='EC2',
            overrides={
                'containerOverrides': [
                    {
                    'name': 'eb-start-ecs-stask',
                    'command': ['sleep 2;echo ${RANDOM_NAME};env;sleep 2'],
                    'environment': [
                        {
                        'name': 'RANDOM_NAME',
                        'value': random_name + ' count=' + str(i)
                        },
                        ],
                    },
                ],
                },
                    
            )
        print(i)
        time.sleep(12)
    print('End of Loop')
EOF

Zip the start_ecs_task

zip -r function.zip start_ecs_tasks.py

To create a Lambda function that starts the ECS tasks, you can use the following AWS CLI command:

FUNCTION_ARN=$(aws lambda create-function \
    --function-name eb-start-ecs-tasks-func \
    --runtime python3.8 \
    --handler start_ecs_tasks.start_ecs_tasks \
    --zip-file fileb://function.zip \
    --timeout 900 \
    --role ${ROLE_ARN} --query 'FunctionArn' --output text)

To create an EventBridge scheduler that triggers the Lambda function to run every 5 minutes, you can use the following AWS CLI command:

cat <<EOF> EnvokeFunctionPolicy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": "lambda:InvokeFunction",
            "Resource": "arn:aws:lambda:us-east-1:${AWS_ACCOUNT_ID}:function:*"
        }
    ]
}
EOF

ENVOKE_FUNC=$(aws iam create-policy --policy-name A_eb-EnvokeFunction --policy-document file://EnvokeFunctionPolicy.json \
   --query 'Policy.Arn' --output text)

cat <<EOF> Scheduler-Execution-Role.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Service": "scheduler.amazonaws.com"
            },
            "Action": "sts:AssumeRole"
        }
    ]
}
EOF

SCHEDULE_EXEC_ARN=$(aws iam create-role --role-name A_eb-SchedulerExecutionRole \
    --assume-role-policy-document file://Scheduler-Execution-Role.json \
    --query 'Role.Arn' --output text)

aws iam attach-role-policy --policy-arn ${ENVOKE_FUNC} --role-name A_eb-SchedulerExecutionRole

Create Event Bridge Schedule that will start in 2 min

cat <<EOF> create-eb-schedule.sh
aws scheduler create-schedule --name A_eb-lambda-start-ecs-task-schedule --schedule-expression 'rate(5 minutes)' \
--target '{"RoleArn": "${SCHEDULE_EXEC_ARN}", "Arn":"${FUNCTION_ARN}" }' \
--flexible-time-window '{ "Mode": "FLEXIBLE", "MaximumWindowInMinutes": 2}'
EOF

chmod +x create-eb-schedule.sh

./create-eb-schedule.sh 

Cleanup

aws scheduler delete-schedule --name A_eb-lambda-start-ecs-task-schedule

aws iam list-attached-role-policies --role-name A_eb-SchedulerExecutionRole | jq '.AttachedPolicies[].PolicyArn' | xargs -I {} aws iam detach-role-policy --role-name A_eb-SchedulerExecutionRole --policy-arn {}

aws iam delete-role --role-name A_eb-SchedulerExecutionRole

aws iam delete-policy --policy-arn ${ENVOKE_FUNC}

aws lambda delete-function --function-name eb-start-ecs-tasks-func

rm -rf function.zip

rm -rf start_ecs_tasks.py

aws iam list-attached-role-policies --role-name ${ROLE_NAME} | jq '.AttachedPolicies[].PolicyArn' | xargs -I {} aws iam detach-role-policy --role-name ${ROLE_NAME} --policy-arn {}

aws iam delete-role --role-name ${ROLE_NAME} 

aws ecs list-task-definitions --family-prefix eb-start-ecs-stask | jq -r '.taskDefinitionArns[]' | xargs -I {} aws ecs deregister-task-definition --task-definition {}