Skip to content

Instantly share code, notes, and snippets.

  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save kesor/2bd99f5bd8e596db7f17d6b997ef3c3d to your computer and use it in GitHub Desktop.
import os
import boto3
import datetime
import dateutil
DEFAULT_MAX_MEM = 3000
LOW_CLUSTER_CPU_TH = 20
HIGH_CLUSTER_CPU_TH = 65
CONTAINERS_MAX_MEM = {
'cluster1': 1200,
'cluster2': 3000,
'cluster3': 800,
}
def ec2_average_cpu(instances, client):
sum = 0
count = 0
for instance in instances['containerInstances']:
response = client.get_metric_statistics(
Namespace='AWS/EC2',
MetricName='CPUUtilization',
Dimensions=[
{
'Name': 'InstanceId',
'Value': instance['ec2InstanceId'],
},
],
StartTime=datetime.datetime.utcnow() - datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Period=300,
Statistics=['Average'],
)
sum += response['Datapoints'][0]['Average']
count += 1
return sum/count
def lambda_handler(event, context):
ecs = boto3.client('ecs')
cw = boto3.client('cloudwatch')
# Iterate over all clusters
for clus in ecs.list_clusters()['clusterArns']:
if clus.split('/')[1][0:5] != 'awseb':
cluster = clus.split('/')[1]
print('Calculating schedulable containers for %s' % cluster)
instance_list = ecs.list_container_instances(cluster=cluster, status='ACTIVE')
shouldScale = 1 # 0 - In (Down), 1 - No, 2 - Out (Up)
if not instance_list['containerInstanceArns']:
print 'No instances in cluster {}, moving on'.format(cluster)
continue
instances = ecs.describe_container_instances(cluster=cluster,
containerInstances=instance_list['containerInstanceArns'])
schedulable_containers = 0
clusterAvailableMemory = 0
ec2TotalMemory = 0
ec2AverageCpu = ec2_average_cpu(instances, cw)
print 'EC2 AVG CPU: {}'.format(ec2AverageCpu)
for instance in instances['containerInstances']:
if not ec2TotalMemory:
registered_resources = {resource['name']: resource for resource in instance['registeredResources']}
ec2TotalMemory = registered_resources['MEMORY']['integerValue']
remaining_resources = {resource['name']: resource for resource in instance['remainingResources']}
max_mem = CONTAINERS_MAX_MEM[cluster] if CONTAINERS_MAX_MEM.get(cluster) else DEFAULT_MAX_MEM
containers_by_mem = int(remaining_resources['MEMORY']['integerValue'] / max_mem)
schedulable_containers += containers_by_mem
clusterAvailableMemory += remaining_resources['MEMORY']['integerValue']
print('Schedulable containers: %s' % schedulable_containers)
# If [(total cluster available memory) - (1 ec2 memory)] > [(memory required for current containers) + (1 container memory)] -> remove instance
if schedulable_containers > 0:
# If there's memory room in the cluster to scale (else scale anyway) -> maybe should scale ->
if ec2AverageCpu > HIGH_CLUSTER_CPU_TH:
# SCALE OUT!
print 'SCALE OUT based on CPU: {} > {}'.format(ec2AverageCpu, HIGH_CLUSTER_CPU_TH)
shouldScale = 2
elif ec2AverageCpu < LOW_CLUSTER_CPU_TH:
if (clusterAvailableMemory - ec2TotalMemory) > ((len(instances['containerInstances'])*ec2TotalMemory)-clusterAvailableMemory) + max_mem:
# If The available memory - A potenial reduce in ec2, still holds the utilized memory + room for one more -> scale in (down)
shouldScale = 0
else:
# SCALE OUT!
print 'SCALE OUT - no room for containers'
shouldScale = 2
cw.put_metric_data(Namespace='AWS/ECS',
MetricData=[{
'MetricName': 'ShouldScaleEC2'.format(cluster),
'Dimensions': [{
'Name': 'ClusterName',
'Value': cluster
}],
'Timestamp': datetime.datetime.now(dateutil.tz.tzlocal()),
'Value': shouldScale
}])
print('Metric was sent to CloudWatch')
return {}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment