An attempt to reproduce a reported problem (see https://github.com/aws/amazon-ecs-agent/issues/74)

run-many-coreos-container-instances.rb

require 'aws-sdk'
require 'base64'

COREOS_AMI="ami-4b1c763c"

ec2 = Aws::EC2::Client.new(region: 'eu-west-1')
ecs = Aws::ECS::Client.new(region: 'eu-west-1')

NUM_INSTANCES=10

role = ENV['ECS_ROLE']
if role == ""
	puts "set a role"
	exit
end
keyname = ENV['ECS_KEYNAME']
if keyname == ""
	puts "set a keyname"
	exit
end

user_data=<<EOF
#cloud-config

coreos:
 units:
   -
     name: amazon-ecs-agent.service
     command: start
     runtime: true
     content: |
       [Unit]
       Description=Amazon ECS Agent
       After=docker.service
       Requires=docker.service

       [Service]
       TimeoutStartSec=0
       TimeoutStopSec=0
       Restart=on-failure
       SyslogIdentifier=ecs-agent
       ExecStartPre=-/bin/mkdir -p /var/log/ecs /var/ecs-data
       ExecStartPre=-/usr/bin/docker stop ecs-agent
       ExecStartPre=-/usr/bin/docker kill ecs-agent
       ExecStartPre=-/usr/bin/docker rm ecs-agent
       ExecStartPre=-/usr/bin/docker pull euank/euank-test-agent:5bfc746
       ExecStart=/usr/bin/docker run --name ecs-agent -v /var/run/docker.sock:/var/run/docker.sock -v /var/log/ecs:/log -v /var/ecs-data:/data -p 127.0.0.1:51678:51678 -e AWS_DEFAULT_REGION=eu-west-1 -e ECS_CLUSTER=default -e ECS_LOGLEVEL=debug -e ECS_DATADIR=/data/ euank/euank-test-agent:5bfc746
EOF

if ecs.list_container_instances.container_instance_arns.size != 0
	puts "Please terminate all container instances in eu-west-1 prior to running this"
	exit
end

round_num = 1
while true
	resp = ec2.run_instances(
		min_count: NUM_INSTANCES,
		max_count: NUM_INSTANCES,
		image_id: COREOS_AMI,
		instance_type: "t2.small",
		instance_initiated_shutdown_behavior: "terminate",
		user_data: Base64.encode64(user_data),
		key_name: keyname,
		iam_instance_profile: {name: role},
	)
	launched_instances = resp.instances.map(&:instance_id).sort
	puts "Launched instances: " + launched_instances.join(", ")

	# Wait and verify that all register as container instances within 20 minutes
	start_time = Time.now.to_i
	ci_instance_arns = []
	ci_instances = []
	while (Time.now.to_i - start_time) < 20 * 60
		ci_instance_arns = ecs.list_container_instances.container_instance_arns
		if ci_instance_arns.size == NUM_INSTANCES
			desc_ci_instances = ecs.describe_container_instances(container_instances: ci_instance_arns).container_instances
			ci_instances = desc_ci_instances.map(&:ec2_instance_id).sort
			if ci_instances == launched_instances
				break
			else
				puts "Ten container instances, but they didn't match our EC2 IDs: " + ci_instances.join(", ")
				exit
			end
		end
		sleep(10)
	end

	if ci_instances != launched_instances
		puts "Not enough container instances running:" + (launched_instances - ci_instances).join(", ")
		exit
	end
	# Give them up to 5 seconds for agentConnected = true
	sleep(5)

	described_cis = ecs.describe_container_instances(container_instances: ci_instance_arns)

	disconnected_cis = described_cis.container_instances.reject{|ci| ci.agent_connected}
	if disconnected_cis.size > 0
		puts "Some container instances did not connect"
		puts disconnected_cis.map(&:container_instance_arn).join(", ")
		exit
	end

	resp = ec2.terminate_instances(instance_ids: launched_instances)

	while ecs.list_container_instances.container_instance_arns.size != 0
		sleep(5)
	end
	puts "Round completed with no errors: #{round_num}"
	round_num += 1
end