Attempt to reproduce #1497 CPU usage increased dramatically 0.8.1-RC1 -> master

.gitignore

stats-*.txt

README.md

This is an attempt to reproduce the problems described in

mesosphere/marathon#1497

Tools used:

• docker (http://www.docker.com)
• fig (http://www.fig.sh)
• python2 with SimpleHTTPServer (you can use python3 with "cmd": "python3 -m http.server $PORT" instead)
• ab (apache bench http://httpd.apache.org/docs/2.2/en/programs/ab.html)
• httpie for the "http" command (https://github.com/jakubroztocil/httpie)

I used Mac OS with boot2docker for testing but Linux with docker should work to.

Steps I tried to reproduce the problem:

• adjust config.sh if necessary
• fig up to start the docker containers
• execute ./show-config.sh
• Open the old Marathon GUI at http://${DOCKER_HOST_IP}:8080/
• Open the new Marathon GUI at http://${DOCKER_HOST_IP}:8081/
• Open cadvisor with the CPU usage for both Marathons at http://${DOCKER_HOST_IP}:9000/
• Startup apps with tasks with the ./one-app.sh or ./ten-apps.sh scripts
• Wait for the deployments to finish (you can watch cadvisor for fun)
• Run ./query-apps.sh, watch CPU usage
• Look at query-apps.sh results in stats-8080.txt for old and stats-8081.txt for new Marathon.

config.sh

set -e

# substitute if you do not use boot2docker
DOCKER_HOST_IP=`boot2docker ip`

docker-compose.yml

cadvisor:
  image: google/cadvisor:latest
  volumes:
    - /:/rootfs/ro
    - /var/run:/var/run:rw
    - /sys:/sys:ro
    - /var/lib/docker:/var/lib/docker:ro
  ports:
    - '9000:8080'

zookeeper:
  image: jplock/zookeeper
  ports:
    - '2181:2181'

mesosmaster:
  image: 'mesosphere/mesos-master:0.22.1-1.0.ubuntu1404'
  ports: 
    - '5050:5050'
  links:
    - zookeeper
  command: '--zk=zk://zookeeper:2181/mesos --quorum=1 --registry=in_memory'

mesosslave1:
  image: 'mesosphere/mesos-slave:0.22.1-1.0.ubuntu1404'
  hostname: 'mesosslave1'
  ports:
    - '5051'
  expose:
    - '31000:32000'
  links:
    - zookeeper
  command: '--master=zk://zookeeper:2181/mesos'

mesosslave2:
  image: 'mesosphere/mesos-slave:0.22.1-1.0.ubuntu1404'
  hostname: 'mesosslave2'
  ports:
    - '5051'
  expose:
    - '31000:32000'
  links:
    - zookeeper
  command: '--master=zk://zookeeper:2181/mesos'

mesosslave3:
  image: 'mesosphere/mesos-slave:0.22.1-1.0.ubuntu1404'
  hostname: 'mesosslave3'
  ports:
    - '5051'
  expose:
    - '31000:32000'
  links:
    - zookeeper
  command: '--master=zk://zookeeper:2181/mesos'

marathonold:
  image: 'mesosphere/marathon:v0.8.1-RC1'
  ports:
    - '8080:8080'
  links:
    - zookeeper
    - mesosslave1 # health checks
    - mesosslave2 # health checks
    - mesosslave3 # health checks
  command: '--master zk://zookeeper:2181/mesos --zk zk://zookeeper:2181/marathonold'

marathonnew:
  image: 'mesosphere/marathon:v0.8.2-RC3'
  ports:
    - '8081:8080'
  links:
    - zookeeper
    - mesosslave1 # health checks
    - mesosslave2 # health checks
    - mesosslave3 # health checks
  command: '--master zk://zookeeper:2181/mesos --zk zk://zookeeper:2181/marathonnew --max_tasks_per_offer 50'

one-app.sh

#!/bin/bash

. $(dirname $0)/config.sh

for port in 8080 8081; do 
  http -v PUT $DOCKER_HOST_IP:$port/v2/apps/service-private-health @service-private-health-100.json
done

query-apps.sh

#!/bin/bash

. $(dirname $0)/config.sh 

for port in 8080 8081; do 
  ab -t 30 -s 3 -c 3  http://`boot2docker ip`:$port/v2/apps?embed=apps.tasks 2>&1 | tee stats-$port.txt & 
done

service-private-health-100.json

{
  "cmd": "python -m SimpleHTTPServer $PORT0",
  "cpus": 0.0001,
  "disk": 0,
  "instances": 100,
  "mem": 16,
  "ports": [
    0
  ],
  "healthChecks": [
    {
      "path": "/",
      "protocol": "HTTP",
      "portIndex": 0,
      "gracePeriodSeconds": 120,
      "intervalSeconds": 3,
      "timeoutSeconds": 2,
      "maxConsecutiveFailures": 3
    }
  ]
}

service-private-health-5.json

{
  "cmd": "python -m SimpleHTTPServer $PORT0",
  "cpus": 0.0001,
  "disk": 0,
  "instances": 5,
  "mem": 16,
  "ports": [
    0
  ],
  "healthChecks": [
    {
      "path": "/",
      "protocol": "HTTP",
      "portIndex": 0,
      "gracePeriodSeconds": 120,
      "intervalSeconds": 3,
      "timeoutSeconds": 2,
      "maxConsecutiveFailures": 3
    }
  ]
}

show-config.sh

#!/bin/bash

. $(dirname $0)/config.sh

echo "Old Marathon at http://${DOCKER_HOST_IP}:8080"
echo "New Marathon at http://${DOCKER_HOST_IP}:8081"
echo "CAdvisor at http://${DOCKER_HOST_IP}:9000"
echo "Mesos Master at http://${DOCKER_HOST_IP}:5050"

ten-apps.sh

#!/bin/bash

. $(dirname $0)/config.sh

for app in {1..10}; do
  for port in 8080 8081; do 
    http -v PUT $DOCKER_HOST_IP:$port/v2/apps/service-private-health$app @service-private-health-5.json
  done
done