Some roughly formatted benchmark data.
time ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml
# ...
Executed in 11.36 secs fish external
usr time 36.54 secs 0.00 micros 36.54 secs
sys time 16.28 secs 583.00 micros 16.28 secs
time ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e benchmark_host_count=100
# ...
Executed in 41.32 secs fish external
usr time 147.68 secs 528.00 micros 147.68 secs
sys time 70.04 secs 112.00 micros 70.04 secs
time ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e benchmark_host_count=200 -e benchmark_task_count=200
# ...
Executed in 360.67 secs fish external
usr time 20.83 mins 0.00 micros 20.83 mins
sys time 10.84 mins 360.00 micros 10.84 mins
With the default offline API client against localhost (no running API server):
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.3"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.3" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.3" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="client:offline"
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 310 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T11:30:12.313649-04:00 | 00:29:25.955674 |
| 309 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T11:24:58.979017-04:00 | 00:03:22.571607 |
| 308 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T11:22:05.943304-04:00 | 00:00:52.728591 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
With export ARA_API_CLIENT=http
against localhost: ara-manage runserver
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,version:1.5.3"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,version:1.5.3" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,version:1.5.3" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="client:http"
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 314 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T12:25:58.707298-04:00 | 00:17:24.335043 |
| 313 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T12:23:34.120747-04:00 | 00:01:37.972033 |
| 311 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T12:14:12.684240-04:00 | 00:00:30.101029 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
With export ARA_API_CLIENT=http
against localhost: gunicorn --access-logfile - --workers=9 --bind 127.0.0.1:8000 ara.server.wsgi
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.3"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.3" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.3" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="backend:gunicorn"
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 322 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T14:39:49.979617-04:00 | 00:13:47.462996 |
| 316 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T13:14:17.244041-04:00 | 00:01:09.821960 |
| 315 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T13:12:09.157927-04:00 | 00:00:20.606147 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
With the default offline API client against localhost (no running API server):
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.1"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.1" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:offline,version:1.5.1" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="version:1.5.1"
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 328 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T16:08:19.132407-04:00 | 00:30:22.647469 |
| 327 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T16:04:40.701513-04:00 | 00:03:31.300917 |
| 326 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T16:03:37.630983-04:00 | 00:00:56.030091 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
With export ARA_API_CLIENT=http
against localhost: ara-manage runserver
:
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,backend:django,version:1.5.1"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,backend:django,version:1.5.1" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=client:http,backend:django,version:1.5.1" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="backend:django"
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 325 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T15:00:09.476913-04:00 | 00:17:28.111219 |
| 324 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T14:58:21.897583-04:00 | 00:01:39.595000 |
| 323 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T14:57:42.707173-04:00 | 00:00:32.229069 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
With export ARA_API_CLIENT=http
against localhost: gunicorn --access-logfile - --workers=9 --bind 127.0.0.1:8000 ara.server.wsgi
:
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.1"
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.1" -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e "ara_playbook_labels=backend:gunicorn,client:http,version:1.5.1" -e benchmark_host_count=200 -e benchmark_task_count=200
# ara playbook list --label="backend:gunicorn" --limit 3
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| id | status | path | tasks | results | hosts | started | duration |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
| 334 | completed | ...ara/tests/integration/benchmark.yaml | 202 | 40201 | 201 | 2020-10-31T17:22:05.171776-04:00 | 00:21:38.136877 |
| 333 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 5101 | 101 | 2020-10-31T17:19:38.631320-04:00 | 00:02:19.691135 |
| 332 | completed | ...ara/tests/integration/benchmark.yaml | 52 | 1276 | 26 | 2020-10-31T17:18:55.759791-04:00 | 00:00:36.259820 |
+-----+-----------+-----------------------------------------+-------+---------+-------+----------------------------------+-----------------+
Testing 0.16.8 is also interesting science and also because it doesn't have an API and the callback talks directly to the database.
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e benchmark_host_count=100
# ansible-playbook -i 'localhost,' -c local tests/integration/benchmark.yaml -e benchmark_host_count=200 -e benchmark_host_tasks=200
# ara playbook list
+--------------------------------------+-------------------------------------------------------------------------------+---------------------+----------+----------+-----------------+
| ID | Path | Time Start | Duration | Complete | Ansible Version |
+--------------------------------------+-------------------------------------------------------------------------------+---------------------+----------+----------+-----------------+
| 0f448e3a-f64f-4928-9f5a-251f7eb4372d | /home/dmsimard/dev/git/ansible-community/ara/tests/integration/benchmark.yaml | 2020-10-31 19:06:50 | 0:00:28 | True | 2.10.2 |
| af6c0d50-eb05-47f8-bba0-e5e3ef609eb9 | /home/dmsimard/dev/git/ansible-community/ara/tests/integration/benchmark.yaml | 2020-10-31 19:07:35 | 0:01:56 | True | 2.10.2 |
| 20e3ab2e-8296-471a-a3c6-1bc56931fdf4 | /home/dmsimard/dev/git/ansible-community/ara/tests/integration/benchmark.yaml | 2020-10-31 19:14:23 | 0:19:05 | True | 2.10.2 |
+--------------------------------------+-------------------------------------------------------------------------------+---------------------+----------+----------+-----------------+