2020-11-30 automation hub incident

core_reservedresource.sql

SELECT pulp_id, pulp_created, pulp_last_updated, resource, worker_id FROM core_reservedresource;


               pulp_id                |         pulp_created          |       pulp_last_updated       | resource |              worker_id               
--------------------------------------+-------------------------------+-------------------------------+----------+--------------------------------------
 0d096eaa-4896-4007-9946-45cb1c083bdf | 2020-11-30 17:36:54.181421+00 | 2020-11-30 17:36:54.181436+00 | 13       | a5e56c14-ca30-4af2-8737-6a00c01f8111
(1 row)

core_task.sql

SELECT pulp_id, pulp_created, pulp_last_updated, state, name, started_at, finished_at, error, worker_id, parent_task_id, task_group_id 
FROM core_task 
WHERE pulp_id='fb7ebf19-9bb1-407a-955b-f7e642537aeb';


               pulp_id                |         pulp_created          |       pulp_last_updated       | state  |                          name                           |          started_at           |          finished_at          |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  error                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |              worker_id               | parent_task_id | task_group_id 
--------------------------------------+-------------------------------+-------------------------------+--------+---------------------------------------------------------+-------------------------------+-------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------+----------------+---------------
 fb7ebf19-9bb1-407a-955b-f7e642537aeb | 2020-11-21 11:43:43.949779+00 | 2020-11-30 17:37:12.635517+00 | failed | galaxy_ng.app.tasks.synclist.curate_synclist_repository | 2020-11-30 17:37:07.230584+00 | 2020-11-30 17:37:12.635315+00 | {"traceback": "  File \"/venv/lib64/python3.6/site-packages/rq/worker.py\", line 945, in perform_job\n    started_job_registry=started_job_registry)\n  File \"/venv/lib64/python3.6/site-packages/pulpcore/tasking/worker.py\", line 154, in handle_job_success\n    return super().handle_job_success(job, queue, started_job_registry)\n  File \"/venv/lib64/python3.6/site-packages/rq/worker.py\", line 894, in handle_job_success\n    pipeline.watch(job.dependents_key)\n  File \"/venv/lib64/python3.6/site-packages/redis/client.py\", line 4041, in watch\n    return self.execute_command('WATCH', *names)\n  File \"/venv/lib64/python3.6/site-packages/redis/client.py\", line 3822, in execute_command\n    return self.immediate_execute_command(*args, **kwargs)\n  File \"/venv/lib64/python3.6/site-packages/redis/client.py\", line 3837, in immediate_execute_command\n    self.shard_hint)\n  File \"/venv/lib64/python3.6/site-packages/redis/connection.py\", line 1192, in get_connection\n    connection.connect()\n  File \"/venv/lib64/python3.6/site-packages/redis/connection.py\", line 563, in connect\n    raise ConnectionError(self._error_message(e))\n", "description": "Error 111 connecting to redis:6379. Connection refused."} | a5e56c14-ca30-4af2-8737-6a00c01f8111 |                | 
(1 row)

core_taskreservedresource.sql

SELECT pulp_id, pulp_created, pulp_last_updated, resource_id, task_id FROM core_taskreservedresource;


               pulp_id                |         pulp_created          |       pulp_last_updated       |             resource_id              |               task_id                
--------------------------------------+-------------------------------+-------------------------------+--------------------------------------+--------------------------------------
 06ef281a-5de3-4659-8aa4-dda131fabaf5 | 2020-11-30 17:36:56.695176+00 | 2020-11-30 17:36:56.695193+00 | 0d096eaa-4896-4007-9946-45cb1c083bdf | fb7ebf19-9bb1-407a-955b-f7e642537aeb
(1 row)