Last active
August 29, 2015 14:21
-
-
Save acdha/f4d393f5ad6134681ffe to your computer and use it in GitHub Desktop.
Tracking down sporadic failures in requests under multiprocessing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| from __future__ import absolute_import, print_function, unicode_literals | |
| import multiprocessing | |
| import sys | |
| import time | |
| import requests | |
| MSG = ''' | |
| <add> | |
| <doc> | |
| <field name="django_id">{pk}</field> | |
| <field boost="1.0" name="pub_date">2009-07-17T18:30:00Z</field> | |
| <field boost="1.0" name="name_exact">daniel2</field> | |
| <field name="id">core.mockmodel.{pk}</field> | |
| <field name="django_ct">core.mockmodel</field> | |
| <field boost="1.0" name="text">Indexed! | |
| {pk}</field> | |
| <field boost="1.0" name="name">daniel2</field> | |
| </doc> | |
| </add> | |
| '''.strip() | |
| def worker_msg(*args, **kwargs): | |
| print(time.time(), multiprocessing.current_process().name, *args, **kwargs) | |
| BAD_IDEA = requests.Session() | |
| def worker(i): | |
| worker_msg('Starting %d' % i) | |
| try: | |
| resp = BAD_IDEA.post('http://localhost:9001/solr/update/?commit=true', | |
| data=MSG.format(pk=i).encode('utf-8'), | |
| headers={'Content-Type': 'text/xml; charset=utf-8'}) | |
| return i, resp.status_code | |
| except requests.exceptions.Timeout as exc: | |
| worker_msg(i, 'Timeout: %s' % exc) | |
| return i, 'Timeout: %s' % exc | |
| except requests.exceptions.ConnectionError as exc: | |
| worker_msg(i, 'Connection Error: %s' % exc) | |
| return i, 'Connection Error: %s' % exc | |
| except Exception as exc: | |
| worker_msg(i, 'Unknown error: %s' % exc) | |
| return i, 'Unknown error: %s' % exc | |
| print('Making a single request to confirm that the server is up') | |
| print('HTTP', worker(1)[1]) | |
| print() | |
| print('Making requests using workers') | |
| pool = multiprocessing.Pool(13) | |
| for idx, status in pool.map(worker, range(0, 100)): | |
| if status != 200: | |
| print('Task %s returned HTTP %s' % (idx, status), file=sys.stderr) | |
| pool.close() | |
| pool.join() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Making a single request to confirm that the server is up | |
| 1431459109.588979 MainProcess Starting 1 | |
| HTTP 200 | |
| Making requests using workers | |
| 1431459109.6548 ForkPoolWorker-8 Starting 14 | |
| 1431459109.750935 ForkPoolWorker-8 14 Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| 1431459109.75097 ForkPoolWorker-8 Starting 15 | |
| 1431459109.808141 ForkPoolWorker-8 Starting 28 | |
| 1431459109.862666 ForkPoolWorker-8 Starting 29 | |
| 1431459109.899878 ForkPoolWorker-8 Starting 40 | |
| 1431459109.92231 ForkPoolWorker-8 Starting 41 | |
| 1431459109.997877 ForkPoolWorker-8 Starting 58 | |
| 1431459110.111616 ForkPoolWorker-8 Starting 59 | |
| 1431459110.166103 ForkPoolWorker-8 Starting 82 | |
| 1431459110.225764 ForkPoolWorker-8 Starting 83 | |
| 1431459109.66049 ForkPoolWorker-10 Starting 18 | |
| 1431459109.906246 ForkPoolWorker-10 18 Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| 1431459109.906281 ForkPoolWorker-10 Starting 19 | |
| 1431459109.965611 ForkPoolWorker-10 Starting 50 | |
| 1431459109.994873 ForkPoolWorker-10 Starting 51 | |
| 1431459110.035029 ForkPoolWorker-10 Starting 62 | |
| 1431459110.153783 ForkPoolWorker-10 Starting 63 | |
| 1431459109.656707 ForkPoolWorker-9 Starting 16 | |
| 1431459110.084159 ForkPoolWorker-9 16 Unknown error: IncompleteRead(9 bytes read) | |
| 1431459110.084614 ForkPoolWorker-9 Starting 17 | |
| 1431459110.132029 ForkPoolWorker-9 Starting 72 | |
| 1431459110.24841 ForkPoolWorker-9 Starting 73 | |
| 1431459109.650919 ForkPoolWorker-6 Starting 10 | |
| 1431459110.039318 ForkPoolWorker-6 Starting 11 | |
| 1431459110.196485 ForkPoolWorker-6 11 Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| 1431459110.19689 ForkPoolWorker-6 Starting 86 | |
| 1431459110.264897 ForkPoolWorker-6 Starting 87 | |
| 1431459109.646812 ForkPoolWorker-2 Starting 2 | |
| 1431459109.788665 ForkPoolWorker-2 2 Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| 1431459109.788788 ForkPoolWorker-2 Starting 3 | |
| 1431459109.814883 ForkPoolWorker-2 Starting 30 | |
| 1431459109.851345 ForkPoolWorker-2 Starting 31 | |
| 1431459109.862548 ForkPoolWorker-2 Starting 36 | |
| 1431459109.936169 ForkPoolWorker-2 Starting 37 | |
| 1431459109.990181 ForkPoolWorker-2 Starting 52 | |
| 1431459110.030231 ForkPoolWorker-2 Starting 53 | |
| 1431459110.05294 ForkPoolWorker-2 Starting 66 | |
| 1431459110.174221 ForkPoolWorker-2 Starting 67 | |
| 1431459110.237049 ForkPoolWorker-2 Starting 90 | |
| 1431459110.307158 ForkPoolWorker-2 Starting 91 | |
| 1431459109.663221 ForkPoolWorker-11 Starting 20 | |
| 1431459109.702617 ForkPoolWorker-11 Starting 21 | |
| 1431459110.327456 ForkPoolWorker-11 21 Unknown error: IncompleteRead(34 bytes read) | |
| 1431459109.646859 ForkPoolWorker-4 Starting 6 | |
| 1431459109.738965 ForkPoolWorker-4 Starting 7 | |
| 1431459110.086862 ForkPoolWorker-4 Starting 70 | |
| 1431459110.270058 ForkPoolWorker-4 70 Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| 1431459110.270092 ForkPoolWorker-4 Starting 71 | |
| 1431459109.644764 ForkPoolWorker-1 Starting 0 | |
| 1431459109.723281 ForkPoolWorker-1 Starting 1 | |
| 1431459109.887543 ForkPoolWorker-1 1 Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| 1431459109.888083 ForkPoolWorker-1 Starting 38 | |
| 1431459109.913002 ForkPoolWorker-1 Starting 39 | |
| 1431459109.960272 ForkPoolWorker-1 Starting 48 | |
| 1431459109.992492 ForkPoolWorker-1 Starting 49 | |
| 1431459110.032634 ForkPoolWorker-1 Starting 60 | |
| 1431459110.066533 ForkPoolWorker-1 Starting 61 | |
| 1431459110.132642 ForkPoolWorker-1 Starting 74 | |
| 1431459110.188115 ForkPoolWorker-1 Starting 75 | |
| 1431459109.650628 ForkPoolWorker-5 Starting 8 | |
| 1431459109.852028 ForkPoolWorker-5 Starting 9 | |
| 1431459109.906387 ForkPoolWorker-5 9 Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| 1431459109.906744 ForkPoolWorker-5 Starting 44 | |
| 1431459109.944897 ForkPoolWorker-5 Starting 45 | |
| 1431459109.990157 ForkPoolWorker-5 Starting 54 | |
| 1431459110.069496 ForkPoolWorker-5 Starting 55 | |
| 1431459110.138358 ForkPoolWorker-5 Starting 76 | |
| 1431459110.196592 ForkPoolWorker-5 Starting 77 | |
| 1431459110.266388 ForkPoolWorker-5 Starting 94 | |
| 1431459110.314024 ForkPoolWorker-5 Starting 95 | |
| 1431459109.646423 ForkPoolWorker-3 Starting 4 | |
| 1431459109.789525 ForkPoolWorker-3 4 Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| 1431459109.789567 ForkPoolWorker-3 Starting 5 | |
| 1431459109.831028 ForkPoolWorker-3 Starting 34 | |
| 1431459109.871641 ForkPoolWorker-3 Starting 35 | |
| 1431459109.904634 ForkPoolWorker-3 Starting 42 | |
| 1431459109.951415 ForkPoolWorker-3 Starting 43 | |
| 1431459109.992347 ForkPoolWorker-3 Starting 56 | |
| 1431459110.032331 ForkPoolWorker-3 Starting 57 | |
| 1431459110.058498 ForkPoolWorker-3 Starting 68 | |
| 1431459110.12768 ForkPoolWorker-3 Starting 69 | |
| 1431459110.171665 ForkPoolWorker-3 Starting 84 | |
| 1431459110.232507 ForkPoolWorker-3 Starting 85 | |
| 1431459109.65254 ForkPoolWorker-7 Starting 12 | |
| 1431459109.750964 ForkPoolWorker-7 12 Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| 1431459109.750998 ForkPoolWorker-7 Starting 13 | |
| 1431459109.770999 ForkPoolWorker-7 Starting 26 | |
| 1431459109.781467 ForkPoolWorker-7 Starting 27 | |
| 1431459109.817306 ForkPoolWorker-7 Starting 32 | |
| 1431459109.853025 ForkPoolWorker-7 Starting 33 | |
| 1431459109.92391 ForkPoolWorker-7 Starting 46 | |
| 1431459110.010622 ForkPoolWorker-7 Starting 47 | |
| 1431459110.050688 ForkPoolWorker-7 Starting 64 | |
| 1431459110.091679 ForkPoolWorker-7 Starting 65 | |
| 1431459110.159641 ForkPoolWorker-7 Starting 80 | |
| 1431459110.218933 ForkPoolWorker-7 Starting 81 | |
| 1431459110.28484 ForkPoolWorker-7 Starting 98 | |
| 1431459110.342486 ForkPoolWorker-7 Starting 99 | |
| 1431459109.669266 ForkPoolWorker-12 Starting 22 | |
| 1431459110.084871 ForkPoolWorker-12 22 Unknown error: IncompleteRead(145 bytes read) | |
| 1431459110.085253 ForkPoolWorker-12 Starting 23 | |
| 1431459110.139914 ForkPoolWorker-12 Starting 78 | |
| 1431459110.190865 ForkPoolWorker-12 Starting 79 | |
| 1431459110.263004 ForkPoolWorker-12 Starting 92 | |
| 1431459110.310426 ForkPoolWorker-12 Starting 93 | |
| 1431459109.671623 ForkPoolWorker-13 Starting 24 | |
| 1431459109.686195 ForkPoolWorker-13 Starting 25 | |
| 1431459110.196238 ForkPoolWorker-13 25 Unknown error: IncompleteRead(84 bytes read) | |
| 1431459110.197065 ForkPoolWorker-13 Starting 88 | |
| 1431459110.214118 ForkPoolWorker-13 Starting 89 | |
| 1431459110.277635 ForkPoolWorker-13 Starting 96 | |
| 1431459110.330216 ForkPoolWorker-13 Starting 97 | |
| Task 1 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| Task 2 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| Task 3 returned HTTP 503 | |
| Task 4 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| Task 9 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| Task 11 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| Task 12 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| Task 13 returned HTTP 503 | |
| Task 14 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| Task 16 returned HTTP Unknown error: IncompleteRead(9 bytes read) | |
| Task 17 returned HTTP 503 | |
| Task 18 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('<?xml version="1.0" encoding="UTF-8"?>\n',)) | |
| Task 21 returned HTTP Unknown error: IncompleteRead(34 bytes read) | |
| Task 22 returned HTTP Unknown error: IncompleteRead(145 bytes read) | |
| Task 23 returned HTTP 503 | |
| Task 25 returned HTTP Unknown error: IncompleteRead(84 bytes read) | |
| Task 26 returned HTTP 503 | |
| Task 29 returned HTTP 503 | |
| Task 30 returned HTTP 503 | |
| Task 31 returned HTTP 503 | |
| Task 32 returned HTTP 503 | |
| Task 35 returned HTTP 503 | |
| Task 37 returned HTTP 503 | |
| Task 38 returned HTTP 503 | |
| Task 39 returned HTTP 503 | |
| Task 40 returned HTTP 503 | |
| Task 43 returned HTTP 503 | |
| Task 44 returned HTTP 503 | |
| Task 45 returned HTTP 503 | |
| Task 47 returned HTTP 503 | |
| Task 48 returned HTTP 503 | |
| Task 49 returned HTTP 503 | |
| Task 50 returned HTTP 503 | |
| Task 51 returned HTTP 503 | |
| Task 52 returned HTTP 503 | |
| Task 53 returned HTTP 503 | |
| Task 55 returned HTTP 503 | |
| Task 56 returned HTTP 503 | |
| Task 57 returned HTTP 503 | |
| Task 59 returned HTTP 503 | |
| Task 60 returned HTTP 503 | |
| Task 61 returned HTTP 503 | |
| Task 64 returned HTTP 503 | |
| Task 65 returned HTTP 503 | |
| Task 67 returned HTTP 503 | |
| Task 68 returned HTTP 503 | |
| Task 69 returned HTTP 503 | |
| Task 70 returned HTTP Connection Error: ('Connection aborted.', BadStatusLine('0\r\n',)) | |
| Task 71 returned HTTP 503 | |
| Task 73 returned HTTP 503 | |
| Task 74 returned HTTP 503 | |
| Task 76 returned HTTP 503 | |
| Task 77 returned HTTP 503 | |
| Task 78 returned HTTP 503 | |
| Task 79 returned HTTP 503 | |
| Task 80 returned HTTP 503 | |
| Task 81 returned HTTP 503 | |
| Task 82 returned HTTP 503 | |
| Task 83 returned HTTP 503 | |
| Task 84 returned HTTP 503 | |
| Task 86 returned HTTP 503 | |
| Task 87 returned HTTP 503 | |
| Task 88 returned HTTP 503 | |
| Task 89 returned HTTP 503 | |
| Task 90 returned HTTP 503 | |
| Task 91 returned HTTP 503 | |
| Task 92 returned HTTP 503 | |
| Task 94 returned HTTP 503 | |
| Task 95 returned HTTP 503 | |
| Task 96 returned HTTP 503 | |
| Task 98 returned HTTP 503 | |
| Task 99 returned HTTP 503 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment