Created
October 14, 2023 17:50
-
-
Save sbenthall/7c59aa9755140120614b80cd9aaf521e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ pytest tests/ | |
================================================================================ test session starts ================================================================================ | |
platform linux -- Python 3.9.17, pytest-7.4.0, pluggy-1.2.0 | |
rootdir: /home/sb/projects/bigbang/tests | |
configfile: pytest.ini | |
plugins: anyio-3.7.1 | |
collected 77 items | |
tests/test_bigbang_io.py . [ 1%] | |
tests/analysis/test_listserv.py ................ [ 22%] | |
tests/ingress/test_listserv.py s.....s...... [ 38%] | |
tests/ingress/test_pipermail.py EEEE [ 44%] | |
tests/ingress/test_w3c.py .......... [ 57%] | |
tests/unit/test_archive.py ... [ 61%] | |
tests/unit/test_bigbang.py ........ [ 71%] | |
tests/unit/test_listserv.py ............... [ 90%] | |
tests/unit/test_mailman.py ....... [100%] | |
====================================================================================== ERRORS ======================================================================================= | |
________________________________________________________ ERROR at setup of TestPipermailMessageParser.test__message_content _________________________________________________________ | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, method = 'GET' | |
url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), timeout = Timeout(connect=None, read=None, total=None), chunked = False | |
response_conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, preload_content = False, decode_content = False, enforce_content_length = True | |
def _make_request( | |
self, | |
conn: BaseHTTPConnection, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | None = None, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
chunked: bool = False, | |
response_conn: BaseHTTPConnection | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
enforce_content_length: bool = True, | |
) -> BaseHTTPResponse: | |
""" | |
Perform a request on a given urllib connection object taken from our | |
pool. | |
:param conn: | |
a connection from one of our connection pools | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param response_conn: | |
Set this to ``None`` if you will handle releasing the connection or | |
set the connection to have the response release it. | |
:param preload_content: | |
If True, the response's body will be preloaded during construction. | |
:param decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param enforce_content_length: | |
Enforce content length checking. Body returned by server must match | |
value of Content-Length header, if present. Otherwise, raise error. | |
""" | |
self.num_requests += 1 | |
timeout_obj = self._get_timeout(timeout) | |
timeout_obj.start_connect() | |
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) | |
try: | |
# Trigger any extra validation we need to do. | |
try: | |
> self._validate_conn(conn) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:467: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:1092: in _validate_conn | |
conn.connect() | |
bigbang-env/lib/python3.9/site-packages/urllib3/connection.py:642: in connect | |
sock_and_verified = _ssl_wrap_socket_and_match_hostname( | |
bigbang-env/lib/python3.9/site-packages/urllib3/connection.py:783: in _ssl_wrap_socket_and_match_hostname | |
ssl_sock = ssl_wrap_socket( | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/ssl_.py:469: in ssl_wrap_socket | |
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/ssl_.py:513: in _ssl_wrap_socket_impl | |
return ssl_context.wrap_socket(sock, server_hostname=server_hostname) | |
/usr/lib/python3.9/ssl.py:501: in wrap_socket | |
return self.sslsocket_class._create( | |
/usr/lib/python3.9/ssl.py:1041: in _create | |
self.do_handshake() | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <ssl.SSLSocket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>, block = False | |
@_sslcopydoc | |
def do_handshake(self, block=False): | |
self._check_connected() | |
timeout = self.gettimeout() | |
try: | |
if timeout == 0.0 and block: | |
self.settimeout(None) | |
> self._sslobj.do_handshake() | |
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129) | |
/usr/lib/python3.9/ssl.py:1310: SSLCertVerificationError | |
During handling of the above exception, another exception occurred: | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, method = 'GET', url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), redirect = False, assert_same_host = False, timeout = Timeout(connect=None, read=None, total=None) | |
pool_timeout = None, release_conn = False, chunked = False, body_pos = None, preload_content = False, decode_content = False, response_kw = {} | |
parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/pipermail/accred-model/2018-August.txt.gz', query=None, fragment=None), destination_scheme = None, conn = None | |
release_this_conn = True, http_tunnel_required = False, err = None, clean_exit = False | |
def urlopen( # type: ignore[override] | |
self, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | bool | int | None = None, | |
redirect: bool = True, | |
assert_same_host: bool = True, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
pool_timeout: int | None = None, | |
release_conn: bool | None = None, | |
chunked: bool = False, | |
body_pos: _TYPE_BODY_POSITION | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
**response_kw: typing.Any, | |
) -> BaseHTTPResponse: | |
""" | |
Get a connection from the pool and perform an HTTP request. This is the | |
lowest level call for making a request, so you'll need to specify all | |
the raw details. | |
.. note:: | |
More commonly, it's appropriate to use a convenience method | |
such as :meth:`request`. | |
.. note:: | |
`release_conn` will only behave as expected if | |
`preload_content=False` because we want to make | |
`preload_content=False` the default behaviour someday soon without | |
breaking backwards compatibility. | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param redirect: | |
If True, automatically handle redirects (status codes 301, 302, | |
303, 307, 308). Each redirect counts as a retry. Disabling retries | |
will disable redirect, too. | |
:param assert_same_host: | |
If ``True``, will make sure that the host of the pool requests is | |
consistent else will raise HostChangedError. When ``False``, you can | |
use the pool on an HTTP proxy and request foreign hosts. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param pool_timeout: | |
If set and the pool is set to block=True, then this method will | |
block for ``pool_timeout`` seconds and raise EmptyPoolError if no | |
connection is available within the time period. | |
:param bool preload_content: | |
If True, the response's body will be preloaded into memory. | |
:param bool decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param release_conn: | |
If False, then the urlopen call will not release the connection | |
back into the pool once a response is received (but will release if | |
you read the entire contents of the response such as when | |
`preload_content=True`). This is useful if you're not preloading | |
the response's content immediately. You will need to call | |
``r.release_conn()`` on the response ``r`` to return the connection | |
back into the pool. If None, it takes the value of ``preload_content`` | |
which defaults to ``True``. | |
:param bool chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param int body_pos: | |
Position to seek to in file-like body in the event of a retry or | |
redirect. Typically this won't need to be set because urllib3 will | |
auto-populate the value when needed. | |
""" | |
parsed_url = parse_url(url) | |
destination_scheme = parsed_url.scheme | |
if headers is None: | |
headers = self.headers | |
if not isinstance(retries, Retry): | |
retries = Retry.from_int(retries, redirect=redirect, default=self.retries) | |
if release_conn is None: | |
release_conn = preload_content | |
# Check host | |
if assert_same_host and not self.is_same_host(url): | |
raise HostChangedError(self, url, retries) | |
# Ensure that the URL we're connecting to is properly encoded | |
if url.startswith("/"): | |
url = to_str(_encode_target(url)) | |
else: | |
url = to_str(parsed_url.url) | |
conn = None | |
# Track whether `conn` needs to be released before | |
# returning/raising/recursing. Update this variable if necessary, and | |
# leave `release_conn` constant throughout the function. That way, if | |
# the function recurses, the original value of `release_conn` will be | |
# passed down into the recursive call, and its value will be respected. | |
# | |
# See issue #651 [1] for details. | |
# | |
# [1] <https://github.com/urllib3/urllib3/issues/651> | |
release_this_conn = release_conn | |
http_tunnel_required = connection_requires_http_tunnel( | |
self.proxy, self.proxy_config, destination_scheme | |
) | |
# Merge the proxy headers. Only done when not using HTTP CONNECT. We | |
# have to copy the headers dict so we can safely change it without those | |
# changes being reflected in anyone else's copy. | |
if not http_tunnel_required: | |
headers = headers.copy() # type: ignore[attr-defined] | |
headers.update(self.proxy_headers) # type: ignore[union-attr] | |
# Must keep the exception bound to a separate variable or else Python 3 | |
# complains about UnboundLocalError. | |
err = None | |
# Keep track of whether we cleanly exited the except block. This | |
# ensures we do proper cleanup in finally. | |
clean_exit = False | |
# Rewind body position, if needed. Record current position | |
# for future rewinds in the event of a redirect/retry. | |
body_pos = set_file_position(body, body_pos) | |
try: | |
# Request a connection from the queue. | |
timeout_obj = self._get_timeout(timeout) | |
conn = self._get_conn(timeout=pool_timeout) | |
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] | |
# Is this a closed/new connection that requires CONNECT tunnelling? | |
if self.proxy is not None and http_tunnel_required and conn.is_closed: | |
try: | |
self._prepare_proxy(conn) | |
except (BaseSSLError, OSError, SocketTimeout) as e: | |
self._raise_timeout( | |
err=e, url=self.proxy.url, timeout_value=conn.timeout | |
) | |
raise | |
# If we're going to release the connection in ``finally:``, then | |
# the response doesn't need to know about the connection. Otherwise | |
# it will also try to release it and we'll have a double-release | |
# mess. | |
response_conn = conn if not release_conn else None | |
# Make the request on the HTTPConnection object | |
> response = self._make_request( | |
conn, | |
method, | |
url, | |
timeout=timeout_obj, | |
body=body, | |
headers=headers, | |
chunked=chunked, | |
retries=retries, | |
response_conn=response_conn, | |
preload_content=preload_content, | |
decode_content=decode_content, | |
**response_kw, | |
) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:790: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, method = 'GET' | |
url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), timeout = Timeout(connect=None, read=None, total=None), chunked = False | |
response_conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, preload_content = False, decode_content = False, enforce_content_length = True | |
def _make_request( | |
self, | |
conn: BaseHTTPConnection, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | None = None, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
chunked: bool = False, | |
response_conn: BaseHTTPConnection | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
enforce_content_length: bool = True, | |
) -> BaseHTTPResponse: | |
""" | |
Perform a request on a given urllib connection object taken from our | |
pool. | |
:param conn: | |
a connection from one of our connection pools | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param response_conn: | |
Set this to ``None`` if you will handle releasing the connection or | |
set the connection to have the response release it. | |
:param preload_content: | |
If True, the response's body will be preloaded during construction. | |
:param decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param enforce_content_length: | |
Enforce content length checking. Body returned by server must match | |
value of Content-Length header, if present. Otherwise, raise error. | |
""" | |
self.num_requests += 1 | |
timeout_obj = self._get_timeout(timeout) | |
timeout_obj.start_connect() | |
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) | |
try: | |
# Trigger any extra validation we need to do. | |
try: | |
self._validate_conn(conn) | |
except (SocketTimeout, BaseSSLError) as e: | |
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) | |
raise | |
# _validate_conn() starts the connection to an HTTPS proxy | |
# so we need to wrap errors with 'ProxyError' here too. | |
except ( | |
OSError, | |
NewConnectionError, | |
TimeoutError, | |
BaseSSLError, | |
CertificateError, | |
SSLError, | |
) as e: | |
new_e: Exception = e | |
if isinstance(e, (BaseSSLError, CertificateError)): | |
new_e = SSLError(e) | |
# If the connection didn't successfully connect to it's proxy | |
# then there | |
if isinstance( | |
new_e, (OSError, NewConnectionError, TimeoutError, SSLError) | |
) and (conn and conn.proxy and not conn.has_connected_to_proxy): | |
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) | |
> raise new_e | |
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:491: SSLError | |
The above exception was the direct cause of the following exception: | |
self = <requests.adapters.HTTPAdapter object at 0x7f4de6f15f40>, request = <PreparedRequest [GET]>, stream = False, timeout = Timeout(connect=None, read=None, total=None) | |
verify = '/home/sb/projects/bigbang/bigbang/icann_certificate.pem', cert = None, proxies = OrderedDict() | |
def send( | |
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None | |
): | |
"""Sends PreparedRequest object. Returns Response object. | |
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent. | |
:param stream: (optional) Whether to stream the request content. | |
:param timeout: (optional) How long to wait for the server to send | |
data before giving up, as a float, or a :ref:`(connect timeout, | |
read timeout) <timeouts>` tuple. | |
:type timeout: float or tuple or urllib3 Timeout object | |
:param verify: (optional) Either a boolean, in which case it controls whether | |
we verify the server's TLS certificate, or a string, in which case it | |
must be a path to a CA bundle to use | |
:param cert: (optional) Any user-provided SSL certificate to be trusted. | |
:param proxies: (optional) The proxies dictionary to apply to the request. | |
:rtype: requests.Response | |
""" | |
try: | |
conn = self.get_connection(request.url, proxies) | |
except LocationValueError as e: | |
raise InvalidURL(e, request=request) | |
self.cert_verify(conn, request.url, verify, cert) | |
url = self.request_url(request, proxies) | |
self.add_headers( | |
request, | |
stream=stream, | |
timeout=timeout, | |
verify=verify, | |
cert=cert, | |
proxies=proxies, | |
) | |
chunked = not (request.body is None or "Content-Length" in request.headers) | |
if isinstance(timeout, tuple): | |
try: | |
connect, read = timeout | |
timeout = TimeoutSauce(connect=connect, read=read) | |
except ValueError: | |
raise ValueError( | |
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " | |
f"or a single float to set both timeouts to the same value." | |
) | |
elif isinstance(timeout, TimeoutSauce): | |
pass | |
else: | |
timeout = TimeoutSauce(connect=timeout, read=timeout) | |
try: | |
> resp = conn.urlopen( | |
method=request.method, | |
url=url, | |
body=request.body, | |
headers=request.headers, | |
redirect=False, | |
assert_same_host=False, | |
preload_content=False, | |
decode_content=False, | |
retries=self.max_retries, | |
timeout=timeout, | |
chunked=chunked, | |
) | |
bigbang-env/lib/python3.9/site-packages/requests/adapters.py:486: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:844: in urlopen | |
retries = retries.increment( | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = Retry(total=0, connect=None, read=False, redirect=None, status=None), method = 'GET', url = '/pipermail/accred-model/2018-August.txt.gz', response = None | |
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)')) | |
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, _stacktrace = <traceback object at 0x7f4de7066c80> | |
def increment( | |
self, | |
method: str | None = None, | |
url: str | None = None, | |
response: BaseHTTPResponse | None = None, | |
error: Exception | None = None, | |
_pool: ConnectionPool | None = None, | |
_stacktrace: TracebackType | None = None, | |
) -> Retry: | |
"""Return a new Retry object with incremented retry counters. | |
:param response: A response object, or None, if the server did not | |
return a response. | |
:type response: :class:`~urllib3.response.BaseHTTPResponse` | |
:param Exception error: An error encountered during the request, or | |
None if the response was received successfully. | |
:return: A new ``Retry`` object. | |
""" | |
if self.total is False and error: | |
# Disabled, indicate to re-raise the error. | |
raise reraise(type(error), error, _stacktrace) | |
total = self.total | |
if total is not None: | |
total -= 1 | |
connect = self.connect | |
read = self.read | |
redirect = self.redirect | |
status_count = self.status | |
other = self.other | |
cause = "unknown" | |
status = None | |
redirect_location = None | |
if error and self._is_connection_error(error): | |
# Connect retry? | |
if connect is False: | |
raise reraise(type(error), error, _stacktrace) | |
elif connect is not None: | |
connect -= 1 | |
elif error and self._is_read_error(error): | |
# Read retry? | |
if read is False or method is None or not self._is_method_retryable(method): | |
raise reraise(type(error), error, _stacktrace) | |
elif read is not None: | |
read -= 1 | |
elif error: | |
# Other retry? | |
if other is not None: | |
other -= 1 | |
elif response and response.get_redirect_location(): | |
# Redirect retry? | |
if redirect is not None: | |
redirect -= 1 | |
cause = "too many redirects" | |
response_redirect_location = response.get_redirect_location() | |
if response_redirect_location: | |
redirect_location = response_redirect_location | |
status = response.status | |
else: | |
# Incrementing because of a server error like a 500 in | |
# status_forcelist and the given method is in the allowed_methods | |
cause = ResponseError.GENERIC_ERROR | |
if response and response.status: | |
if status_count is not None: | |
status_count -= 1 | |
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) | |
status = response.status | |
history = self.history + ( | |
RequestHistory(method, url, error, status, redirect_location), | |
) | |
new_retry = self.new( | |
total=total, | |
connect=connect, | |
read=read, | |
redirect=redirect, | |
status=status_count, | |
other=other, | |
history=history, | |
) | |
if new_retry.is_exhausted(): | |
reason = error or ResponseError(cause) | |
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] | |
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='mm.icann.org', port=443): Max retries exceeded with url: /pipermail/accred-model/2018-August.txt.gz (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)'))) | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/retry.py:515: MaxRetryError | |
During handling of the above exception, another exception occurred: | |
self = <tests.ingress.test_pipermail.TestPipermailMessageParser object at 0x7f4de71d9520> | |
@pytest.fixture(name="msg", scope="module") | |
def get_message(self): | |
> file = requests.get( | |
"https://mm.icann.org/pipermail/accred-model/2018-August.txt.gz", | |
verify=os.path.join(CONFIG.config_path, "icann_certificate.pem"), | |
) | |
tests/ingress/test_pipermail.py:26: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/requests/api.py:73: in get | |
return request("get", url, params=params, **kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/api.py:59: in request | |
return session.request(method=method, url=url, **kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/sessions.py:589: in request | |
resp = self.send(prep, **send_kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/sessions.py:703: in send | |
r = adapter.send(request, **kwargs) | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <requests.adapters.HTTPAdapter object at 0x7f4de6f15f40>, request = <PreparedRequest [GET]>, stream = False, timeout = Timeout(connect=None, read=None, total=None) | |
verify = '/home/sb/projects/bigbang/bigbang/icann_certificate.pem', cert = None, proxies = OrderedDict() | |
def send( | |
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None | |
): | |
"""Sends PreparedRequest object. Returns Response object. | |
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent. | |
:param stream: (optional) Whether to stream the request content. | |
:param timeout: (optional) How long to wait for the server to send | |
data before giving up, as a float, or a :ref:`(connect timeout, | |
read timeout) <timeouts>` tuple. | |
:type timeout: float or tuple or urllib3 Timeout object | |
:param verify: (optional) Either a boolean, in which case it controls whether | |
we verify the server's TLS certificate, or a string, in which case it | |
must be a path to a CA bundle to use | |
:param cert: (optional) Any user-provided SSL certificate to be trusted. | |
:param proxies: (optional) The proxies dictionary to apply to the request. | |
:rtype: requests.Response | |
""" | |
try: | |
conn = self.get_connection(request.url, proxies) | |
except LocationValueError as e: | |
raise InvalidURL(e, request=request) | |
self.cert_verify(conn, request.url, verify, cert) | |
url = self.request_url(request, proxies) | |
self.add_headers( | |
request, | |
stream=stream, | |
timeout=timeout, | |
verify=verify, | |
cert=cert, | |
proxies=proxies, | |
) | |
chunked = not (request.body is None or "Content-Length" in request.headers) | |
if isinstance(timeout, tuple): | |
try: | |
connect, read = timeout | |
timeout = TimeoutSauce(connect=connect, read=read) | |
except ValueError: | |
raise ValueError( | |
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " | |
f"or a single float to set both timeouts to the same value." | |
) | |
elif isinstance(timeout, TimeoutSauce): | |
pass | |
else: | |
timeout = TimeoutSauce(connect=timeout, read=timeout) | |
try: | |
resp = conn.urlopen( | |
method=request.method, | |
url=url, | |
body=request.body, | |
headers=request.headers, | |
redirect=False, | |
assert_same_host=False, | |
preload_content=False, | |
decode_content=False, | |
retries=self.max_retries, | |
timeout=timeout, | |
chunked=chunked, | |
) | |
except (ProtocolError, OSError) as err: | |
raise ConnectionError(err, request=request) | |
except MaxRetryError as e: | |
if isinstance(e.reason, ConnectTimeoutError): | |
# TODO: Remove this in 3.0.0: see #2811 | |
if not isinstance(e.reason, NewConnectionError): | |
raise ConnectTimeout(e, request=request) | |
if isinstance(e.reason, ResponseError): | |
raise RetryError(e, request=request) | |
if isinstance(e.reason, _ProxyError): | |
raise ProxyError(e, request=request) | |
if isinstance(e.reason, _SSLError): | |
# This branch is for urllib3 v1.22 and later. | |
> raise SSLError(e, request=request) | |
E requests.exceptions.SSLError: HTTPSConnectionPool(host='mm.icann.org', port=443): Max retries exceeded with url: /pipermail/accred-model/2018-August.txt.gz (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)'))) | |
bigbang-env/lib/python3.9/site-packages/requests/adapters.py:517: SSLError | |
____________________________________________________________ ERROR at setup of TestPipermailMessageParser.test__to_dict _____________________________________________________________ | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, method = 'GET' | |
url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), timeout = Timeout(connect=None, read=None, total=None), chunked = False | |
response_conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, preload_content = False, decode_content = False, enforce_content_length = True | |
def _make_request( | |
self, | |
conn: BaseHTTPConnection, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | None = None, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
chunked: bool = False, | |
response_conn: BaseHTTPConnection | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
enforce_content_length: bool = True, | |
) -> BaseHTTPResponse: | |
""" | |
Perform a request on a given urllib connection object taken from our | |
pool. | |
:param conn: | |
a connection from one of our connection pools | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param response_conn: | |
Set this to ``None`` if you will handle releasing the connection or | |
set the connection to have the response release it. | |
:param preload_content: | |
If True, the response's body will be preloaded during construction. | |
:param decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param enforce_content_length: | |
Enforce content length checking. Body returned by server must match | |
value of Content-Length header, if present. Otherwise, raise error. | |
""" | |
self.num_requests += 1 | |
timeout_obj = self._get_timeout(timeout) | |
timeout_obj.start_connect() | |
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) | |
try: | |
# Trigger any extra validation we need to do. | |
try: | |
> self._validate_conn(conn) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:467: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:1092: in _validate_conn | |
conn.connect() | |
bigbang-env/lib/python3.9/site-packages/urllib3/connection.py:642: in connect | |
sock_and_verified = _ssl_wrap_socket_and_match_hostname( | |
bigbang-env/lib/python3.9/site-packages/urllib3/connection.py:783: in _ssl_wrap_socket_and_match_hostname | |
ssl_sock = ssl_wrap_socket( | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/ssl_.py:469: in ssl_wrap_socket | |
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/ssl_.py:513: in _ssl_wrap_socket_impl | |
return ssl_context.wrap_socket(sock, server_hostname=server_hostname) | |
/usr/lib/python3.9/ssl.py:501: in wrap_socket | |
return self.sslsocket_class._create( | |
/usr/lib/python3.9/ssl.py:1041: in _create | |
self.do_handshake() | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <ssl.SSLSocket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>, block = False | |
@_sslcopydoc | |
def do_handshake(self, block=False): | |
self._check_connected() | |
timeout = self.gettimeout() | |
try: | |
if timeout == 0.0 and block: | |
self.settimeout(None) | |
> self._sslobj.do_handshake() | |
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129) | |
/usr/lib/python3.9/ssl.py:1310: SSLCertVerificationError | |
During handling of the above exception, another exception occurred: | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, method = 'GET', url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), redirect = False, assert_same_host = False, timeout = Timeout(connect=None, read=None, total=None) | |
pool_timeout = None, release_conn = False, chunked = False, body_pos = None, preload_content = False, decode_content = False, response_kw = {} | |
parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/pipermail/accred-model/2018-August.txt.gz', query=None, fragment=None), destination_scheme = None, conn = None | |
release_this_conn = True, http_tunnel_required = False, err = None, clean_exit = False | |
def urlopen( # type: ignore[override] | |
self, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | bool | int | None = None, | |
redirect: bool = True, | |
assert_same_host: bool = True, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
pool_timeout: int | None = None, | |
release_conn: bool | None = None, | |
chunked: bool = False, | |
body_pos: _TYPE_BODY_POSITION | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
**response_kw: typing.Any, | |
) -> BaseHTTPResponse: | |
""" | |
Get a connection from the pool and perform an HTTP request. This is the | |
lowest level call for making a request, so you'll need to specify all | |
the raw details. | |
.. note:: | |
More commonly, it's appropriate to use a convenience method | |
such as :meth:`request`. | |
.. note:: | |
`release_conn` will only behave as expected if | |
`preload_content=False` because we want to make | |
`preload_content=False` the default behaviour someday soon without | |
breaking backwards compatibility. | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param redirect: | |
If True, automatically handle redirects (status codes 301, 302, | |
303, 307, 308). Each redirect counts as a retry. Disabling retries | |
will disable redirect, too. | |
:param assert_same_host: | |
If ``True``, will make sure that the host of the pool requests is | |
consistent else will raise HostChangedError. When ``False``, you can | |
use the pool on an HTTP proxy and request foreign hosts. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param pool_timeout: | |
If set and the pool is set to block=True, then this method will | |
block for ``pool_timeout`` seconds and raise EmptyPoolError if no | |
connection is available within the time period. | |
:param bool preload_content: | |
If True, the response's body will be preloaded into memory. | |
:param bool decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param release_conn: | |
If False, then the urlopen call will not release the connection | |
back into the pool once a response is received (but will release if | |
you read the entire contents of the response such as when | |
`preload_content=True`). This is useful if you're not preloading | |
the response's content immediately. You will need to call | |
``r.release_conn()`` on the response ``r`` to return the connection | |
back into the pool. If None, it takes the value of ``preload_content`` | |
which defaults to ``True``. | |
:param bool chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param int body_pos: | |
Position to seek to in file-like body in the event of a retry or | |
redirect. Typically this won't need to be set because urllib3 will | |
auto-populate the value when needed. | |
""" | |
parsed_url = parse_url(url) | |
destination_scheme = parsed_url.scheme | |
if headers is None: | |
headers = self.headers | |
if not isinstance(retries, Retry): | |
retries = Retry.from_int(retries, redirect=redirect, default=self.retries) | |
if release_conn is None: | |
release_conn = preload_content | |
# Check host | |
if assert_same_host and not self.is_same_host(url): | |
raise HostChangedError(self, url, retries) | |
# Ensure that the URL we're connecting to is properly encoded | |
if url.startswith("/"): | |
url = to_str(_encode_target(url)) | |
else: | |
url = to_str(parsed_url.url) | |
conn = None | |
# Track whether `conn` needs to be released before | |
# returning/raising/recursing. Update this variable if necessary, and | |
# leave `release_conn` constant throughout the function. That way, if | |
# the function recurses, the original value of `release_conn` will be | |
# passed down into the recursive call, and its value will be respected. | |
# | |
# See issue #651 [1] for details. | |
# | |
# [1] <https://github.com/urllib3/urllib3/issues/651> | |
release_this_conn = release_conn | |
http_tunnel_required = connection_requires_http_tunnel( | |
self.proxy, self.proxy_config, destination_scheme | |
) | |
# Merge the proxy headers. Only done when not using HTTP CONNECT. We | |
# have to copy the headers dict so we can safely change it without those | |
# changes being reflected in anyone else's copy. | |
if not http_tunnel_required: | |
headers = headers.copy() # type: ignore[attr-defined] | |
headers.update(self.proxy_headers) # type: ignore[union-attr] | |
# Must keep the exception bound to a separate variable or else Python 3 | |
# complains about UnboundLocalError. | |
err = None | |
# Keep track of whether we cleanly exited the except block. This | |
# ensures we do proper cleanup in finally. | |
clean_exit = False | |
# Rewind body position, if needed. Record current position | |
# for future rewinds in the event of a redirect/retry. | |
body_pos = set_file_position(body, body_pos) | |
try: | |
# Request a connection from the queue. | |
timeout_obj = self._get_timeout(timeout) | |
conn = self._get_conn(timeout=pool_timeout) | |
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] | |
# Is this a closed/new connection that requires CONNECT tunnelling? | |
if self.proxy is not None and http_tunnel_required and conn.is_closed: | |
try: | |
self._prepare_proxy(conn) | |
except (BaseSSLError, OSError, SocketTimeout) as e: | |
self._raise_timeout( | |
err=e, url=self.proxy.url, timeout_value=conn.timeout | |
) | |
raise | |
# If we're going to release the connection in ``finally:``, then | |
# the response doesn't need to know about the connection. Otherwise | |
# it will also try to release it and we'll have a double-release | |
# mess. | |
response_conn = conn if not release_conn else None | |
# Make the request on the HTTPConnection object | |
> response = self._make_request( | |
conn, | |
method, | |
url, | |
timeout=timeout_obj, | |
body=body, | |
headers=headers, | |
chunked=chunked, | |
retries=retries, | |
response_conn=response_conn, | |
preload_content=preload_content, | |
decode_content=decode_content, | |
**response_kw, | |
) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:790: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, method = 'GET' | |
url = '/pipermail/accred-model/2018-August.txt.gz', body = None | |
headers = {'User-Agent': 'python-requests/2.31.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'} | |
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None), timeout = Timeout(connect=None, read=None, total=None), chunked = False | |
response_conn = <urllib3.connection.HTTPSConnection object at 0x7f4de6fc87f0>, preload_content = False, decode_content = False, enforce_content_length = True | |
def _make_request( | |
self, | |
conn: BaseHTTPConnection, | |
method: str, | |
url: str, | |
body: _TYPE_BODY | None = None, | |
headers: typing.Mapping[str, str] | None = None, | |
retries: Retry | None = None, | |
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, | |
chunked: bool = False, | |
response_conn: BaseHTTPConnection | None = None, | |
preload_content: bool = True, | |
decode_content: bool = True, | |
enforce_content_length: bool = True, | |
) -> BaseHTTPResponse: | |
""" | |
Perform a request on a given urllib connection object taken from our | |
pool. | |
:param conn: | |
a connection from one of our connection pools | |
:param method: | |
HTTP request method (such as GET, POST, PUT, etc.) | |
:param url: | |
The URL to perform the request on. | |
:param body: | |
Data to send in the request body, either :class:`str`, :class:`bytes`, | |
an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
:param headers: | |
Dictionary of custom headers to send, such as User-Agent, | |
If-None-Match, etc. If None, pool headers are used. If provided, | |
these headers completely replace any pool-specific headers. | |
:param retries: | |
Configure the number of retries to allow before raising a | |
:class:`~urllib3.exceptions.MaxRetryError` exception. | |
Pass ``None`` to retry until you receive a response. Pass a | |
:class:`~urllib3.util.retry.Retry` object for fine-grained control | |
over different types of retries. | |
Pass an integer number to retry connection errors that many times, | |
but no other types of errors. Pass zero to never retry. | |
If ``False``, then retries are disabled and any exception is raised | |
immediately. Also, instead of raising a MaxRetryError on redirects, | |
the redirect response will be returned. | |
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
:param timeout: | |
If specified, overrides the default timeout for this one | |
request. It may be a float (in seconds) or an instance of | |
:class:`urllib3.util.Timeout`. | |
:param chunked: | |
If True, urllib3 will send the body using chunked transfer | |
encoding. Otherwise, urllib3 will send the body using the standard | |
content-length form. Defaults to False. | |
:param response_conn: | |
Set this to ``None`` if you will handle releasing the connection or | |
set the connection to have the response release it. | |
:param preload_content: | |
If True, the response's body will be preloaded during construction. | |
:param decode_content: | |
If True, will attempt to decode the body based on the | |
'content-encoding' header. | |
:param enforce_content_length: | |
Enforce content length checking. Body returned by server must match | |
value of Content-Length header, if present. Otherwise, raise error. | |
""" | |
self.num_requests += 1 | |
timeout_obj = self._get_timeout(timeout) | |
timeout_obj.start_connect() | |
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) | |
try: | |
# Trigger any extra validation we need to do. | |
try: | |
self._validate_conn(conn) | |
except (SocketTimeout, BaseSSLError) as e: | |
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) | |
raise | |
# _validate_conn() starts the connection to an HTTPS proxy | |
# so we need to wrap errors with 'ProxyError' here too. | |
except ( | |
OSError, | |
NewConnectionError, | |
TimeoutError, | |
BaseSSLError, | |
CertificateError, | |
SSLError, | |
) as e: | |
new_e: Exception = e | |
if isinstance(e, (BaseSSLError, CertificateError)): | |
new_e = SSLError(e) | |
# If the connection didn't successfully connect to it's proxy | |
# then there | |
if isinstance( | |
new_e, (OSError, NewConnectionError, TimeoutError, SSLError) | |
) and (conn and conn.proxy and not conn.has_connected_to_proxy): | |
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) | |
> raise new_e | |
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129) | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:491: SSLError | |
The above exception was the direct cause of the following exception: | |
self = <requests.adapters.HTTPAdapter object at 0x7f4de6f15f40>, request = <PreparedRequest [GET]>, stream = False, timeout = Timeout(connect=None, read=None, total=None) | |
verify = '/home/sb/projects/bigbang/bigbang/icann_certificate.pem', cert = None, proxies = OrderedDict() | |
def send( | |
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None | |
): | |
"""Sends PreparedRequest object. Returns Response object. | |
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent. | |
:param stream: (optional) Whether to stream the request content. | |
:param timeout: (optional) How long to wait for the server to send | |
data before giving up, as a float, or a :ref:`(connect timeout, | |
read timeout) <timeouts>` tuple. | |
:type timeout: float or tuple or urllib3 Timeout object | |
:param verify: (optional) Either a boolean, in which case it controls whether | |
we verify the server's TLS certificate, or a string, in which case it | |
must be a path to a CA bundle to use | |
:param cert: (optional) Any user-provided SSL certificate to be trusted. | |
:param proxies: (optional) The proxies dictionary to apply to the request. | |
:rtype: requests.Response | |
""" | |
try: | |
conn = self.get_connection(request.url, proxies) | |
except LocationValueError as e: | |
raise InvalidURL(e, request=request) | |
self.cert_verify(conn, request.url, verify, cert) | |
url = self.request_url(request, proxies) | |
self.add_headers( | |
request, | |
stream=stream, | |
timeout=timeout, | |
verify=verify, | |
cert=cert, | |
proxies=proxies, | |
) | |
chunked = not (request.body is None or "Content-Length" in request.headers) | |
if isinstance(timeout, tuple): | |
try: | |
connect, read = timeout | |
timeout = TimeoutSauce(connect=connect, read=read) | |
except ValueError: | |
raise ValueError( | |
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " | |
f"or a single float to set both timeouts to the same value." | |
) | |
elif isinstance(timeout, TimeoutSauce): | |
pass | |
else: | |
timeout = TimeoutSauce(connect=timeout, read=timeout) | |
try: | |
> resp = conn.urlopen( | |
method=request.method, | |
url=url, | |
body=request.body, | |
headers=request.headers, | |
redirect=False, | |
assert_same_host=False, | |
preload_content=False, | |
decode_content=False, | |
retries=self.max_retries, | |
timeout=timeout, | |
chunked=chunked, | |
) | |
bigbang-env/lib/python3.9/site-packages/requests/adapters.py:486: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/urllib3/connectionpool.py:844: in urlopen | |
retries = retries.increment( | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = Retry(total=0, connect=None, read=False, redirect=None, status=None), method = 'GET', url = '/pipermail/accred-model/2018-August.txt.gz', response = None | |
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)')) | |
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f4de6f15e20>, _stacktrace = <traceback object at 0x7f4de7066c80> | |
def increment( | |
self, | |
method: str | None = None, | |
url: str | None = None, | |
response: BaseHTTPResponse | None = None, | |
error: Exception | None = None, | |
_pool: ConnectionPool | None = None, | |
_stacktrace: TracebackType | None = None, | |
) -> Retry: | |
"""Return a new Retry object with incremented retry counters. | |
:param response: A response object, or None, if the server did not | |
return a response. | |
:type response: :class:`~urllib3.response.BaseHTTPResponse` | |
:param Exception error: An error encountered during the request, or | |
None if the response was received successfully. | |
:return: A new ``Retry`` object. | |
""" | |
if self.total is False and error: | |
# Disabled, indicate to re-raise the error. | |
raise reraise(type(error), error, _stacktrace) | |
total = self.total | |
if total is not None: | |
total -= 1 | |
connect = self.connect | |
read = self.read | |
redirect = self.redirect | |
status_count = self.status | |
other = self.other | |
cause = "unknown" | |
status = None | |
redirect_location = None | |
if error and self._is_connection_error(error): | |
# Connect retry? | |
if connect is False: | |
raise reraise(type(error), error, _stacktrace) | |
elif connect is not None: | |
connect -= 1 | |
elif error and self._is_read_error(error): | |
# Read retry? | |
if read is False or method is None or not self._is_method_retryable(method): | |
raise reraise(type(error), error, _stacktrace) | |
elif read is not None: | |
read -= 1 | |
elif error: | |
# Other retry? | |
if other is not None: | |
other -= 1 | |
elif response and response.get_redirect_location(): | |
# Redirect retry? | |
if redirect is not None: | |
redirect -= 1 | |
cause = "too many redirects" | |
response_redirect_location = response.get_redirect_location() | |
if response_redirect_location: | |
redirect_location = response_redirect_location | |
status = response.status | |
else: | |
# Incrementing because of a server error like a 500 in | |
# status_forcelist and the given method is in the allowed_methods | |
cause = ResponseError.GENERIC_ERROR | |
if response and response.status: | |
if status_count is not None: | |
status_count -= 1 | |
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) | |
status = response.status | |
history = self.history + ( | |
RequestHistory(method, url, error, status, redirect_location), | |
) | |
new_retry = self.new( | |
total=total, | |
connect=connect, | |
read=read, | |
redirect=redirect, | |
status=status_count, | |
other=other, | |
history=history, | |
) | |
if new_retry.is_exhausted(): | |
reason = error or ResponseError(cause) | |
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] | |
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='mm.icann.org', port=443): Max retries exceeded with url: /pipermail/accred-model/2018-August.txt.gz (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)'))) | |
bigbang-env/lib/python3.9/site-packages/urllib3/util/retry.py:515: MaxRetryError | |
During handling of the above exception, another exception occurred: | |
self = <tests.ingress.test_pipermail.TestPipermailMessageParser object at 0x7f4de71d9520> | |
@pytest.fixture(name="msg", scope="module") | |
def get_message(self): | |
> file = requests.get( | |
"https://mm.icann.org/pipermail/accred-model/2018-August.txt.gz", | |
verify=os.path.join(CONFIG.config_path, "icann_certificate.pem"), | |
) | |
tests/ingress/test_pipermail.py:26: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang-env/lib/python3.9/site-packages/requests/api.py:73: in get | |
return request("get", url, params=params, **kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/api.py:59: in request | |
return session.request(method=method, url=url, **kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/sessions.py:589: in request | |
resp = self.send(prep, **send_kwargs) | |
bigbang-env/lib/python3.9/site-packages/requests/sessions.py:703: in send | |
r = adapter.send(request, **kwargs) | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
self = <requests.adapters.HTTPAdapter object at 0x7f4de6f15f40>, request = <PreparedRequest [GET]>, stream = False, timeout = Timeout(connect=None, read=None, total=None) | |
verify = '/home/sb/projects/bigbang/bigbang/icann_certificate.pem', cert = None, proxies = OrderedDict() | |
def send( | |
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None | |
): | |
"""Sends PreparedRequest object. Returns Response object. | |
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent. | |
:param stream: (optional) Whether to stream the request content. | |
:param timeout: (optional) How long to wait for the server to send | |
data before giving up, as a float, or a :ref:`(connect timeout, | |
read timeout) <timeouts>` tuple. | |
:type timeout: float or tuple or urllib3 Timeout object | |
:param verify: (optional) Either a boolean, in which case it controls whether | |
we verify the server's TLS certificate, or a string, in which case it | |
must be a path to a CA bundle to use | |
:param cert: (optional) Any user-provided SSL certificate to be trusted. | |
:param proxies: (optional) The proxies dictionary to apply to the request. | |
:rtype: requests.Response | |
""" | |
try: | |
conn = self.get_connection(request.url, proxies) | |
except LocationValueError as e: | |
raise InvalidURL(e, request=request) | |
self.cert_verify(conn, request.url, verify, cert) | |
url = self.request_url(request, proxies) | |
self.add_headers( | |
request, | |
stream=stream, | |
timeout=timeout, | |
verify=verify, | |
cert=cert, | |
proxies=proxies, | |
) | |
chunked = not (request.body is None or "Content-Length" in request.headers) | |
if isinstance(timeout, tuple): | |
try: | |
connect, read = timeout | |
timeout = TimeoutSauce(connect=connect, read=read) | |
except ValueError: | |
raise ValueError( | |
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " | |
f"or a single float to set both timeouts to the same value." | |
) | |
elif isinstance(timeout, TimeoutSauce): | |
pass | |
else: | |
timeout = TimeoutSauce(connect=timeout, read=timeout) | |
try: | |
resp = conn.urlopen( | |
method=request.method, | |
url=url, | |
body=request.body, | |
headers=request.headers, | |
redirect=False, | |
assert_same_host=False, | |
preload_content=False, | |
decode_content=False, | |
retries=self.max_retries, | |
timeout=timeout, | |
chunked=chunked, | |
) | |
except (ProtocolError, OSError) as err: | |
raise ConnectionError(err, request=request) | |
except MaxRetryError as e: | |
if isinstance(e.reason, ConnectTimeoutError): | |
# TODO: Remove this in 3.0.0: see #2811 | |
if not isinstance(e.reason, NewConnectionError): | |
raise ConnectTimeout(e, request=request) | |
if isinstance(e.reason, ResponseError): | |
raise RetryError(e, request=request) | |
if isinstance(e.reason, _ProxyError): | |
raise ProxyError(e, request=request) | |
if isinstance(e.reason, _SSLError): | |
# This branch is for urllib3 v1.22 and later. | |
> raise SSLError(e, request=request) | |
E requests.exceptions.SSLError: HTTPSConnectionPool(host='mm.icann.org', port=443): Max retries exceeded with url: /pipermail/accred-model/2018-August.txt.gz (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)'))) | |
bigbang-env/lib/python3.9/site-packages/requests/adapters.py:517: SSLError | |
_________________________________________________________ ERROR at setup of TestPipermailMailList.test__mailinglist_content _________________________________________________________ | |
self = <tests.ingress.test_pipermail.TestPipermailMailList object at 0x7f4de71d9940> | |
@pytest.fixture(name="mlist", scope="module") | |
def get_mailinglist_from_url(self): | |
> mlist = PipermailMailList.from_url( | |
name="accred-model", | |
url=url_list, | |
select={ | |
"years": 2018, | |
"fields": "total", | |
}, | |
) | |
tests/ingress/test_pipermail.py:60: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang/ingress/pipermail.py:256: in from_url | |
period_urls = cls.get_period_urls(url, select) | |
bigbang/ingress/pipermail.py:348: in get_period_urls | |
periods, urls_of_periods = cls.get_all_periods_and_their_urls(url) | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
url = 'https://mm.icann.org/pipermail/accred-model' | |
@staticmethod | |
def get_all_periods_and_their_urls( | |
url: str, | |
) -> Tuple[List[str], List[str]]: | |
""" | |
Pipermail groups messages into monthly time bundles. This method | |
obtains all the URLs of time bundles and are downaloadable as gzip'd files. | |
Returns | |
------- | |
Returns a tuple of two lists that look like: | |
(['April 2017', 'January 2001', ...], ['ulr1', 'url2', ...]) | |
""" | |
# wait between loading messages, for politeness | |
time.sleep(0.5) | |
soup = get_website_content( | |
url, | |
verify=os.path.join(CONFIG.config_path, "icann_certificate.pem"), | |
) | |
periods = [] | |
urls_of_periods = [] | |
if soup != "RequestException": | |
> rows = soup.select(f'a[href*=".txt.gz"]') | |
E AttributeError: 'NoneType' object has no attribute 'select' | |
bigbang/ingress/pipermail.py:394: AttributeError | |
_______________________________________________________________ ERROR at setup of TestPipermailMailList.test__to_dict _______________________________________________________________ | |
self = <tests.ingress.test_pipermail.TestPipermailMailList object at 0x7f4de71d9940> | |
@pytest.fixture(name="mlist", scope="module") | |
def get_mailinglist_from_url(self): | |
> mlist = PipermailMailList.from_url( | |
name="accred-model", | |
url=url_list, | |
select={ | |
"years": 2018, | |
"fields": "total", | |
}, | |
) | |
tests/ingress/test_pipermail.py:60: | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
bigbang/ingress/pipermail.py:256: in from_url | |
period_urls = cls.get_period_urls(url, select) | |
bigbang/ingress/pipermail.py:348: in get_period_urls | |
periods, urls_of_periods = cls.get_all_periods_and_their_urls(url) | |
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | |
url = 'https://mm.icann.org/pipermail/accred-model' | |
@staticmethod | |
def get_all_periods_and_their_urls( | |
url: str, | |
) -> Tuple[List[str], List[str]]: | |
""" | |
Pipermail groups messages into monthly time bundles. This method | |
obtains all the URLs of time bundles and are downaloadable as gzip'd files. | |
Returns | |
------- | |
Returns a tuple of two lists that look like: | |
(['April 2017', 'January 2001', ...], ['ulr1', 'url2', ...]) | |
""" | |
# wait between loading messages, for politeness | |
time.sleep(0.5) | |
soup = get_website_content( | |
url, | |
verify=os.path.join(CONFIG.config_path, "icann_certificate.pem"), | |
) | |
periods = [] | |
urls_of_periods = [] | |
if soup != "RequestException": | |
> rows = soup.select(f'a[href*=".txt.gz"]') | |
E AttributeError: 'NoneType' object has no attribute 'select' | |
bigbang/ingress/pipermail.py:394: AttributeError |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment