adiroiban/example run

## example run

Python 2.7.12 (default, Jul  1 2016, 15:12:24)
>>> import cryptography
>>> import OpenSSL
>>> cryptography.__version__
'1.7.2'
>>> OpenSSL.__version__
'16.2.0'
>>>

For google.com I got

$ py docs/web/examples/httpclient.py https://www.google.com
The response body will consist of 257 bytes.
Got some: <HTML><HEAD><meta http-equiv="content-type" conten
Response done
-----------------------
1488704564.24 after-done
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
-----------------------
1488704564.34 before-close
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
-----------------------
1488704564.38 after-close
-----------------------
getReaders: [<twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]

For something like https://maranet.sharepoint.com

$ py docs/web/examples/httpclient.py https://maranet.sharepoint.com
The response body will consist of 13 bytes.
Got some: 403 FORBIDDEN
Response done
-----------------------
1488705165.79 after-done
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
-----------------------
1488705165.89 before-close
-----------------------
getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
-----------------------
1488705283.68 after-close
-----------------------
getReaders: [<twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]

## httpsclient.py
#!/usr/bin/env python
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.

"""
This example demonstrates how to make a simple http client.

Usage:
    httpclient.py <url>

Don't forget the http:// when you type the web address!
"""

from __future__ import print_function

import sys
import time
from pprint import pprint

from twisted import version
from twisted.python import log
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.web.iweb import UNKNOWN_LENGTH
from twisted.web.http_headers import Headers
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone


class WriteToStdout(Protocol):
    def connectionMade(self):
        self.onConnLost = Deferred()

    def dataReceived(self, data):
        """
        Print out the html page received.
        """
        print('Got some:', data[:50])

    def connectionLost(self, reason):
        if not reason.check(ResponseDone):
            reason.printTraceback()
        else:
            print('Response done')
        self.onConnLost.callback(None)

def cb_connection_done(result, pool):
    """
    Called after the HTTP request was done.
    """
    show_reactor('after-done')
    reactor.callLater(0.1, cl_close, pool)

@inlineCallbacks
def cl_close(pool):
    """
    Called to close the persistent pool.
    """
    show_reactor('before-close')
    yield pool.closeCachedConnections()
    show_reactor('after-close')
    reactor.stop()

def show_reactor(step):
    print('-----------------------')
    print('%s %s' % (time.time(), step))
    print('-----------------------')
    print('getReaders: %s' % (reactor.getReaders(),))


def main(reactor, url):
    """
    We create a custom UserAgent and send a GET request to a web server.
    """
    userAgent = 'Twisted/%s (httpclient.py)' % (version.short(),)
    pool = HTTPConnectionPool(reactor, persistent=True)
    agent = Agent(reactor, pool=pool)
    d = agent.request(
        'GET', url, Headers({'user-agent': [userAgent]}))
    def cbResponse(response):
        """
        Prints out the response returned by the web server.
        """
        proto = WriteToStdout()
        if response.length is not UNKNOWN_LENGTH:
            print('The response body will consist of', response.length, 'bytes.')
        else:
            print('The response body length is unknown.')
        response.deliverBody(proto)
        return proto.onConnLost
    d.addCallback(cbResponse)
    d.addBoth(cb_connection_done, pool)
    d.addErrback(log.err)
    reactor.run()


if __name__ == '__main__':
    main(reactor, *sys.argv[1:])

## patch_tls.py
def patch_shutdownTLS(self):
    """
    Initiate, or reply to, the shutdown handshake of the TLS layer.
    """

    def try_shutdown():
        try:
            shutdownSuccess = self._tlsConnection.shutdown()
        except SSL.Error:
            # Mid-handshake, a call to shutdown() can result in a
            # WantWantReadError, or rather an SSL_ERR_WANT_READ; but pyOpenSSL
            # doesn't allow us to get at the error.  See:
            # https://github.com/pyca/pyopenssl/issues/91
            #
            # If the underlying BIO is non-blocking, SSL_shutdown() will also
            # return when the underlying BIO could not satisfy the needs of
            # SSL_shutdown() to continue the handshake.
            # In this case a call to SSL_get_error() with the return value of
            # SSL_shutdown() will yield SSL_ERROR_WANT_READ or
            # SSL_ERROR_WANT_WRITE.
            # The calling process then must repeat the call after taking
            # appropriate action to satisfy the needs of SSL_shutdown()
            shutdownSuccess = False

        self._flushSendBIO()

        if shutdownSuccess:
            # Both sides have shutdown.
            # This will also happen if we haven't started
            # negotiation at all yet, in which case shutdown succeeds
            # immediately.
            if self._loseConnectionOnTLSShutdown:
                # We will lose the connection on shutdown.
                self.transport.loseConnection()
            return

        try:
            low_transport = self.transport.socket
        except AttributeError:
            # The transport was already closed.
            return

        shutdown_start = getattr(self, '__shutdown_start', None)
        if shutdown_start is None:
            self.__shutdown_start = time()

        if time() - self.__shutdown_start > 2:
            self.transport.loseConnection()
            return

        # The shutdown was not successful.
        # We assume that it was due to WantWantReadError, so we try to trigger
        # the shutdown state by reading the low level socket and passing it
        # to the BIO.
        try:
            data = low_transport.recv(2 ** 15)
            if data:
                # Only write if we got something on the wire.
                self._tlsConnection.bio_write(data)
            self._flushReceiveBIO()
        except socket.error as error:
            if error.errno in [errno.EAGAIN, tcp.EWOULDBLOCK]:
                # Socket is not yet ready
                reactor.callLater(0.01, try_shutdown)
            elif error.errno == tcp.ECONNRESET:
                # Remote was already closed.
                self.transport.loseConnection()
            else:
                raise

    try_shutdown()


TLSMemoryBIOProtocol._shutdownTLS = patch_shutdownTLS

	Python 2.7.12 (default, Jul 1 2016, 15:12:24)
	>>> import cryptography
	>>> import OpenSSL
	>>> cryptography.__version__
	'1.7.2'
	>>> OpenSSL.__version__
	'16.2.0'
	>>>

	For google.com I got

	$ py docs/web/examples/httpclient.py https://www.google.com
	The response body will consist of 257 bytes.
	Got some: <HTML><HEAD><meta http-equiv="content-type" conten
	Response done
	-----------------------
	1488704564.24 after-done
	-----------------------
	getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
	-----------------------
	1488704564.34 before-close
	-----------------------
	getReaders: [<<class 'twisted.internet.tcp.Client'> to ('www.google.com', 443) at 7fd3197eb750>, <twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]
	-----------------------
	1488704564.38 after-close
	-----------------------
	getReaders: [<twisted.internet.posixbase._UnixWaker object at 0x7fd31a052e50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7fd3197eb790>]

	For something like https://maranet.sharepoint.com

	$ py docs/web/examples/httpclient.py https://maranet.sharepoint.com
	The response body will consist of 13 bytes.
	Got some: 403 FORBIDDEN
	Response done
	-----------------------
	1488705165.79 after-done
	-----------------------
	getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
	-----------------------
	1488705165.89 before-close
	-----------------------
	getReaders: [<<class 'twisted.internet.tcp.Client'> to ('104.146.222.51', 443) at 7f7aa546ec50>, <twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
	-----------------------
	1488705283.68 after-close
	-----------------------
	getReaders: [<twisted.internet.posixbase._SIGCHLDWaker object at 0x7f7aa57ac310>, <twisted.internet.posixbase._UnixWaker object at 0x7f7aa5c63f10>]
	#!/usr/bin/env python
	# Copyright (c) Twisted Matrix Laboratories.
	# See LICENSE for details.

	"""
	This example demonstrates how to make a simple http client.

	Usage:
	httpclient.py <url>

	Don't forget the http:// when you type the web address!
	"""

	from __future__ import print_function

	import sys
	import time
	from pprint import pprint

	from twisted import version
	from twisted.python import log
	from twisted.internet.defer import Deferred, inlineCallbacks
	from twisted.internet import reactor
	from twisted.internet.protocol import Protocol
	from twisted.web.iweb import UNKNOWN_LENGTH
	from twisted.web.http_headers import Headers
	from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone


	class WriteToStdout(Protocol):
	def connectionMade(self):
	self.onConnLost = Deferred()

	def dataReceived(self, data):
	"""
	Print out the html page received.
	"""
	print('Got some:', data[:50])

	def connectionLost(self, reason):
	if not reason.check(ResponseDone):
	reason.printTraceback()
	else:
	print('Response done')
	self.onConnLost.callback(None)

	def cb_connection_done(result, pool):
	"""
	Called after the HTTP request was done.
	"""
	show_reactor('after-done')
	reactor.callLater(0.1, cl_close, pool)

	@inlineCallbacks
	def cl_close(pool):
	"""
	Called to close the persistent pool.
	"""
	show_reactor('before-close')
	yield pool.closeCachedConnections()
	show_reactor('after-close')
	reactor.stop()

	def show_reactor(step):
	print('-----------------------')
	print('%s %s' % (time.time(), step))
	print('-----------------------')
	print('getReaders: %s' % (reactor.getReaders(),))


	def main(reactor, url):
	"""
	We create a custom UserAgent and send a GET request to a web server.
	"""
	userAgent = 'Twisted/%s (httpclient.py)' % (version.short(),)
	pool = HTTPConnectionPool(reactor, persistent=True)
	agent = Agent(reactor, pool=pool)
	d = agent.request(
	'GET', url, Headers({'user-agent': [userAgent]}))
	def cbResponse(response):
	"""
	Prints out the response returned by the web server.
	"""
	proto = WriteToStdout()
	if response.length is not UNKNOWN_LENGTH:
	print('The response body will consist of', response.length, 'bytes.')
	else:
	print('The response body length is unknown.')
	response.deliverBody(proto)
	return proto.onConnLost
	d.addCallback(cbResponse)
	d.addBoth(cb_connection_done, pool)
	d.addErrback(log.err)
	reactor.run()


	if __name__ == '__main__':
	main(reactor, *sys.argv[1:])
	def patch_shutdownTLS(self):
	"""
	Initiate, or reply to, the shutdown handshake of the TLS layer.
	"""

	def try_shutdown():
	try:
	shutdownSuccess = self._tlsConnection.shutdown()
	except SSL.Error:
	# Mid-handshake, a call to shutdown() can result in a
	# WantWantReadError, or rather an SSL_ERR_WANT_READ; but pyOpenSSL
	# doesn't allow us to get at the error. See:
	# https://github.com/pyca/pyopenssl/issues/91
	#
	# If the underlying BIO is non-blocking, SSL_shutdown() will also
	# return when the underlying BIO could not satisfy the needs of
	# SSL_shutdown() to continue the handshake.
	# In this case a call to SSL_get_error() with the return value of
	# SSL_shutdown() will yield SSL_ERROR_WANT_READ or
	# SSL_ERROR_WANT_WRITE.
	# The calling process then must repeat the call after taking
	# appropriate action to satisfy the needs of SSL_shutdown()
	shutdownSuccess = False

	self._flushSendBIO()

	if shutdownSuccess:
	# Both sides have shutdown.
	# This will also happen if we haven't started
	# negotiation at all yet, in which case shutdown succeeds
	# immediately.
	if self._loseConnectionOnTLSShutdown:
	# We will lose the connection on shutdown.
	self.transport.loseConnection()
	return

	try:
	low_transport = self.transport.socket
	except AttributeError:
	# The transport was already closed.
	return

	shutdown_start = getattr(self, '__shutdown_start', None)
	if shutdown_start is None:
	self.__shutdown_start = time()

	if time() - self.__shutdown_start > 2:
	self.transport.loseConnection()
	return

	# The shutdown was not successful.
	# We assume that it was due to WantWantReadError, so we try to trigger
	# the shutdown state by reading the low level socket and passing it
	# to the BIO.
	try:
	data = low_transport.recv(2 ** 15)
	if data:
	# Only write if we got something on the wire.
	self._tlsConnection.bio_write(data)
	self._flushReceiveBIO()
	except socket.error as error:
	if error.errno in [errno.EAGAIN, tcp.EWOULDBLOCK]:
	# Socket is not yet ready
	reactor.callLater(0.01, try_shutdown)
	elif error.errno == tcp.ECONNRESET:
	# Remote was already closed.
	self.transport.loseConnection()
	else:
	raise

	try_shutdown()


	TLSMemoryBIOProtocol._shutdownTLS = patch_shutdownTLS