I/O implementation leveraging requests' chunked response handling

chunked.py

from io import RawIOBase


class ChunkedResponseIO(RawIOBase):
    """
    Turns a generator of chunks into raw I/O, which can in turn be fed to 
    something like an `io.BufferedReader`.
    
    .. code-block:: python
    
        # requesting an imaginary big file, to be sent chunked
        response = requests.get('http://big-files.net/big-file.bin', 
                                stream=True)  # make sure to request a stream
        # using 8096 bytes chunk size, turn the response into a BufferedReader
        reader = io.BufferedReader(ChunkedResponseIO(
            chunk for chunk in response.iter_content(8096)
        ))
    """

    def __init__(self, chunks):
        self._chunks = chunks
        self._current = None

    def seekable(self):
        return False

    def readable(self):
        return True

    def writable(self):
        return False

    def readinto(self, b):
        try:
            # reuse current (partial) chunk or use the next chunk if the current one is None or empty
            self._current = self._current or next(self._chunks)
            # determine the max number of bytes to read
            num = len(b)
            # split current chunk into a buffer to be used right now and one for the next call
            # note that readinto is not required to *fill* b, even if we could
            buf, self._current = self._current[:num], self._current[num:]
            # put the data to be used now into b
            b[:len(buf)] = buf
            # indicate the amount of bytes we've put into b
            return len(buf)
        except StopIteration:
            # raised when both self._current and self._chunks are empty: end of stream
            return 0