Algoritmo de paginação e stream de dados

query.py

class Query:
    def __init__(
        self, session, query, parameters=None, limit=None, pack_size=1000
    ):
        self.session = session
        self.query = query
        self.parameters = parameters
        self.pack_size = pack_size
        self.limit = limit

    def paginate(self):
        offset = 0
        max_limit = self.limit or self.pack_size
        while not self.limit or offset * self.pack_size < self.limit:
            results = self.session.execute(
                text(
                    "{query} LIMIT {limit}{offset}".format(
                        query=self.query,
                        limit=max(
                            min(
                                self.pack_size,
                                max_limit - offset * self.pack_size,
                            ),
                            0,
                        ),
                        offset=" OFFSET {}".format(offset * self.pack_size)
                        if offset
                        else "",
                    )
                ),
                self.parameters,
            ).fetchall()
            if not len(results):
                break
            for row in results:
                row = dict(row)
                yield tuple([(key, value) for key, value in row.items()])
            offset += 1