Tame PycURL

v1.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import pycurl


def iter_chunks(data, delimiter):
    chunks = data.split(delimiter)

    for chunk in chunks:
        chunk = chunk.strip()

        if chunk:
            yield chunk


class StreamConsumer(object):

    def __init__(self, conf):
        self.delimiter = conf.get('delimiter', '\r\n')
        self.limit = float(conf.get('limit', 'inf'))
        self.url = str(conf['url'])
        # ... and other config stuff

        self.counter = 0  # a record counter
        self.buff = ''  # stream buffer
        self.conn = None  # cURL object

    def _on_receive(self, data):
        self.buff += data

        if data.endswith(self.delimiter):
            op = self.buffer.strip()

            # for the possibility of multiple data chunks sent at once
            for chunk in iter_chunks(op, self.delimiter):
                # do whatever you like with the actual data
                # print chunk for example...
                self.counter += 1  # increment record counter

            if self.counter >= self.limit:
                return -1  # the most graceful way to close stream connection

    def _setup(self):
        # Force resets
        self.counter = 0
        self.buff = ''

        if self.conn:
            try:
                self.conn.close()
            except:
                pass
            finally:
                self.conn = None

        self.conn = pycurl.Curl()
        self.conn.setopt(pycurl.URL, self.url)
        # ...other configs

        # data handler
        self.conn.setopt(pycurl.WRITEFUNCTION, self._on_receive)

    def start(self):
        try:
            self._setup()
            self.conn.perform()  # stream should not return
        except pycurl.error as e:
            if e[0] == 23:
                pass  # this is the return -1 raised error code
            else:
                print self.conn.getinfo(pycurl.HTTP_CODE), e
        except Exception as e:
            print self.conn.getinfo(pycurl.HTTP_CODE), e
        finally:
            print self.counter  # number of records this session