gistfile1.txt

2013-12-21 13:28:18,725 tasks ERROR --- GEVENT STACKTRACE 0:   File "/usr/local/lib/python2.7/dist-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/concurrency/base.py", line 34, in apply_target
    ret = target(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 327, in _fast_trace_task
    return _tasks[task].__trace__(uuid, args, kwargs, request)[0]
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 238, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 416, in __protected_call__
    return self.run(*args, **kwargs)
  File "/home/servictigo/src/servictigo/tasks.py", line 272, in dumpstacks
    logger.error('--- GEVENT STACKTRACE %d: %s', i, ''.join(traceback.format_stack(ob.gr_frame)))

2013-12-21 13:28:18,728 tasks ERROR --- GEVENT STACKTRACE 1:   File "/usr/local/bin/celery", line 9, in <module>
    load_entry_point('celery==3.1.7', 'console_scripts', 'celery')()
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 30, in main
    main()
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 80, in main
    cmd.execute_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 723, in execute_from_commandline
    super(CeleryCommand, self).execute_from_commandline(argv)))
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 303, in execute_from_commandline
    return self.handle_argv(self.prog_name, argv[1:])
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 715, in handle_argv
    return self.execute(command, argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 669, in execute
    ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/worker.py", line 175, in run_from_argv
    return self(*args, **options)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 266, in __call__
    ret = self.run(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/worker.py", line 208, in run
    state_db=self.node_format(state_db, hostname), **kwargs
  File "/usr/local/lib/python2.7/dist-packages/celery/worker/__init__.py", line 206, in start
    self.blueprint.start(self)
  File "/usr/local/lib/python2.7/dist-packages/celery/bootsteps.py", line 123, in start
    step.start(parent)
  File "/usr/local/lib/python2.7/dist-packages/celery/bootsteps.py", line 373, in start
    return self.obj.start()
  File "/usr/local/lib/python2.7/dist-packages/celery/worker/consumer.py", line 270, in start
    blueprint.start(self)
  File "/usr/local/lib/python2.7/dist-packages/celery/bootsteps.py", line 123, in start
    step.start(parent)
  File "/usr/local/lib/python2.7/dist-packages/celery/worker/consumer.py", line 786, in start
    c.loop(*c.loop_args())
  File "/usr/local/lib/python2.7/dist-packages/celery/worker/loops.py", line 99, in synloop
    connection.drain_events(timeout=2.0)
  File "/usr/local/lib/python2.7/dist-packages/kombu/connection.py", line 279, in drain_events
    return self.transport.drain_events(self.connection, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/kombu/transport/virtual/__init__.py", line 830, in drain_events
    item, channel = get(timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/transport/redis.py", line 296, in get
    events = self.poller.poll(timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/utils/eventio.py", line 65, in poll
    return self._poll(timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/utils/eventio.py", line 213, in _poll
    self._rfd, self._wfd, self._efd, timeout,
  File "/usr/local/lib/python2.7/dist-packages/gevent/select.py", line 63, in select
    result.event.wait(timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/gevent/event.py", line 74, in wait
    result = get_hub().switch()
  File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 164, in switch
    return greenlet.switch(self)

2013-12-21 13:28:18,745 tasks ERROR --- GEVENT STACKTRACE 2:   File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 179, in run
    result = core.dispatch()
  File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 150, in _wait_helper
    current.switch(ev)

2013-12-21 13:28:18,746 tasks ERROR --- GEVENT STACKTRACE 3:   File "/usr/local/lib/python2.7/dist-packages/celery/worker/pidbox.py", line 110, in loop
    connection.drain_events(timeout=1.0)
  File "/usr/local/lib/python2.7/dist-packages/kombu/connection.py", line 279, in drain_events
    return self.transport.drain_events(self.connection, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/kombu/transport/virtual/__init__.py", line 830, in drain_events
    item, channel = get(timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/transport/redis.py", line 296, in get
    events = self.poller.poll(timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/utils/eventio.py", line 65, in poll
    return self._poll(timeout)
  File "/usr/local/lib/python2.7/dist-packages/kombu/utils/eventio.py", line 213, in _poll
    self._rfd, self._wfd, self._efd, timeout,
  File "/usr/local/lib/python2.7/dist-packages/gevent/select.py", line 63, in select
    result.event.wait(timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/gevent/event.py", line 74, in wait
    result = get_hub().switch()
  File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 164, in switch
    return greenlet.switch(self)

2013-12-21 13:28:18,795 tasks ERROR --- GEVENT STACKTRACE 4:   File "/usr/local/lib/python2.7/dist-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/concurrency/base.py", line 34, in apply_target
    ret = target(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 327, in _fast_trace_task
    return _tasks[task].__trace__(uuid, args, kwargs, request)[0]
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 238, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 416, in __protected_call__
    return self.run(*args, **kwargs)
  File "/home/servictigo/src/servictigo/tasks.py", line 192, in check
    timeout=timeout, verify=certificate_verification)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 335, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 438, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 292, in send
    timeout=timeout
  File "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/connectionpool.py", line 428, in urlopen
    body=body, headers=headers)
  File "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/connectionpool.py", line 288, in _make_request
    httplib_response = conn.getresponse(buffering=True)
  File "/usr/lib/python2.7/httplib.py", line 1030, in getresponse
    response.begin()
  File "/usr/lib/python2.7/httplib.py", line 407, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.7/httplib.py", line 365, in _read_status
    line = self.fp.readline()
  File "/usr/lib/python2.7/socket.py", line 447, in readline
    data = self._sock.recv(self._rbufsize)
  File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 432, in recv
    wait_read(sock.fileno(), timeout=self.timeout, event=self._read_event)
  File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 169, in wait_read
    switch_result = get_hub().switch()
  File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 164, in switch
    return greenlet.switch(self)

2013-12-21 13:28:18,805 tasks ERROR --- GEVENT STACKTRACE 5:   File "/usr/local/lib/python2.7/dist-packages/gevent/greenlet.py", line 390, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/concurrency/base.py", line 34, in apply_target
    ret = target(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 327, in _fast_trace_task
    return _tasks[task].__trace__(uuid, args, kwargs, request)[0]
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 238, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 416, in __protected_call__
    return self.run(*args, **kwargs)
  File "/home/servictigo/src/servictigo/tasks.py", line 192, in check
    timeout=timeout, verify=certificate_verification)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 335, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 454, in send
    history = [resp for resp in gen] if allow_redirects else []
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 144, in resolve_redirects
    allow_redirects=False,
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 438, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 340, in send
    r.content
  File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 594, in content
    self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
  File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 541, in generate
    chunk = self.raw.read(chunk_size, decode_content=True)
  File "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/response.py", line 171, in read
    data = self._fp.read(amt)
  File "/usr/lib/python2.7/httplib.py", line 561, in read
    s = self.fp.read(amt)
  File "/usr/lib/python2.7/socket.py", line 380, in read
    data = self._sock.recv(left)
  File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 432, in recv
    wait_read(sock.fileno(), timeout=self.timeout, event=self._read_event)
  File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 169, in wait_read
    switch_result = get_hub().switch()
  File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 164, in switch
    return greenlet.switch(self)