gistfile1.py

class RefreshAllFeeds(PeriodicTask):
    """Periodic Task to refresh all the feeds.

    Splits the feeds into slices, depending how many feeds there are in
    total and how many iterations you want it to run in.

    :keyword iterations: The number of iterations you want the
        work to complete in (default: 4).

    """
    routing_key = ".".join([ROUTING_KEY_PREFIX, "allrefresh"])
    run_every = REFRESH_EVERY
    ignore_result = True

    def run(self, iterations=4, **kwargs):
        total = get_feeds.count()
        win = REFRESH_EVERY * 0.80
        size = ceil(win / iterations) * floor(total / win)

        for i in xrange(iterations):
            RefreshFeedSlice.apply_async((i*size, (i+1)*size),
                                   countdown=ceil(win / iterations)*i)