PersistenceCollectionCrawlQueue

PersistenceCollectionCrawlQueue.php

<?php

class PersistenceCollectionCrawlQueue extends \Spatie\Crawler\CrawlQueue\CollectionCrawlQueue
{
    protected $last_saved_time;

    protected function flushQueueOnDisk()
    {
        if (!$this->last_saved_time || time() - $this->last_saved_time > 60) {
            $this->last_saved_time = time();
            file_put_contents(QUEUE_FILE, serialize([$this->urls, $this->pendingUrls]));
        }
    }

    public function __construct()
    {
        $saved_data = file_get_contents(QUEUE_FILE);
        if ($saved_data) {
            $saved_data = unserialize($saved_data);
            $this->urls = collect($saved_data[0]);
            $this->pendingUrls = collect($saved_data[1]);
        } else {
            $this->urls = collect();
            $this->pendingUrls = collect();
        }
    }

    public function add(\Spatie\Crawler\CrawlUrl $url): \Spatie\Crawler\CrawlQueue\CrawlQueue
    {
        parent::add($url);
        $this->flushQueueOnDisk();
        return $this;
    }

    public function markAsProcessed(\Spatie\Crawler\CrawlUrl $crawlUrl)
    {
        parent::markAsProcessed($crawlUrl);
        $this->flushQueueOnDisk();
    }
}