selfoss-debug.php

$sourcesDao = new \daos\Sources();
$itemsDao = new \daos\Items();
@set_time_limit(5000);
@error_reporting(E_ERROR);
function filter($source, $title, $content) {
    if (strlen(trim($source['filter'])) != 0) {
        $resultTitle = @preg_match($source['filter'], $title);
        $resultContent = @preg_match($source['filter'], $content);
        if ($resultTitle === false || $resultContent === false) {
            pr('filter error: ' . $source['filter']);

            return true; // do not filter out item
        }
        // test filter
        if ($resultTitle == 0 && $resultContent == 0) {
            return false;
        }
    }

    return true;
}
function pr($text) {
    echo $text . PHP_EOL;
}
foreach ($sourcesDao->getByLastUpdate() as $source) {
    // logging
    pr('start fetching source "' . $source['title'] . ' (id: ' . $source['id'] . ') ');

    // get spout
    $spoutLoader = new \helpers\SpoutLoader();
    $spout = $spoutLoader->get($source['spout']);
    if ($spout === false) {
        pr('unknown spout: ' . $source['spout']);

        return;
    }
    pr('spout successfully loaded: ' . $source['spout']);

    // receive content
    pr('fetch content');
    try {
        $spout->load(
            json_decode(html_entity_decode($source['params']), true)
        );
    } catch (\Exception $e) {
        pr('error loading feed content for ' . $source['title']);
        var_dump($e);
    }

    // current date
    $minDate = new \DateTime();
    $minDate->sub(new \DateInterval('P' . \F3::get('items_lifetime') . 'D'));
    pr('minimum date: ' . $minDate->format('Y-m-d H:i:s'));

    // insert new items in database
    pr('start item fetching');

    $itemsInFeed = [];
    foreach ($spout as $item) {
        $itemsInFeed[] = $item->getId();
    }
    $itemsFound = $itemsDao->findAll($itemsInFeed, $source['id']);

    $lasticon = false;
    $itemsSeen = [];
    foreach ($spout as $item) {
        // item already in database?
        if (isset($itemsFound[$item->getId()])) {
            pr('item "' . $item->getTitle() . '" already in database.');
            $itemsSeen[] = $itemsFound[$item->getId()];
            // continue;
        }

        // test date: continue with next if item too old
        $itemDate = new \DateTime($item->getDate());
        // if date cannot be parsed it will default to epoch. Change to current time.
        if ($itemDate->getTimestamp() == 0) {
            $itemDate = new \DateTime();
        }
        if ($itemDate < $minDate) {
            pr('item "' . $item->getTitle() . '" (' . $item->getDate() . ') older than ' . \F3::get('items_lifetime') . ' days');
            // continue;
        }

        // date in future? Set current date
        $now = new \DateTime();
        if ($itemDate > $now) {
            $itemDate = $now;
        }

        // insert new item
        pr('start insertion of new item "' . $item->getTitle() . '"');

        $content = '';
        try {
            // fetch content
            $content = $item->getContent();
        } catch (\Exception $e) {
            $content = 'Error: Content not fetched. Reason: ' . $e->getMessage();
            pr('Can not fetch "' . $item->getTitle() . '"');
            var_dump($e);
        }

        // sanitize title
        $title = $item->getTitle();
        if (strlen(trim($title)) == 0) {
            $title = '[' . \F3::get('lang_no_title') . ']';
        }

        // Check sanitized title against filter
        if (filter($source, $title, $content) === false) {
            pr($title . ' filtered out');
            // continue;
        }

        // sanitize author
        $author = $item->getAuthor();

        pr('item content sanitized');

        try {
            $icon = $item->getIcon();
        } catch (\Exception $e) {
            pr('icon: error');
            var_dump($e);

            return;
        }

        $newItem = [
            'title' => $title,
            'content' => '…',
            'source' => $source['id'],
            'datetime' => $itemDate->format('Y-m-d H:i:s'),
            'uid' => $item->getId(),
            'thumbnail' => $item->getThumbnail(),
            'icon' => $icon !== false ? $icon : '',
            'link' => htmLawed($item->getLink(), ['deny_attribute' => '*', 'elements' => '-*']),
            'author' => $author
        ];

        // save thumbnail
        $thumbnail = $item->getThumbnail();

        if (strlen(trim($thumbnail)) > 0) {
            $extension = 'jpg';
            $imageHelper = new \helpers\Image();
            $thumbnailAsJpg = $imageHelper->loadImage($thumbnail, $extension, 500, 500);
            if ($thumbnailAsJpg === false) {
                $newItem['thumbnail'] = '';
                pr('thumbnail generation error: ' . $thumbnail);
            }
        }

        $icon = $item->getIcon();
        // save icon
        if (strlen(trim($icon)) > 0) {
            $extension = 'png';
            if ($icon == $lasticon) {
                pr('use last icon: ' . $lasticon);
                $newItem['icon'] = md5($lasticon) . '.' . $extension;
            } else {
                $imageHelper = new \helpers\Image();
                $iconAsPng = $imageHelper->loadImage($icon, $extension, 30, null);
                if ($iconAsPng !== false) {
                    $lasticon = $icon;
                } else {
                    $newItem['icon'] = '';
                    pr('icon generation error: ' . $icon);
                }
            }
        } else {
            pr('no icon for this feed');
        }

        var_dump($newItem);

        pr('Memory usage: ' . memory_get_usage());
        pr('Memory peak usage: ' . memory_get_peak_usage());
    }

    // destroy feed object (prevent memory issues)
    pr('destroy spout object');
    $spout->destroy();
    pr(str_repeat(PHP_EOL, 10) . '==============================================================');
}
die;