$sourcesDao = new \daos\Sources(); | |
$itemsDao = new \daos\Items(); | |
@set_time_limit(5000); | |
@error_reporting(E_ERROR); | |
function filter($source, $title, $content) { | |
if (strlen(trim($source['filter'])) != 0) { | |
$resultTitle = @preg_match($source['filter'], $title); | |
$resultContent = @preg_match($source['filter'], $content); | |
if ($resultTitle === false || $resultContent === false) { | |
pr('filter error: ' . $source['filter']); | |
return true; // do not filter out item | |
} | |
// test filter | |
if ($resultTitle == 0 && $resultContent == 0) { | |
return false; | |
} | |
} | |
return true; | |
} | |
function pr($text) { | |
echo $text . PHP_EOL; | |
} | |
foreach ($sourcesDao->getByLastUpdate() as $source) { | |
// logging | |
pr('start fetching source "' . $source['title'] . ' (id: ' . $source['id'] . ') '); | |
// get spout | |
$spoutLoader = new \helpers\SpoutLoader(); | |
$spout = $spoutLoader->get($source['spout']); | |
if ($spout === false) { | |
pr('unknown spout: ' . $source['spout']); | |
return; | |
} | |
pr('spout successfully loaded: ' . $source['spout']); | |
// receive content | |
pr('fetch content'); | |
try { | |
$spout->load( | |
json_decode(html_entity_decode($source['params']), true) | |
); | |
} catch (\Exception $e) { | |
pr('error loading feed content for ' . $source['title']); | |
var_dump($e); | |
} | |
// current date | |
$minDate = new \DateTime(); | |
$minDate->sub(new \DateInterval('P' . \F3::get('items_lifetime') . 'D')); | |
pr('minimum date: ' . $minDate->format('Y-m-d H:i:s')); | |
// insert new items in database | |
pr('start item fetching'); | |
$itemsInFeed = []; | |
foreach ($spout as $item) { | |
$itemsInFeed[] = $item->getId(); | |
} | |
$itemsFound = $itemsDao->findAll($itemsInFeed, $source['id']); | |
$lasticon = false; | |
$itemsSeen = []; | |
foreach ($spout as $item) { | |
// item already in database? | |
if (isset($itemsFound[$item->getId()])) { | |
pr('item "' . $item->getTitle() . '" already in database.'); | |
$itemsSeen[] = $itemsFound[$item->getId()]; | |
// continue; | |
} | |
// test date: continue with next if item too old | |
$itemDate = new \DateTime($item->getDate()); | |
// if date cannot be parsed it will default to epoch. Change to current time. | |
if ($itemDate->getTimestamp() == 0) { | |
$itemDate = new \DateTime(); | |
} | |
if ($itemDate < $minDate) { | |
pr('item "' . $item->getTitle() . '" (' . $item->getDate() . ') older than ' . \F3::get('items_lifetime') . ' days'); | |
// continue; | |
} | |
// date in future? Set current date | |
$now = new \DateTime(); | |
if ($itemDate > $now) { | |
$itemDate = $now; | |
} | |
// insert new item | |
pr('start insertion of new item "' . $item->getTitle() . '"'); | |
$content = ''; | |
try { | |
// fetch content | |
$content = $item->getContent(); | |
} catch (\Exception $e) { | |
$content = 'Error: Content not fetched. Reason: ' . $e->getMessage(); | |
pr('Can not fetch "' . $item->getTitle() . '"'); | |
var_dump($e); | |
} | |
// sanitize title | |
$title = $item->getTitle(); | |
if (strlen(trim($title)) == 0) { | |
$title = '[' . \F3::get('lang_no_title') . ']'; | |
} | |
// Check sanitized title against filter | |
if (filter($source, $title, $content) === false) { | |
pr($title . ' filtered out'); | |
// continue; | |
} | |
// sanitize author | |
$author = $item->getAuthor(); | |
pr('item content sanitized'); | |
try { | |
$icon = $item->getIcon(); | |
} catch (\Exception $e) { | |
pr('icon: error'); | |
var_dump($e); | |
return; | |
} | |
$newItem = [ | |
'title' => $title, | |
'content' => '…', | |
'source' => $source['id'], | |
'datetime' => $itemDate->format('Y-m-d H:i:s'), | |
'uid' => $item->getId(), | |
'thumbnail' => $item->getThumbnail(), | |
'icon' => $icon !== false ? $icon : '', | |
'link' => htmLawed($item->getLink(), ['deny_attribute' => '*', 'elements' => '-*']), | |
'author' => $author | |
]; | |
// save thumbnail | |
$thumbnail = $item->getThumbnail(); | |
if (strlen(trim($thumbnail)) > 0) { | |
$extension = 'jpg'; | |
$imageHelper = new \helpers\Image(); | |
$thumbnailAsJpg = $imageHelper->loadImage($thumbnail, $extension, 500, 500); | |
if ($thumbnailAsJpg === false) { | |
$newItem['thumbnail'] = ''; | |
pr('thumbnail generation error: ' . $thumbnail); | |
} | |
} | |
$icon = $item->getIcon(); | |
// save icon | |
if (strlen(trim($icon)) > 0) { | |
$extension = 'png'; | |
if ($icon == $lasticon) { | |
pr('use last icon: ' . $lasticon); | |
$newItem['icon'] = md5($lasticon) . '.' . $extension; | |
} else { | |
$imageHelper = new \helpers\Image(); | |
$iconAsPng = $imageHelper->loadImage($icon, $extension, 30, null); | |
if ($iconAsPng !== false) { | |
$lasticon = $icon; | |
} else { | |
$newItem['icon'] = ''; | |
pr('icon generation error: ' . $icon); | |
} | |
} | |
} else { | |
pr('no icon for this feed'); | |
} | |
var_dump($newItem); | |
pr('Memory usage: ' . memory_get_usage()); | |
pr('Memory peak usage: ' . memory_get_peak_usage()); | |
} | |
// destroy feed object (prevent memory issues) | |
pr('destroy spout object'); | |
$spout->destroy(); | |
pr(str_repeat(PHP_EOL, 10) . '=============================================================='); | |
} | |
die; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment