-
-
Save andreaselia/962dc9060b3b77308ab61bfd8bfa7449 to your computer and use it in GitHub Desktop.
<?php | |
namespace App\Console\Commands; | |
use Illuminate\Console\Command; | |
class ScrapeFunko extends Command | |
{ | |
/** | |
* The name and signature of the console command. | |
* | |
* @var string | |
*/ | |
protected $signature = 'scrape:funko'; | |
/** | |
* The console command description. | |
* | |
* @var string | |
*/ | |
protected $description = 'Funko POP! Vinyl Scraper'; | |
/** | |
* The list of funko collection slugs. | |
* | |
* @var array | |
*/ | |
protected $collections = [ | |
'animation', | |
'disney', | |
'games', | |
'heroes', | |
'marvel', | |
'monster-high', | |
'movies', | |
'pets', | |
'rocks', | |
'sports', | |
'star-wars', | |
'television', | |
'the-vault', | |
'the-vote', | |
'ufc', | |
]; | |
/** | |
* Create a new command instance. | |
* | |
* @return void | |
*/ | |
public function __construct() | |
{ | |
parent::__construct(); | |
} | |
/** | |
* Execute the console command. | |
* | |
* @return mixed | |
*/ | |
public function handle() | |
{ | |
foreach ($collections as $collection) { | |
$this->scrape($collection); | |
} | |
} | |
/** | |
* For scraping data for the specified collection. | |
* | |
* @param string $collection | |
* @return boolean | |
*/ | |
public static function scrape($collection) | |
{ | |
$crawler = Goutte::request('GET', env('FUNKO_POP_URL').'/'.$collection); | |
$pages = ($crawler->filter('footer .pagination li')->count() > 0) | |
? $crawler->filter('footer .pagination li:nth-last-child(2)')->text() | |
: 0 | |
; | |
for ($i = 0; $i < $pages + 1; $i++) { | |
if ($i != 0) { | |
$crawler = Goutte::request('GET', env('FUNKO_POP_URL').'/'.$collection.'?page='.$i); | |
} | |
$crawler->filter('.product-item')->each(function ($node) { | |
$sku = explode('#', $node->filter('.product-sku')->text())[1]; | |
$title = trim($node->filter('.title a')->text()); | |
print_r($sku.', '.$title); | |
}); | |
} | |
return true; | |
} | |
} |
i wrote $this->collections in foreach loop and problem resolved
now the error is
[Symfony\Component\Debug\Exception\FatalThrowableError]
Class 'App\Console\Commands\Goutte' not found
i have written goutte in config/app.php
but still getting error
any body can help me with this
[GuzzleHttp\Exception\RequestException] cURL error 60: SSL certificate problem: unable to get local issuer certificate (see http://curl.haxx.se/libcurl/c/libcurl-errors.html)
[GuzzleHttp\Exception\RequestException]
cURL error 56: Recv failure: Connection reset by peer (see http://curl.haxx.se/libcurl/c/libcurl-errors.html)
how to solve this error ?
Try this :
use Goutte\Client;
$goutte = new Client();
$crawler = $goutte->request('GET', $url.'/'.$collection);
I was able to work through the errors to get the scrape command to return nothing in my terminal. How do I see the scrape itself? Moreover, how do I see it on my site? I think this is really neat.
ErrorException : Undefined variable: collections
at /var/www/test.local.com/app/Console/Commands/ScrapeFunko.php:57
Using above code:??
i want result with html tag ??? how to??
i want result with html tag ??? how to??
->html()
use \Goutte::request instead of Goutte::request if you get not found exception
Couldn't connect to server for "http://localhost/animation".
at vendor/symfony/http-client/Chunk/ErrorChunk.php:65
Couldn't connect to server for "http://localhost/animation".
at vendor/symfony/http-client/Chunk/ErrorChunk.php:65
I have no idea if this still even still works, but make sure you have the FUNKO_POP_URL
in your env file set at the funko website URL, e.g. https://www.funko.com
Help me solve this.
[ErrorException]
Undefined variable: collections