Skip to content

Instantly share code, notes, and snippets.

@connor11528
Last active January 6, 2024 21:50
Show Gist options
  • Save connor11528/a36676d3239884bd3193f1defef8ca57 to your computer and use it in GitHub Desktop.
Save connor11528/a36676d3239884bd3193f1defef8ca57 to your computer and use it in GitHub Desktop.
Artisan command to scrape jobs from Greenhouse.io
<?php
namespace App\Console\Commands;
use App\Company;
use App\JobListing;
use Illuminate\Console\Command;
use Goutte\Client;
class ScrapeGreenhouse extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'scrape:greenhouse';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Scrape information from greenhouse dot com';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$endpoints = [
[
'name' => 'Stitch Labs',
'endpoint' => 'stitchlabs'
],
[
'name' => 'Lob',
'endpoint' => 'lob'
],
[
'name' => 'DoorDash',
'endpoint' => 'doordash'
],
[
'name' => 'Instacart',
'endpoint' => 'instacart'
],
[
'name' => 'Sift Science',
'endpoint' => 'siftscience'
],
];
foreach ($endpoints as $endpoint) {
/** @var Company $company */
$company = Company::where('name', $endpoint['name'])->first();
$endpoint['company_id'] = $company->id;
$this->scrape($endpoint);
}
}
/**
* @param $endpoint array
* @return bool
*/
public function scrape($endpoint)
{
$crawler = new Client();
$result = $crawler->request('GET', 'https://boards.greenhouse.io/' . $endpoint['endpoint']);
$result->filter('.opening')->each(function ($node) use ($endpoint) {
$jobListing = new JobListing;
$jobListing->title = $node->filter('a')->text();
$jobListing->url = $node->filter('a')->attr('href');
$jobListing->city = $node->filter('.location')->text();
$jobListing->company_id = $endpoint['company_id'];
$jobListing->active = true;
$jobListing->save();
$this->info('Added job listing: ' . $jobListing->title . ' for ' . $endpoint['name'] . "\n");
});
return true;
}
}
@connor11528
Copy link
Author

Currently having issue with deploy to prod: Class 'Weidner\Goutte\GoutteServiceProvider' not found

dweidner/laravel-goutte#6
dweidner/laravel-goutte#12

Works locally tho...

@connor11528
Copy link
Author

Run with php artisan scrape:greenhouse. The Goutte issue above had to do with my composer dependencies and composer.lock file. Code should be all good

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment