Skip to content

Instantly share code, notes, and snippets.

@talha08
Created March 25, 2016 19:37
Show Gist options
  • Save talha08/1fa19c7ba2ba996f2876 to your computer and use it in GitHub Desktop.
Save talha08/1fa19c7ba2ba996f2876 to your computer and use it in GitHub Desktop.
//http://www.lglproperties.com
//complete
//get url
public function lglproperties()
{
$ch = curl_init("https://lgl.appfolio.com/listings/listings?utf8=%E2%9C%93&filters%5Border_by%5D=rent_desc&filters%5Bproperty_list%5D=&filters%5Bmarket_rent_from%5D=&filters%5Bmarket_rent_to%5D=&filters%5Bbedrooms%5D=&filters%5Bbathrooms%5D=&filters%5Bcities%5D%5B%5D=&filters%5Bpostal_codes%5D%5B%5D=");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$page = curl_exec($ch);
$dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML($page);
libxml_clear_errors();
$xpath = new DOMXpath($dom);
$data = array();
$table_rows = $xpath->query('//h2[@class="listing-item__title js-listing-title"]/a/@href');
foreach ($table_rows as $row => $p) {
foreach ($p->childNodes as $p) {
$data[$row][] = preg_replace('~[\r\n]+~', '', trim($p->nodeValue));
}
//avoid all empty
$data[$row] = array_values(array_filter($data[$row]));
$crawl = new Crawler();
for ($j = 0; $j < count($data); $j++) {
$crawl->web_add = 'http://www.lglproperties.com';
$crawl->detailurl = 'https://lgl.appfolio.com'.$data[$j][0];
}
try {
$crawl->save();
} catch (Exception $ex) {
}
}
}
//http://www.lglproperties.com
//complete
//get all data
public function lglpropertiesData(){
$count = Crawler::where('detailurl', 'like', 'https://lgl.appfolio.com/listings/listings/%')
->get();
foreach ($count as $countNum) {
$ch = curl_init($countNum->detailurl);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$page = curl_exec($ch);
$dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML($page);
libxml_clear_errors();
$xpath = new DOMXpath($dom);
$data = array();
//address
$table_rows = $xpath->query('//div[@class="info"]//h1[@class="align_left"]');
foreach ($table_rows as $row => $tr) {
foreach ($tr->childNodes as $td) {
$data[$row][] = preg_replace('~[\r\n]+~', '', trim($td->nodeValue));
}
$data[$row] = array_values(array_filter($data[$row]));
}
$d= $data[0][0];
preg_match('/[0-9]{5}/', $d, $matches);
if(isset($matches[0])){
$zip= $matches[0];
}
else $zip= 0;
$city = preg_replace('/\d+/u', '', $d);
Crawler::where('detailurl', $countNum->detailurl)->update([
'zip_code' => $zip,
'city' => $city,
'prop_add' => $d
]);
//rent ,deposit
$table_rows = $xpath->query('//div[@class="one_third_column"]/div[@class="u-clearfix"]');
$data = array();
foreach ($table_rows as $row => $tr) {
foreach ($tr->childNodes as $td) {
$data[$row][] = preg_replace('~[\r\n]+~', '', trim($td->nodeValue));
}
$data[$row] = array_values(array_filter($data[$row]));
}
Crawler::where('detailurl', $countNum->detailurl)->update([
'rent' => $data[0][1],
'deposit' => $data[2][1]
]);
//bed, bath
$table_rows = $xpath->query('//div[@class="dark_grey_box small_text"]');
$data = array();
foreach ($table_rows as $row => $tr) {
foreach ($tr->childNodes as $td) {
$data[$row][] = preg_replace('~[\r\n]+~', '', trim($td->nodeValue));
}
$data[$row] = array_values(array_filter($data[$row]));
}
$d =$data [0][0];
preg_match_all('/\d+(\.\d+)?/', $d, $matches);
$dat= $matches[0];
$bedroom = $dat[0];
$bathroom = $dat[1];
Crawler::where('detailurl', $countNum->detailurl)->update([
'bedrooms' => $bedroom,
'baths' => $bathroom
]);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment