Skip to content

Instantly share code, notes, and snippets.

@remoharsono
Created February 10, 2019 03:00
Show Gist options
  • Save remoharsono/1448c8756849dcd3a456e8d4d5525290 to your computer and use it in GitHub Desktop.
Save remoharsono/1448c8756849dcd3a456e8d4d5525290 to your computer and use it in GitHub Desktop.
PHP cURL - Set User Agent as Google Bot
<?php
// taken from: https://github.com/izniburak/google-bot-curl/blob/master/google-bot.php
function googleBot($url)
{
$header = array();
$header[] = 'Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5';
$header[] = 'Cache-Control: max-age=0';
$header[] = 'Content-Type: text/html; charset=utf-8';
$header[] = 'Transfer-Encoding: chunked';
$header[] = 'Connection: keep-alive';
$header[] = 'Keep-Alive: 300';
$header[] = 'Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7';
$header[] = 'Accept-Language: en-us,en;q=0.5';
$header[] = 'Pragma:';
$curl = curl_init();
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
curl_setopt($curl, CURLOPT_HTTPHEADER, $header);
curl_setopt($curl, CURLOPT_REFERER, 'http://www.google.com');
curl_setopt($curl, CURLOPT_ENCODING, 'gzip, deflate');
curl_setopt($curl, CURLOPT_AUTOREFERER, true);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_TIMEOUT, 10);
$body = curl_exec($curl);
curl_close($curl);
return $body;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment