allow google bot to crawl on varnish

more-agent.vcl

sub vcl_recv {

#taken from http://www.useragentstring.com/pages/useragentstring.php

if (req.http.user-agent ~ "008") {
                return (pass);
        }
if (req.http.user-agent ~ "ABACHOBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Accoona-AI-Agent") {
                return (pass);
        }
if (req.http.user-agent ~ "AddSugarSpiderBot") {
                return (pass);
        }
if (req.http.user-agent ~ "AnyApexBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Arachmo") {
                return (pass);
        }
if (req.http.user-agent ~ "B-l-i-t-z-B-O-T") {
                return (pass);
        }
if (req.http.user-agent ~ "Baiduspider") {
                return (pass);
        }
if (req.http.user-agent ~ "BecomeBot") {
                return (pass);
        }
if (req.http.user-agent ~ "BeslistBot") {
                return (pass);
        }
if (req.http.user-agent ~ "BillyBobBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Bimbot") {
                return (pass);
        }
if (req.http.user-agent ~ "Bingbot") {
                return (pass);
        }
if (req.http.user-agent ~ "BlitzBOT") {
                return (pass);
        }
if (req.http.user-agent ~ "boitho.com-dc") {
                return (pass);
        }
if (req.http.user-agent ~ "boitho.com-robot") {
                return (pass);
        }
if (req.http.user-agent ~ "btbot") {
                return (pass);
        }
if (req.http.user-agent ~ "CatchBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Cerberian Drtrs") {
                return (pass);
        }
if (req.http.user-agent ~ "Charlotte") {
                return (pass);
        }
if (req.http.user-agent ~ "ConveraCrawler") {
                return (pass);
        }
if (req.http.user-agent ~ "cosmos") {
                return (pass);
        }
if (req.http.user-agent ~ "Covario IDS") {
                return (pass);
        }
if (req.http.user-agent ~ "DataparkSearch") {
                return (pass);
        }
if (req.http.user-agent ~ "DiamondBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Discobot") {
                return (pass);
        }
if (req.http.user-agent ~ "Dotbot") {
                return (pass);
        }
if (req.http.user-agent ~ "EARTHCOM.info") {
                return (pass);
        }
if (req.http.user-agent ~ "EmeraldShield.com WebBot") {
                return (pass);
        }
if (req.http.user-agent ~ "envolk[ITS]spider") {
                return (pass);
        }
if (req.http.user-agent ~ "EsperanzaBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Exabot") {
                return (pass);
        }
if (req.http.user-agent ~ "FAST Enterprise Crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "FAST-WebCrawler") {
                return (pass);
        }
if (req.http.user-agent ~ "FDSE robot") {
                return (pass);
        }
if (req.http.user-agent ~ "FindLinks") {
                return (pass);
        }
if (req.http.user-agent ~ "FurlBot") {
                return (pass);
        }
if (req.http.user-agent ~ "FyberSpider") {
                return (pass);
        }
if (req.http.user-agent ~ "g2crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "Gaisbot") {
                return (pass);
        }
if (req.http.user-agent ~ "GalaxyBot") {
                return (pass);
        }
if (req.http.user-agent ~ "genieBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Gigabot") {
                return (pass);
        }
if (req.http.user-agent ~ "Girafabot") {
                return (pass);
        }
if (req.http.user-agent ~ "Googlebot") {
                return (pass);
        }
if (req.http.user-agent ~ "Googlebot-Image") {
                return (pass);
        }
if (req.http.user-agent ~ "GurujiBot") {
                return (pass);
        }
if (req.http.user-agent ~ "HappyFunBot") {
                return (pass);
        }
if (req.http.user-agent ~ "hl_ftien_spider") {
                return (pass);
        }
if (req.http.user-agent ~ "Holmes") {
                return (pass);
        }
if (req.http.user-agent ~ "htdig") {
                return (pass);
        }
if (req.http.user-agent ~ "iaskspider") {
                return (pass);
        }
if (req.http.user-agent ~ "ia_archiver") {
                return (pass);
        }
if (req.http.user-agent ~ "iCCrawler") {
                return (pass);
        }
if (req.http.user-agent ~ "ichiro") {
                return (pass);
        }
if (req.http.user-agent ~ "igdeSpyder") {
                return (pass);
        }
if (req.http.user-agent ~ "IRLbot") {
                return (pass);
        }
if (req.http.user-agent ~ "IssueCrawler") {
                return (pass);
        }
if (req.http.user-agent ~ "Jaxified Bot") {
                return (pass);
        }
if (req.http.user-agent ~ "Jyxobot") {
                return (pass);
        }
if (req.http.user-agent ~ "KoepaBot") {
                return (pass);
        }
if (req.http.user-agent ~ "L.webis") {
                return (pass);
        }
if (req.http.user-agent ~ "LapozzBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Larbin") {
                return (pass);
        }
if (req.http.user-agent ~ "LDSpider") {
                return (pass);
        }
if (req.http.user-agent ~ "LexxeBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Linguee Bot") {
                return (pass);
        }
if (req.http.user-agent ~ "LinkWalker") {
                return (pass);
        }
if (req.http.user-agent ~ "lmspider") {
                return (pass);
        }
if (req.http.user-agent ~ "lwp-trivial") {
                return (pass);
        }
if (req.http.user-agent ~ "mabontland") {
                return (pass);
        }
if (req.http.user-agent ~ "magpie-crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "Mediapartners-Google") {
                return (pass);
        }
if (req.http.user-agent ~ "MJ12bot") {
                return (pass);
        }
if (req.http.user-agent ~ "MLBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Mnogosearch") {
                return (pass);
        }
if (req.http.user-agent ~ "mogimogi") {
                return (pass);
        }
if (req.http.user-agent ~ "MojeekBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Moreoverbot") {
                return (pass);
        }
if (req.http.user-agent ~ "Morning Paper") {
                return (pass);
        }
if (req.http.user-agent ~ "msnbot") {
                return (pass);
        }
if (req.http.user-agent ~ "MSRBot") {
                return (pass);
        }
if (req.http.user-agent ~ "MVAClient") {
                return (pass);
        }
if (req.http.user-agent ~ "mxbot") {
                return (pass);
        }
if (req.http.user-agent ~ "NetResearchServer") {
                return (pass);
        }
if (req.http.user-agent ~ "NetSeer Crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "NewsGator") {
                return (pass);
        }
if (req.http.user-agent ~ "NG-Search") {
                return (pass);
        }
if (req.http.user-agent ~ "nicebot") {
                return (pass);
        }
if (req.http.user-agent ~ "noxtrumbot") {
                return (pass);
        }
if (req.http.user-agent ~ "Nusearch Spider") {
                return (pass);
        }
if (req.http.user-agent ~ "NutchCVS") {
                return (pass);
        }
if (req.http.user-agent ~ "Nymesis") {
                return (pass);
        }
if (req.http.user-agent ~ "obot") {
                return (pass);
        }
if (req.http.user-agent ~ "oegp") {
                return (pass);
        }
if (req.http.user-agent ~ "omgilibot") {
                return (pass);
        }
if (req.http.user-agent ~ "OmniExplorer_Bot") {
                return (pass);
        }
if (req.http.user-agent ~ "OOZBOT") {
                return (pass);
        }
if (req.http.user-agent ~ "Orbiter") {
                return (pass);
        }
if (req.http.user-agent ~ "PageBitesHyperBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Peew") {
                return (pass);
        }
if (req.http.user-agent ~ "polybot") {
                return (pass);
        }
if (req.http.user-agent ~ "Pompos") {
                return (pass);
        }
if (req.http.user-agent ~ "PostPost") {
                return (pass);
        }
if (req.http.user-agent ~ "Psbot") {
                return (pass);
        }
if (req.http.user-agent ~ "PycURL") {
                return (pass);
        }
if (req.http.user-agent ~ "Qseero") {
                return (pass);
        }
if (req.http.user-agent ~ "Radian6") {
                return (pass);
        }
if (req.http.user-agent ~ "RAMPyBot") {
                return (pass);
        }
if (req.http.user-agent ~ "RufusBot") {
                return (pass);
        }
if (req.http.user-agent ~ "SandCrawler") {
                return (pass);
        }
if (req.http.user-agent ~ "SBIder") {
                return (pass);
        }
if (req.http.user-agent ~ "ScoutJet") {
                return (pass);
        }
if (req.http.user-agent ~ "Scrubby") {
                return (pass);
        }
if (req.http.user-agent ~ "SearchSight") {
                return (pass);
        }
if (req.http.user-agent ~ "Seekbot") {
                return (pass);
        }
if (req.http.user-agent ~ "semanticdiscovery") {
                return (pass);
        }
if (req.http.user-agent ~ "Sensis Web Crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "SEOChat::Bot") {
                return (pass);
        }
if (req.http.user-agent ~ "SeznamBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Shim-Crawler") {
                return (pass);
        }
if (req.http.user-agent ~ "ShopWiki") {
                return (pass);
        }
if (req.http.user-agent ~ "Shoula robot") {
                return (pass);
        }
if (req.http.user-agent ~ "silk") {
                return (pass);
        }
if (req.http.user-agent ~ "Sitebot") {
                return (pass);
        }
if (req.http.user-agent ~ "Snappy") {
                return (pass);
        }
if (req.http.user-agent ~ "sogou spider") {
                return (pass);
        }
if (req.http.user-agent ~ "Sosospider") {
                return (pass);
        }
if (req.http.user-agent ~ "Speedy Spider") {
                return (pass);
        }
if (req.http.user-agent ~ "Sqworm") {
                return (pass);
        }
if (req.http.user-agent ~ "StackRambler") {
                return (pass);
        }
if (req.http.user-agent ~ "suggybot") {
                return (pass);
        }
if (req.http.user-agent ~ "SurveyBot") {
                return (pass);
        }
if (req.http.user-agent ~ "SynooBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Teoma") {
                return (pass);
        }
if (req.http.user-agent ~ "TerrawizBot") {
                return (pass);
        }
if (req.http.user-agent ~ "TheSuBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Thumbnail.CZ robot") {
                return (pass);
        }
if (req.http.user-agent ~ "TinEye") {
                return (pass);
        }
if (req.http.user-agent ~ "truwoGPS") {
                return (pass);
        }
if (req.http.user-agent ~ "TurnitinBot") {
                return (pass);
        }
if (req.http.user-agent ~ "TweetedTimes Bot") {
                return (pass);
        }
if (req.http.user-agent ~ "TwengaBot") {
                return (pass);
        }
if (req.http.user-agent ~ "updated") {
                return (pass);
        }
if (req.http.user-agent ~ "Urlfilebot") {
                return (pass);
        }
if (req.http.user-agent ~ "Vagabondo") {
                return (pass);
        }
if (req.http.user-agent ~ "VoilaBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Vortex") {
                return (pass);
        }
if (req.http.user-agent ~ "voyager") {
                return (pass);
        }
if (req.http.user-agent ~ "VYU2") {
                return (pass);
        }
if (req.http.user-agent ~ "webcollage") {
                return (pass);
        }
if (req.http.user-agent ~ "Websquash.com") {
                return (pass);
        }
if (req.http.user-agent ~ "wf84") {
                return (pass);
        }
if (req.http.user-agent ~ "WoFindeIch Robot") {
                return (pass);
        }
if (req.http.user-agent ~ "WomlpeFactory") {
                return (pass);
        }
if (req.http.user-agent ~ "Xaldon_WebSpider") {
                return (pass);
        }
if (req.http.user-agent ~ "yacy") {
                return (pass);
        }
if (req.http.user-agent ~ "Yahoo! Slurp") {
                return (pass);
        }
if (req.http.user-agent ~ "Yahoo! Slurp China") {
                return (pass);
        }
if (req.http.user-agent ~ "YahooSeeker") {
                return (pass);
        }
if (req.http.user-agent ~ "YahooSeeker-Testing") {
                return (pass);
        }
if (req.http.user-agent ~ "YandexBot") {
                return (pass);
        }
if (req.http.user-agent ~ "YandexImages") {
                return (pass);
        }
if (req.http.user-agent ~ "YandexMetrika") {
                return (pass);
        }
if (req.http.user-agent ~ "Yasaklibot") {
                return (pass);
        }
if (req.http.user-agent ~ "Yeti") {
                return (pass);
        }
if (req.http.user-agent ~ "YodaoBot") {
                return (pass);
        }
if (req.http.user-agent ~ "yoogliFetchAgent") {
                return (pass);
        }
if (req.http.user-agent ~ "YoudaoBot") {
                return (pass);
        }
if (req.http.user-agent ~ "Zao") {
                return (pass);
        }
if (req.http.user-agent ~ "Zealbot") {
                return (pass);
        }
if (req.http.user-agent ~ "zspider") {
                return (pass);
        }
if (req.http.user-agent ~ "ZyBorg") {
                return (pass);
        }
}

protocol.vcl

sub vcl_recv {

#add to /usr/local/varnish/etc/varnish/security/rules/protocol.vcl

if (req.http.user-agent ~ "Googlebot") {
                # Force a cache miss
                #       set req.hash_always_miss = true;
                return (pass);
#       pass;
        }
}