thayton

Keybase proof

I hereby claim:

• I am thayton on github.
• I am thayton (https://keybase.io/thayton) on keybase.
• I have a public key ASDwUwmXo5-1N9a9CTNzTATM_EndqNJMp3O_hzTBhaLOmQo

To claim this, I am signing this object:

thayton

#!/usr/bin/env python

import time
import json
import requests

class Scraper(object):
    def __init__(self):
        self.url = 'http://preowned.ferrari.com/vdata'
        self.session = requests.Session()

thayton

#!/usr/bin/env python

import json
import requests

from bs4 import BeautifulSoup

class L3ComScraper(object):
    def __init__(self):
        self.session = requests.Session()

thayton

/*
 * Companion code for article at http://toddhayton.com/...
 *
 * Setup:
 *   $ mkdir scraper/
 *   $ cd scraper/
 *   $ npm init -y
 *   $ npm install puppeteer --save
 *   $ npm install node-fetch --save
 *   $ npm install progress-stream --save

thayton

node_modules/

thayton

import json
import requests

class LennarScraper(object):
    def __init__(self):
        self.url = 'https://www.lennar.com/Services/Rest/SearchMethods.svc/GetInventoryTabDetails'
        self.session = requests.Session()
        self.data = {
            "CommunityID":"4531",
            "pageState":{

thayton

import re
import json
import urlparse
import requests

from bs4 import BeautifulSoup

class SheaHomesScraper(object):
    def __init__(self):
        self.url = 'https://www.sheahomes.com/new-homes/colorado/denver-area/parker/stonewalk-at-stepping-stone/'

thayton

/*                                                                                                                                                                                   
 * $ npm init -y                                                                                                                                                                     
 * $ npm install puppeteer --save                                                                                                                                                    
 * $ node sheahomes.js                                                                                                                                                               
 */
const puppeteer = require('puppeteer');
const url = 'https://www.sheahomes.com/new-homes/colorado/denver-area/parker/stonewalk-at-stepping-stone/';

async function getLot(page, url) {
    await page.goto(url);

thayton

#!/usr/bin/env python

#######################################################################
# Setup:
#   python3 -m venv venv
#   source venv/bin/activate
#   pip install lxml requests beautifulsoup4
#
# Usage:
#   python eygbl.py

thayton

/*
 * Companion code for article at http://toddhayton.com/2018/08/01/scraping-with-puppeteer/
 *
 * Setup:
 *   $ mkdir scraper/
 *   $ cd scraper/
 *   $ npm init -y
 *   $ npm install puppeteer --save
 * 
 * Usage: