Skip to content

Instantly share code, notes, and snippets.

@fitrh

fitrh/result_for_first_4page.json

Last active December 30, 2022 03:01
Show Gist options
  • Save fitrh/19ba013db9c485b285b3cab9af8c3226 to your computer and use it in GitHub Desktop.
Save fitrh/19ba013db9c485b285b3cab9af8c3226 to your computer and use it in GitHub Desktop.
Download ZIP
seniordiscountclub webscraping task
Raw
result_for_first_4page.json
{
"outlets": [
{
"name": "\"Don't Fret Pet!\"",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=37291",
"logo": "/chiip/GetLogo.ashx?business=19522&latest=0",
"address": "ALL AREAS SOUTH-EAST QUEENSLAND 4561",
"phone": "1300 307 021",
"web": "https://www.dontfretpet.com.au/",
"facebook": "https://www.facebook.com/dontfretpet",
"discount_category": "Boarding Kennels and Catteries",
"discount_detail": "10% Discount on dog boarding in a minder\u2019s home."
},
{
"name": "@ Beans",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=35589",
"logo": "",
"address": "59 Albany Creek Rd ASPLEY 4034",
"phone": "0431 788 312",
"web": "",
"facebook": "",
"discount_category": "Cafes",
"discount_detail": "10% discount on presentation of a seniors card."
},
{
"name": "1300 GET TINT",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=36626",
"logo": "/chiip/GetLogo.ashx?business=19039&latest=0",
"address": "U 2 20 Spine St SUMNER 4074",
"phone": "1300 438 846",
"web": "http://www.1300gettint.com.au",
"facebook": "https://www.facebook.com/1300gettint",
"discount_category": "Window Tinting",
"discount_detail": "We are pleased to offer 10% discount on all window tinting on homes, offices and vehicles."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=23224",
"logo": "",
"address": "Strathpine Square - 328 Gympie Rd STRATHPINE 4500",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=34590",
"logo": "",
"address": "Post Office Bldg - 155 Bourbong St BUNDABERG 4670",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=34588",
"logo": "",
"address": "Bundaberg Burnett Dental - 11 Barolin St BUNDABERG 4670",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=34589",
"logo": "",
"address": "Northshore Medical Cnt - 50 Northshore Blvd BURDELL 4818",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=34593",
"logo": "",
"address": "179 Nicklin Way WARANA 4575",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=31828",
"logo": "",
"address": "Chatswood Central Shp Cnt - cnr Chatswood/Magellan Rds SPRINGWOOD 4127",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27299",
"logo": "",
"address": "6 Barry St WEST GLADSTONE 4680",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27301",
"logo": "",
"address": "Cairns Central Shp Cnt CAIRNS 4870",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27303",
"logo": "",
"address": "Medical Court - Westfield Shp Cnt CARINDALE 4152",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27306",
"logo": "",
"address": "49 Fulham Rd PIMLICO 4812",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27307",
"logo": "",
"address": "34 Sydney St MACKAY 4740",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27308",
"logo": "",
"address": "100 High St ROCKHAMPTON 4701",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27310",
"logo": "",
"address": "Northtown - 1st Floor 280 Flinders St TOWNSVILLE 4810",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27313",
"logo": "",
"address": "Centro Woodlands Shp Cnt DEERAGUN 4818",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27309",
"logo": "",
"address": "Smithfield Shp Cnt SMITHFIELD 4878",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27312",
"logo": "",
"address": "1199 Riverway Dr RASMUSSEN 4815",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27305",
"logo": "",
"address": "Medilink Retail Cnt - 100 Angus Smith Dr DOUGLAS 4814",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27311",
"logo": "",
"address": "362B Bridge St, Wilsonton TOOWOOMBA 4350",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27300",
"logo": "",
"address": "89 Bundock St, Belgian Gardens TOWNSVILLE 4810",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "1300SMILES",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=27302",
"logo": "",
"address": "22 Minchinton St CALOUNDRA 4551",
"phone": "1300 764 537",
"web": "http://www.1300smiles.com.au",
"facebook": "",
"discount_category": "Dentists",
"discount_detail": "10% off the gap amount for general dental services."
},
{
"name": "19 Mosaics",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=37320",
"logo": "",
"address": " ",
"phone": "0413 468 062",
"web": "https://www.19mosaics.com.au/",
"facebook": "https://www.facebook.com/19Mosaics/",
"discount_category": "Art, Craft and Sewing",
"discount_detail": "10% discount for all products."
},
{
"name": "1st Choice Group (Aust) Pty Ltd (Elect Lic 35130)",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=28173",
"logo": "",
"address": "Mobile Service LOGAN 4123",
"phone": "1300 726 211",
"web": "",
"facebook": "",
"discount_category": "Electricians",
"discount_detail": "10% discount on materials and 10% discount on labour."
},
{
"name": "1st Choice Rentals ",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=32269",
"logo": "/chiip/GetLogo.ashx?business=16755&latest=0",
"address": "U 3 81 Bishop St KELVIN GROVE 4059",
"phone": "1300 552 558",
"web": "http://www.1stchoicerentals.com.au",
"facebook": "https://www.facebook.com/1stchoicerentals",
"discount_category": "Electrical Goods",
"discount_detail": "10% off all rental services. Appliances and mobility aids."
},
{
"name": "1st Choice Rentals ",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=32269",
"logo": "/chiip/GetLogo.ashx?business=16755&latest=0",
"address": "U 3 81 Bishop St KELVIN GROVE 4059",
"phone": "1300 552 558",
"web": "http://www.1stchoicerentals.com.au",
"facebook": "https://www.facebook.com/1stchoicerentals",
"discount_category": "Mobility Products and Scooters",
"discount_detail": "10% off all services, appliances and mobility aids."
},
{
"name": "1st Choice Rentals ",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=32269",
"logo": "/chiip/GetLogo.ashx?business=16755&latest=0",
"address": "U 3 81 Bishop St KELVIN GROVE 4059",
"phone": "1300 552 558",
"web": "http://www.1stchoicerentals.com.au",
"facebook": "https://www.facebook.com/1stchoicerentals",
"discount_category": "Electrical Goods",
"discount_detail": "10% off all rental services. Appliances and mobility aids."
},
{
"name": "1st Choice Rentals ",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=32269",
"logo": "/chiip/GetLogo.ashx?business=16755&latest=0",
"address": "U 3 81 Bishop St KELVIN GROVE 4059",
"phone": "1300 552 558",
"web": "http://www.1stchoicerentals.com.au",
"facebook": "https://www.facebook.com/1stchoicerentals",
"discount_category": "Mobility Products and Scooters",
"discount_detail": "10% off all services, appliances and mobility aids."
},
{
"name": "1st Solution Home Services",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=36094",
"logo": "/chiip/GetLogo.ashx?business=18699&latest=0",
"address": "Mobile Business 4207",
"phone": "0411 087 600",
"web": "http://www.1stsolution.com.au",
"facebook": "http://facebook.com/1stsolutionhomeservices",
"discount_category": "Cleaning Products and Services",
"discount_detail": "15% discount on carpet cleaning or pest control."
},
{
"name": "2 shots espresso",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=34255",
"logo": "",
"address": "Alexandra Hills Shp Cnt ALEXANDRA HILLS 4161",
"phone": "0431 370 454",
"web": "http://www.2shots.com.au",
"facebook": "",
"discount_category": "Cafes",
"discount_detail": "10% off full price."
},
{
"name": "23rd Avenue Chempro Chemist",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=32119",
"logo": "",
"address": "cnr Townson Ave/Mawarra St PALM BEACH 4221",
"phone": "(07) 5576 2866",
"web": "http://www.chempro.com.au",
"facebook": "",
"discount_category": "Pharmacies",
"discount_detail": "10% discount excluding prescriptions and specials."
},
{
"name": "360 Degree Wealth Services",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=36419",
"logo": "/chiip/GetLogo.ashx?business=18902&latest=0",
"address": "U 28 56 Sanders St UPPER MOUNT GRAVATT 4122",
"phone": "(07) 3340 3600",
"web": "http://www.360degreewealth.com.au",
"facebook": "https://www.facebook.com/360degreewealth/",
"discount_category": "Financial Services",
"discount_detail": "Free 1 hour initial consultation and 10% discount for the initial financial plan fee."
},
{
"name": "360 FINANCE",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=37270",
"logo": "",
"address": "64 Parramatta Rd UNDERWOOD 4119",
"phone": "(07) 3336 3360",
"web": "https://www.360finance.com.au",
"facebook": "https://www.facebook.com/360financeau",
"discount_category": "Insurance",
"discount_detail": "20% Off comprehensive insurance and $500 discount on application fee."
},
{
"name": "5150 Studios",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=33216",
"logo": "",
"address": "21 Rockwood Dr ORMEAU 4208",
"phone": "(07) 5547 5488",
"web": "http://www.5150Studios.com.au",
"facebook": "",
"discount_category": "Website Services",
"discount_detail": "Present card for 10% off website design portion."
},
{
"name": "66fit",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=36003",
"logo": "/chiip/GetLogo.ashx?business=18608&latest=0",
"address": "U 16 50-52 Kremzow Rd BRENDALE 4500",
"phone": "(07) 3889 9199",
"web": "http://www.66fit.com.au",
"facebook": "http://www.facebook.com/66fitau",
"discount_category": "Sports Stores",
"discount_detail": "10% off all retail prices - In store only."
},
{
"name": "7 Day Chemist",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=35933",
"logo": "",
"address": "150 Horton Pde MAROOCHYDORE 4558",
"phone": "(07) 5443 6033",
"web": "http://www.7daychemist.com.au",
"facebook": "",
"discount_category": "Naturopaths",
"discount_detail": "10% discount to all senior cardholders."
},
{
"name": "8th Avenue Physiotherapy Clinic",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=25795",
"logo": "",
"address": "Shop 6 Eighth Ave PALM BEACH 4221",
"phone": "(07) 5534 3347",
"web": "http://www.8thavephysio.com.au",
"facebook": "",
"discount_category": "Physiotherapists",
"discount_detail": "Initial consultation $70. Standard consultation $59."
},
{
"name": "A & B Fridge Repairs",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=3899",
"logo": "",
"address": "Mobile Service BRISBANE - SOUTHERN SUBURBS 4170",
"phone": "(07) 3399 5261",
"web": "",
"facebook": "",
"discount_category": "Refrigeration Sales and Services",
"discount_detail": "10% off service calls for guaranteed work in your home."
},
{
"name": "A & B Fridge Repairs",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=569",
"logo": "",
"address": "Mobile Service BRISBANE - NORTHERN SUBURBS 4170",
"phone": "(07) 3268 6807",
"web": "",
"facebook": "",
"discount_category": "Refrigeration Sales and Services",
"discount_detail": "10% off service calls for guaranteed work in your home."
},
{
"name": "A & B Fridge Repairs",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=33960",
"logo": "",
"address": "Mobile Service LOGAN & SURROUNDING AREAS 4114",
"phone": "(07) 3399 5261",
"web": "",
"facebook": "",
"discount_category": "Refrigeration Sales and Services",
"discount_detail": "10% off service calls for guaranteed work in your home."
},
{
"name": "A & B Fridge Repairs",
"profile_page": "https://secure.communities.qld.gov.au/chiip/businessSearch/SearchDetails.aspx?OutletID=33961",
"logo": "",
"address": "Mobile Service BEENLEIGH & SURROUNDING AREAS 4207",
"phone": "(07) 3399 5261",
"web": "",
"facebook": "",
"discount_category": "Refrigeration Sales and Services",
"discount_detail": "10% off service calls for guaranteed work in your home."
}
]
}
Raw
seniordiscountclub_webscraping_task.py
import json
from pprint import pp
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
def find_element(driver, locator):
return WebDriverWait(driver=driver, timeout=1000).until(
EC.presence_of_element_located(locator)
)
def html(text):
return BeautifulSoup(text, features="html.parser")
def write_to_json(new_data, filename="result.json"):
data = {"outlets": []}
try:
with open(filename, "r") as f:
data = json.load(f)
except IOError:
pass
with open(filename, "w") as f:
data["outlets"].append(new_data)
f.seek(0)
json.dump(data, f, indent=4)
BASE_URL = "https://secure.communities.qld.gov.au/chiip/businessSearch"
SEARCH_URL = f"{BASE_URL}/SearchDiscounts.aspx"
RESULT_URL = f"{BASE_URL}/SearchResults.aspx?action=srch"
driver = webdriver.Chrome()
driver.get(SEARCH_URL)
btn_find = find_element(driver, (By.ID, "btnRefresh"))
btn_find.click()
last_page = find_element(driver, (By.CSS_SELECTOR, "a[title='Last Page']"))
last_page_idx = int(last_page.get_attribute("href").split("=")[-1])
# for i in range(1, last_page_idx + 1):
for i in range(1, 5):
print(f"\n----------- Page: {i}")
if i > 1:
result_url = f"{RESULT_URL}&pageNum={i}"
driver.get(result_url)
search_list = find_element(driver, (By.ID, "ctl00_cph1_lstResults"))
search_list_html = html(search_list.get_attribute("innerHTML"))
search_entries = search_list_html.find_all("ol", class_="questions")
for j, entry in enumerate(search_entries):
print(f"\nEntry: {j}")
name_anchor = entry.find("span", class_="searchResult searchDetails").a
profile_page_url = f"{BASE_URL}/{name_anchor['href']}"
discount_info = entry.find("div", id="discountInfo")
outlet = {
"name": name_anchor.text,
"profile_page": profile_page_url,
"logo": discount_info.find("img", class_="logoSearchResults")[
"src"
],
}
profile_page = html(requests.get(profile_page_url).text)
for i in ["address", "phone"]:
div = profile_page.find("div", class_=f"outlet{i}")
outlet[i] = "" if div is None else div.span.text
for i in ["web", "facebook"]:
div = profile_page.find("div", class_=f"outlet{i}")
outlet[i] = "" if div is None else div.a.get("href", "")
discount_section = profile_page.find("li", id="section-discount")
discounts = discount_section.find_all("ol", class_="questions")
for discount in discounts:
category = discount.find("div", class_="discountcat")
outlet["discount_category"] = category.find(
"div", class_="searchResult searchDetails"
).a.text
outlet["discount_detail"] = (
category.find("div", class_="discountInfo").find("span").text
)
write_to_json(outlet)
pp(outlet)
driver.quit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment