Codesphere web scraping with Python example

weather_scraping_demo.py

#------- Importing required libraries --------
import pandas as pd
import requests
from bs4 import BeautifulSoup

#------- URL of the WEBPAGE you want to SCRAPE --------------
url = "https://forecast.weather.gov/MapClick.php?lat=41.8843&lon=-87.6324#.YbH0ptBBxPY"

#------- Sending an HTTP request ----------
response = requests.get(url)

#------- Parsing the HTML ----------
soup = BeautifulSoup(response.content, "html.parser")

#------- Scraping the weekly forecast ----------
weekly_forecast = soup.find(id = "seven-day-forecast-container")

#------- Start of Periodic forecast extraction from weekly forecast------

periodic_forecast = weekly_forecast.find_all(class_ = "period-name")

forecast_period_list = []

for period in periodic_forecast:
    forecast_period_list.append(period.text.strip())


#-----------------------------------------------------------------------

#------- Start of temperature forecast extraction from weekly forecast------

forecast_temperatures = weekly_forecast.find_all(class_ = "temp")

forecast_temperature_list = []

for temp in forecast_temperatures:
    forecast_temperature_list.append(temp.text)

#-----------------------------------------------------------------------

#------- Start of forecast description extraction from weekly forecast------

forecast_description = weekly_forecast.find_all(class_ = "short-desc")

forecast_description_list = []

for desc in forecast_description:
    forecast_description_list.append(desc.text.strip())

#-----------------------------------------------------------------------

#---------- Saving the extracted data to a dataframe----------------------

weather_data = pd.DataFrame(
    {
        "Forecast Period": forecast_period_list,
        "Forecast Temperature": forecast_temperature_list,
        "Forecast Description": forecast_description_list,
    }
)

#---------- titles for different columns--------------------------------
column_titles = ["Forecast Period", "Forecast Temperature", "Forecast Description" ]

#----------- Converting the saved weather data to csv file---------------
weather_data.to_csv("weather_data.csv", columns = column_titles, encoding='utf-16be', index = False)