Scraping US state population data from Wikipedia

pop_us_state_2024.csv

scrape_us_state_pop_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zbpNWGif3una"
   },
   "source": [
    "# Scraping Wikipedia U.S. States by Population Data\n",
    "This notebook simply pulls the [*U.S. States Population Data*](https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population) data down from *Wikipedia* and saves it to a *CSV* file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nLqshbIu4ZUx"
   },
   "source": [
    "## Setup\n",
    "First we need to the *scraping libraries* ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "950N0MZz1BXK"
   },
   "outputs": [],
   "source": [
    "# get necessary scraping tools\n",
    "!pip install beautifulsoup4 requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rXa1NZY54i88"
   },
   "source": [
    "## Scraping\n",
    "Now we can scrape the *Wikipedia* page and save the data into the *CSV* file `country_hdi.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MIqrdHP31JOQ"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "\n",
    "def remove_citations(text):\n",
    "    \"\"\"Removes in lingering Wikipedia citation syntax.\"\"\"\n",
    "    # regex for grabbing citations\n",
    "    citation_pattern = r'\\[\\w\\]|\\[\\d+\\]|\\[note\\s*\\d+\\]'\n",
    "    \n",
    "    # use re.sub() to remove citations from the text\n",
    "    text_without_citations = re.sub(citation_pattern, '', text)\n",
    "\n",
    "    # get cleaned text\n",
    "    return text_without_citations\n",
    "\n",
    "def sanitize_csv(text):\n",
    "    \"\"\"All necessary sanitization steps for storing in CSV.\"\"\"\n",
    "    # check cases\n",
    "    match text.strip():\n",
    "        case \"—\":\n",
    "            # check for NaN\n",
    "            sanitized = re.sub(r\"—\", \"NaN\", text)\n",
    "            \n",
    "        case _ if \"non-voting\" in text:\n",
    "            # remove non voting string\n",
    "            sanitized = re.sub(r\"\\([^()]+\\)\", \"\", text)\n",
    "            \n",
    "        case _:\n",
    "            # firt remove citations\n",
    "            cite_free_text = remove_citations(text)\n",
    "    \n",
    "            # now remove unwanted characters\n",
    "            sanitized = re.sub(r'[,\\n\\xa0%+]', '', cite_free_text)\n",
    "\n",
    "    # cleaned\n",
    "    return sanitized.strip()\n",
    "\n",
    "# URL of the Wikipedia page containing the table\n",
    "url = \"https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population\"\n",
    "\n",
    "# send a GET request to the URL\n",
    "response = requests.get(url)\n",
    "\n",
    "# parse the HTML content\n",
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "# find the table containing the data\n",
    "table = soup.find('table', class_='wikitable')\n",
    "\n",
    "# initialize a list to store the table data\n",
    "data = []\n",
    "\n",
    "# iterate over the rows of the table\n",
    "for row in table.find_all('tr'):\n",
    "    # initialize a list to store the data for each row\n",
    "    row_data = []\n",
    "    \n",
    "    # iterate over the cells (columns) in the row\n",
    "    for cell in row.find_all(['td', 'th']):\n",
    "        # append the cell content to the row data list\n",
    "        row_data.append(sanitize_csv(cell.get_text()))\n",
    "\n",
    "     # check if the row contains any non-empty cells\n",
    "    if any(row_data):            \n",
    "        # append the row data to the main data list\n",
    "        data.append(row_data)\n",
    "\n",
    "# define the path to save the CSV file\n",
    "csv_file = \"pop_us_state_2024.csv\"\n",
    "\n",
    "# write the data to a CSV file\n",
    "with open(csv_file, 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    writer.writerows(data)\n",
    "\n",
    "# notify when done\n",
    "print(f\"Data has been scraped and saved to: {csv_file}\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}