towerscraper.py

import requests 
from bs4 import BeautifulSoup
import pandas as pd
import geopandas as gpd
import numpy as np

class TowerScraper:
    """ 
    Initialize scraper object for a given ULS licKey.
    """
    def __init__(self, key):
        self.base_url = "https://wireless2.fcc.gov/UlsApp/UlsSearch/licenseLocSum.jsp?"
        self.key = key
        self.url = self.base_url + "licKey=" + self.key
        self.towers = self.count_towers()
        
        
    """
    Count # of FCC registered towers. 
    """
    def count_towers(self):
        r = requests.get(self.url)
        soup = BeautifulSoup(r.text)
        infoboxes = soup.find_all('td', { 'class' : "cell-pri-medium"} )
        possibles = []
        
        for box in infoboxes:
            try: 
                possibles.append(box.find('b'))
            except:
                pass
            
        possibles = list(filter(None, possibles))
        towers = int(possibles[0].text)
        return towers
    
    
    """
    Returns a DataFrame with tower name, axis, and normalized geographic data.
    """
    def coords(self):
        dfs = []
        for i in range(int(self.towers / 10)):
            print("Grabbing page ", i+1)
            if i == 0:
                frames = pd.read_html(self.url)
                dfs.append(frames[10])
            else:
                frames = pd.read_html(self.url + "&pageNumToReturn=" + str(i + 1))
                dfs.append(frames[10])
        
        # Cleans the df and resets index
        def clean_df(df):
            df = df.dropna()
            df = df.drop(columns=[0])
            df = df.reset_index(drop=True)
            return df
        
        df = pd.concat([clean_df(df) for df in dfs])
        
        df['latlon'] = df[2]
        df['latlon'] = pd.Series([i.split(',') for i in df['latlon']])
        df['LON'] = pd.Series([i[0] for i in df['latlon']])
        df['LAT'] = pd.Series([i[1] for i in df['latlon']])
        
        df = df.drop(columns=[2, 'latlon'])
        
        columns = {1 : 'Name', 3 : 'Axis'}
        
        df = df.rename(columns=columns)
        
        def reformat_lat(latitude):
            latitude = latitude.strip()
            N = 'N' in latitude
            d, m, s = map(float, latitude[:-1].split('-'))
            value = (d + m / 60. + s / 3600.) * (1 if N else -1)
            return value

        def reformat_lon(longitude):
            longitude = longitude.strip()
            W = 'W' in longitude
            d, m, s = map(float, longitude[:-1].split('-'))
            value = (d + m / 60. + s / 3600.) * (-1 if W else 1)
            return value
        
        df['LON'] = df['LON'].apply(reformat_lon)
        df['LAT'] = df['LAT'].apply(reformat_lat)
        
        df.columns = ['Name', 'Axis', 'Lat', 'Lon']
        
        return df.reset_index(drop=True)