Fetch raw data for Gigaclear to see where their network is live

gigaclear_scrape.py

"""
Script to scrape Gigaclear status for each postcode (in this case in GL* areas) 
Writes a Json file with lat-lon, postcode and response for display and
later reference
"""

import requests
import csv
import io
import time

# Get all the GL post codes
# from this file
# postcode file https://www.freemaptools.com/download/full-postcodes/ukpostcodes.zip
postcode_file = 'ukpostcodes.csv'
with io.open(postcode_file) as fin:
    reader = csv.reader(fin)
    gl_postcodes = [line for line in reader if line[1].startswith('GL')


# Base URL
url = "https://www.gigaclear.com/wp-content/themes/gigaclear/gigameter.php?pc="

# GL6, GL7, GL8, GL11 are postcodes I am interested in
for a in ['6', 7', '8', '11']:
    for i, area in enumerate(gl_postcodes[a]):
        pc = area[1].replace(' ', '')
        resp = requests.get(url + pc)
        data = resp.json()
        gigaclear_status2[area[1]] = data
        time.sleep(1)
        if i % 10 == 0:
            print "Complete", i, area[1]


# Combine everything into some JSON
# {postcode: xxx, lat: 123, lon: 456, data: {response}}
gl_all_data = []
for a in ['6', '7', '8', '11']:
    for pc in gl_postcodes[a]:
        p = pc[1]
        g = gigaclear_status2[p]
        gl_all_data.append({'postcode': p, 'lat': pc[2], 'lon': pc[3], 'data': g})


# Write it all to a single file for the notebook
with io.open('/tmp/gl7811gigaclear.json', 'w') as fout:
    for line in gl_all_data:
        fout.write(unicode(json.dumps(line)))
        fout.write(u'\n')