Instantly share code, notes, and snippets.

Embed
What would you like to do?
Python module for downloading planetary data for More or Less
# -*- coding: utf-8 -*-
"""
This is the Python module I used for scraping planetary data for the BBC radio
programme More or Less. You can read more about it here:
http://olihawkins.com/2019/02/1
"""
# Imports --------------------------------------------------------------------
import datetime
import json
import os
import pandas
import requests
import time
from matplotlib import pyplot as plt
from matplotlib import rcParams
plt.style.use('bmh')
rcParams['font.family'] = 'sans-serif'
rcParams['font.sans-serif'] = ['Helvetica Neue']
# Constants ------------------------------------------------------------------
URL = 'https://keisan.casio.com/exec/system/1224746378'
HEADERS = {'content-type': 'application/x-www-form-url-encoded'}
LONGITUDE = 0.0004
LATITUDE = 51.4777
HOUR = 12
TIMEZONE = 0
VOYAGER_DISTANCE = 142.0
DATA_DIR = 'data'
SLEEP_TIME = 1
COLS = [
'Date',
'Mercury',
'Venus',
'Mars',
'Jupiter',
'Saturn',
'Uranus',
'Neptune',
'Pluto',
'Nearest']
# Functions ------------------------------------------------------------------
def request(date):
"""
Send an http POST to the server requesting data for a given date. The
long, lat, time and timezones are all constant. The return value is the raw
html returned.
"""
data = {
'var_λo': LONGITUDE,
'var_φ': LATITUDE,
'var_M': date.month,
'var_D': date.day,
'var_Y': date.year,
'var_H': HOUR,
'var_I': TIMEZONE
}
response = requests.post(URL, headers=HEADERS, data=data)
if response.status_code != 200:
raise Exception(response.status_code, response.text)
return response.text
def parse(date, text):
"""
Parse the text of the response and return a dictionary of each planet's
distance from the Earth, and the nearest planet to the earth, on the given
date.
"""
start = text.find('exedata') + 10
text = text[start:]
end = text.find('addTableClass') - 2
text = text[:end]
planets_list = json.loads(text)[1:]
planets = {'Date': date.isoformat()}
nearest_distance = VOYAGER_DISTANCE
nearest_planet = None
for planet in planets_list:
name = planet[0]
distance = float(planet[3])
if distance < nearest_distance:
nearest_distance = distance
nearest_planet = name
planets[name] = distance
planets['Nearest'] = nearest_planet
return planets
def fetch_date(date):
"""Fetch the planet data for a given date."""
text = request(date)
planets = parse(date, text)
return planets
def fetch_years(years):
"""
Get the data for each day in each of the given years and save it to a
csv. The function returns the dataframe.
"""
start = years[0]
rows = []
for year in years:
dates = get_dates(year)
for date in dates:
print('Getting data for {0} ...'.format(date.isoformat()))
rows.append(fetch_date(date))
time.sleep(SLEEP_TIME)
df = pandas.DataFrame.from_dict(rows)
df = df[COLS]
filename = os.path.join(DATA_DIR, '{0}-{1}.csv'.format(start, year))
df.to_csv(filename, index=False)
return df
def get_dates(year):
"""Get a list of dates for each day in the given calendar year."""
start = datetime.datetime(year, 1, 1)
end = datetime.datetime(year + 1, 1, 1)
delta = datetime.timedelta(1)
days = (end - start).days
dates = []
dt = start
for d in range(days):
dates.append(dt.date())
dt = dt + delta
return dates
def plot_planets(data):
"""Plot the data."""
date = data['Date']
mercury = data['Mercury']
venus = data['Venus']
mars = data['Mars']
plt.figure(figsize=(12, 4))
plt.plot(
date,
mercury,
lw=1.2,
ls='-',
color='#a19ba5',
label='Mercury')
plt.plot(
date,
venus,
lw=1.2,
ls='-',
color='#f44198',
label='Venus')
plt.plot(
date,
mars,
lw=1.2,
ls='-',
color='#0090ff',
label='Mars')
plt.xticks(
[0, 3653, 7305, 10958, 14610, 18263],
['1968', '1978', '1988', '1998', '2008', '2018'])
plt.ylim([0, 3])
plt.legend(loc='upper right', facecolor='#ffffff', fontsize=11)
plt.title('Distance of Mercury, Venus and Mars from Earth in Astronomical Units',
fontsize=13, fontweight='semibold');
plt.tight_layout()
ax = plt.gca()
ttl = ax.title
ttl.set_position([0.5, 1.0])
plt.savefig(os.path.join('plots', 'planets.svg'), format='svg')
plt.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment