Earthquakes

earthquake.py

#!/usr/bin/env python3

import requests
from bs4 import BeautifulSoup
from collections import namedtuple
from datetime import datetime
import csv


Earthquake = namedtuple('Earthquake', 'date, state, area, magnitude, lat, lon')

URL = 'http://sid.ipe.muni.cz/prev.php?lang=cs'


# data list
earthquakes = []


def load_data(from_file=None):

    # read data from opened file or download via requests
    if from_file:
        data = from_file.read()
    else:
        website = requests.get(URL)
        website.encoding = 'windows-1250'  # omg
        data = website.text

    # parse data
    for row in BeautifulSoup(data).find(id='list').find_all('tr')[1:]:
        date, time, states, area, magnitude, lat, lon = map(lambda cell: cell.text, row.find_all('td'))
        # print(date, time, state, area, magnitude, lat, lon)

        # preprocess data
        time = time.split()[0]
        date = datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M')
        magnitude = float(magnitude)
        lat, lon = map(lambda i: float(i[:-1]), (lat, lon))

        # add one record for each state (it can be: "state / second state")
        for state in map(lambda i: i.strip(), states.split('/')):
            earthquakes.append(Earthquake(date=date, state=state, area=area,
                                          magnitude=magnitude, lat=lat, lon=lon))


def save_csv(filename):
    with open(filename, 'w') as out:
        write_csv(out)


def write_csv(out):
    writer = csv.writer(out)
    for earthquake in earthquakes:
        writer.writerow([earthquake.date.strftime('%d.%m.%Y %H:%M'), earthquake.state, earthquake.area,
                         earthquake.magnitude, earthquake.lat, earthquake.lon])


def search_by_magnitude(lower_limit):
    return filter(lambda i: i.magnitude >= lower_limit, earthquakes)


def search_by_state(state):
    return filter(lambda i: i.state.lower() == state.lower(), earthquakes)


def search_by_date(year, month):
    return filter(lambda i: i.date.year == year and i.date.month == month, earthquakes)


def avaliable_states():
    return {e.state for e in earthquakes}


def get_graph_data(state):
    x_data = []
    y_data = []
    for e in search_by_state(state):
        x_data.append(e.date)
        y_data.append(e.magnitude)
    return x_data, y_data

earthquake_web_templates.py

#!/usr/bin/env python3

# custom backend
import earthquake

# matplotlib
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot

# web framework
from flask import Flask, request, make_response, render_template

# others
from io import StringIO, BytesIO
from base64 import b64encode
from datetime import datetime


app = Flask(__name__)


@app.route('/save_csv')
def save_csv():
    out = StringIO()
    earthquake.write_csv(out)

    # download instead of display
    response = make_response(out.getvalue())
    response.headers["Content-Disposition"] = "attachment; filename=earthquakes.csv"
    return response


@app.route('/search_by_state')
def search_by_state():
    state = request.args.get('state', 'Česká republika')
    search_data = earthquake.search_by_state(state)

    return render_template('search_by_state.html', avaliable_states=sorted(earthquake.avaliable_states()),
                           state=state, search_data=search_data)


@app.route('/search_by_date')
def search_by_date():
    now = datetime.now()
    try:
        year = int(request.args.get('year', now.year))
        month = int(request.args.get('month', now.month))
    except ValueError:
        year = now.year
        month = now.month
    search_data = earthquake.search_by_date(year, month)

    return render_template('search_by_date.html', year=year, month=month, search_data=search_data)


@app.route('/search_by_magnitude')
def search_by_magnitude():
    try:
        magnitude = float(request.args.get('magnitude', '0'))
    except ValueError:
        magnitude = 0
    search_data = earthquake.search_by_magnitude(magnitude)

    return render_template('search_by_magnitude.html', magnitude=magnitude, search_data=search_data)


@app.route('/show_graph')
def show_graph():
    state = request.args.get('state', 'Česká republika')

    # plot
    x_data, y_data = earthquake.get_graph_data(state)
    pyplot.rc('font', family='Arial')
    pyplot.title(state)
    pyplot.xlabel("datum")
    pyplot.ylabel("magnituda")
    pyplot.plot_date(x_data, y_data)

    # save to byte buffer and encode as base64
    out = BytesIO()
    pyplot.savefig(out, format='png')
    pyplot.close()
    plot_data = 'data:image/png;base64,' + b64encode(out.getvalue()).decode('utf8')

    return render_template('plot.html', avaliable_states=sorted(earthquake.avaliable_states()),
                           state=state, plot_data=plot_data)


@app.route('/')
def menu():
    return render_template('index.html')


if __name__ == '__main__':

    # load data
    with open("zemetreseni.html", encoding='windows-1250') as earthquake_data:
        earthquake.load_data(from_file=earthquake_data)

    # run webserver
    app.run(debug=True)

requirements.txt

matplotlib==1.4.3
beautifulsoup4==4.3.2
Flask==0.10.1
requests==2.4.3

templates_index.html

{% extends 'layout.html' %}
{% block content %}

<p class="lead">Vítejte, vyberte si prosím akci z menu.</p>

{% endblock %}

templates_layout.html

<!DOCTYPE html>
<html lang="cs">
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <title>Zemětřesení</title>

        <style>
            body {
                padding-top: 1em;
            }
            form {
                margin-bottom: 1em;
            }
        </style>

        <!-- Bootstrap CSS -->
        <link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
    </head>
    <body>

        <div class="container">

            <div class="navbar navbar-default">
                <a class="navbar-brand" href="#">Zemětřesení</a>
                <ul class="nav navbar-nav">
                    <li><a href="save_csv">stáhnout data v CSV</a></li>
                    <li><a href="search_by_magnitude">vyhledávání dle magnitudy</a></li>
                    <li><a href="search_by_state">vyhledávání dle státu</a></li>
                    <li><a href="search_by_date">vyhledávání dle data</a></li>
                    <li><a href="show_graph">zobrazit graf</a></li>
                    <li><a href="save_csv"></a></li>
                </ul>
            </div>

            {% block content %}{% endblock %}
        </div>

    </body>
</html>

templates_partial.data_table.html

<table class="table">
    <thead>
        <tr>
           <th>datum</th>
           <th>stát</th>
           <th>lokace</th>
           <th class="text-right">magnituda</th>
           <th class="text-right">lat</th>
           <th class="text-right">lon</th>
        </tr>
    </thead>
    <tbody>
        {% for earthquake in search_data %}
            <tr>
                <td>{{ earthquake.date.strftime('%d.%m.%Y %H:%M') }}</td>
                <td>{{ earthquake.state }}</td>
                <td>{{ earthquake.area }}</td>
                <td class="text-right">{{ earthquake.magnitude }}</td>
                <td class="text-right">{{ earthquake.lat }}</td>
                <td class="text-right">{{ earthquake.lon }}</td>
            </tr>
        {% endfor %}
    </tbody>
</table>

templates_partial.state_select.html

<form class="form-inline">
    <label>Stát:</label>
    <select name="state" class="form-control">
        {% for option in avaliable_states %}
            <option {{ 'selected' if option == state }}>{{ option }}</option>
        {% endfor %}
    </select>
    <input type="submit" value="Zvolit" class="btn btn-primary">
</form>

templates_plot.html

{% extends 'layout.html' %}
{% block content %}

{% include 'partial.state_select.html' %}

<div class="text-center">
    <img src="{{ plot_data }}">
</div>

{% endblock %}

templates_search_by_date.html

{% extends 'layout.html' %}
{% block content %}

<form class="form-inline">
    <div class="form-group">
        <label>Rok:</label>
        <input type="number" name="year" step="1" value="{{ year }}" class="form-control" required>
    </div>

    <div class="form-group">
        <label>Měsíc:</label>
        <input type="number" name="month" step="1" min="1" max="12" value="{{ month }}" class="form-control" required>
    </div>

    <input type="submit" value="Filtrovat" class="btn btn-primary">
</form>

{% include 'partial.data_table.html' %}

{% endblock %}

templates_search_by_magnitude.html

{% extends 'layout.html' %}
{% block content %}

<form class="form-inline">
    <label>Magnituda:</label>
    <input type="number" name="magnitude" min="0" step="any" value="{{ magnitude }}" class="form-control" required>
    <input type="submit" value="Filtrovat" class="btn btn-primary">
</form>

{% include 'partial.data_table.html' %}

{% endblock %}

templates_search_by_state.html

{% extends 'layout.html' %}
{% block content %}

{% include 'partial.state_select.html' %}

{% include 'partial.data_table.html' %}

{% endblock %}