Skip to content

Instantly share code, notes, and snippets.

@Hokyjack

Hokyjack/Competition Watchdog.ipynb

Created July 12, 2020 22:18
Show Gist options
  • Save Hokyjack/9ec840d8a6067e09ea1d7970f236a41d to your computer and use it in GitHub Desktop.
Save Hokyjack/9ec840d8a6067e09ea1d7970f236a41d to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Competition Watchdog.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import telegram\n",
"from telegram import ParseMode\n",
"\n",
"import numpy as np\n",
"from bs4 import BeautifulSoup\n",
"import pandas as pd\n",
"import requests\n",
"import re\n",
"from datetime import timedelta\n",
"from datetime import datetime\n",
"\n",
"import time"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def get_data_to_pd():\n",
" \n",
" try:\n",
" r = requests.get(\"https://www.dancerank.cz/plans?domain=csts\")\n",
" soup = BeautifulSoup(r.text)\n",
"\n",
" event_list = [l.find_all(\"li\", recursive=False) for l in soup.find_all(\"ul\", class_=\"event-list\")]\n",
" event_list = [item for sublist in event_list for item in sublist]\n",
" except:\n",
" bot.send_message(chat_id=chat_id, text=\"Error at server!\\nWaiting 1 hour...\")\n",
" return None\n",
"\n",
" events = []\n",
"\n",
" for event in event_list:\n",
" name = event.find(\"div\", class_=\"event-info\").text.split(\"\\n\")[0]\n",
" day, date = event.find(\"div\", class_=\"event-info\").text.split(\"\\n\")[1].split()\n",
" date_obj = datetime.strptime(date, '%d.%m.%Y').date()\n",
" comps = event.find(\"ul\", class_=\"comp-list\").find_all(\"li\", recursive=False)\n",
"\n",
" for comp in comps:\n",
" url = f\"https://www.dancerank.cz{comp.find('a')['href']}\"\n",
" if \"Dospělí-E-LAT\" in comp.text:\n",
" pairs, time = re.findall(\"\\((.*?)\\)\", comp.text)\n",
" cat = \"Dospělí-D-LAT\"\n",
" c = {\"comp\": name, \"day\": day, \"date\": date_obj, \"category\": cat, \"time\": time, \"pairs\": pairs, \"url\": url}\n",
" events.append(c)\n",
" if \"Dospělí-D-STT\" in comp.text:\n",
" pairs, time = re.findall(\"\\((.*?)\\)\", comp.text)\n",
" cat = \"Dospělí-D-STT\"\n",
" c = {\"comp\": name, \"day\": day, \"date\": date_obj, \"category\": cat, \"time\": time, \"pairs\": pairs, \"url\": url}\n",
" events.append(c)\n",
"\n",
" pd_events = pd.DataFrame(events)\n",
" pd_events = pd_events[[\"comp\", \"day\", \"date\", \"category\", \"time\", \"pairs\", \"url\"]]\n",
" pd_events['date'] = pd.to_datetime(pd_events['date'])\n",
" \n",
" \n",
" now_date = datetime.now().date().strftime('%Y-%m-%d')\n",
" end = datetime.now().date() + timedelta(days=7)\n",
" end_date = end.strftime('%Y-%m-%d')\n",
" \n",
" mask = (pd_events['date'] >= now_date) & (pd_events['date'] <= end_date)\n",
" df = pd_events.loc[mask]\n",
" df\n",
" \n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<telegram.message.Message at 0x18873449940>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"bot = telegram.Bot(token='<YOUR_SECRET_TOKEN>')\n",
"chat_id = bot.get_updates()[-1].message.chat_id\n",
"\n",
"bot.send_message(chat_id=chat_id, text=\"Competition watchdog initialized!\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"original_table = get_data_to_pd()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>comp</th>\n",
" <th>day</th>\n",
" <th>date</th>\n",
" <th>category</th>\n",
" <th>time</th>\n",
" <th>pairs</th>\n",
" <th>url</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Vánoční cena Fortuny 2019 s GALAVEČEREM - Kong...</td>\n",
" <td>sobota</td>\n",
" <td>2019-12-07</td>\n",
" <td>Dospělí-D-LAT</td>\n",
" <td>16:00</td>\n",
" <td>6</td>\n",
" <td>https://www.dancerank.cz/plan/5912/comps#comp-5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Vánoční cena Fortuny 2019 s GALAVEČEREM - Kong...</td>\n",
" <td>sobota</td>\n",
" <td>2019-12-07</td>\n",
" <td>Dospělí-D-STT</td>\n",
" <td>14:30</td>\n",
" <td>2</td>\n",
" <td>https://www.dancerank.cz/plan/5912/comps#comp-18</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>COOL DANCE - Vánoční cena města Bakov nad Jize...</td>\n",
" <td>neděle</td>\n",
" <td>2019-12-08</td>\n",
" <td>Dospělí-D-LAT</td>\n",
" <td>17:30</td>\n",
" <td>2</td>\n",
" <td>https://www.dancerank.cz/plan/5731/comps#comp-6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>COOL DANCE - Vánoční cena města Bakov nad Jize...</td>\n",
" <td>neděle</td>\n",
" <td>2019-12-08</td>\n",
" <td>Dospělí-D-STT</td>\n",
" <td>13:30</td>\n",
" <td>3</td>\n",
" <td>https://www.dancerank.cz/plan/5731/comps#comp-17</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" comp day date \\\n",
"0 Vánoční cena Fortuny 2019 s GALAVEČEREM - Kong... sobota 2019-12-07 \n",
"1 Vánoční cena Fortuny 2019 s GALAVEČEREM - Kong... sobota 2019-12-07 \n",
"2 COOL DANCE - Vánoční cena města Bakov nad Jize... neděle 2019-12-08 \n",
"3 COOL DANCE - Vánoční cena města Bakov nad Jize... neděle 2019-12-08 \n",
"\n",
" category time pairs \\\n",
"0 Dospělí-D-LAT 16:00 6 \n",
"1 Dospělí-D-STT 14:30 2 \n",
"2 Dospělí-D-LAT 17:30 2 \n",
"3 Dospělí-D-STT 13:30 3 \n",
"\n",
" url \n",
"0 https://www.dancerank.cz/plan/5912/comps#comp-5 \n",
"1 https://www.dancerank.cz/plan/5912/comps#comp-18 \n",
"2 https://www.dancerank.cz/plan/5731/comps#comp-6 \n",
"3 https://www.dancerank.cz/plan/5731/comps#comp-17 "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"original_table"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"original_table = get_data_to_pd()\n",
"# original_table[\"pairs\"][0] = 8\n",
"# original_table[\"comp\"][3] = 'Danza LIFE! Cup 2018'\n",
"\n",
"while True:\n",
" \n",
" new_table = get_data_to_pd()\n",
" while new_table is None:\n",
" time.sleep(3600)\n",
" new_table = get_data_to_pd()\n",
" \n",
" if len(new_table) == len(original_table) and np.all(new_table[\"comp\"] == original_table[\"comp\"]):\n",
" \n",
" for i, row in new_table.iterrows():\n",
" new_pairs = new_table[\"pairs\"].iloc[i]\n",
" old_pairs = original_table[\"pairs\"].iloc[i]\n",
"\n",
" if old_pairs != new_pairs:\n",
" message = f\"<strong>Number of pairs changed from {old_pairs} to {new_pairs}!</strong>\\n\"\n",
" message += f'{row[\"comp\"]}\\n{row[\"day\"]} {row[\"date\"]}\\n{row[\"category\"]} {row[\"time\"]}\\nPairs: {new_pairs}\\n'\n",
" url = row[\"url\"]\n",
" message += f\"<a href='{url}'>See all pairs</a>\"\n",
" bot.send_message(chat_id=chat_id, text=message, parse_mode=ParseMode.HTML)\n",
" \n",
" else:\n",
" message = \"<strong>Competition List updated!</strong>\\n\"\n",
" message += \"\\n\".join([c for c in new_table[\"comp\"]])\n",
" bot.send_message(chat_id=chat_id, text=message, parse_mode=ParseMode.HTML)\n",
" \n",
" original_table = new_table\n",
" time.sleep(60)\n",
" #break"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment