Created
December 26, 2019 04:46
-
-
Save bfeldman89/ec25026fa0efef7baaad84f82fc9fba6 to your computer and use it in GitHub Desktop.
hourly script to automate my scripts of pythonanywhere.com
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# !/usr/bin/env python3 | |
""" | |
This module runs all of my scrapers that don't require the pdfkit module bc pythonanywhere doesn't support it. | |
It is scheduled to run every hour at 20min past the hour. The idea is that my locally run cron job that runs at | |
5min past the hour shouldn't take more than 15min. The pythonanywhere scheduled task is as follows: | |
20 * * * * source ~/.bashrc && cd ~/code && python3 pa_sched.py && python3 jail_scrapers/scrapers.py | |
""" | |
import os | |
import time | |
def main(): | |
""" | |
Every even hour, clean the jail data. | |
Every odd hour, scrape SOS & MDOC. | |
Every hour, 9am - 6pm, send tweet if one's on deck. | |
At 7am & 7pm, check WaPo data & wrangle feedly/Safari bookmarks | |
""" | |
this_hour = int(time.strftime("%H")) | |
if (this_hour % 2) == 0: | |
os.chdir(f"/{os.getenv('HOME')}/code/jail_scrapers") | |
from jail_scrapers import polish_data | |
polish_data.main() | |
print('polish_data.main() π') | |
else: | |
os.chdir(f"/{os.getenv('HOME')}/code/mdoc_scraper") | |
from mdoc_scraper import mdoc_scraper | |
mdoc_scraper.main() | |
print('mdoc_scraper.main() π') | |
from sos_scraper import sos_scraper | |
sos_scraper.main() | |
print('sos_scraper.main() π') | |
if this_hour in {7, 19}: | |
from reading_list import muh_news | |
muh_news.main() | |
print('muh_news.main() π') | |
from police_shootings import police_shootings | |
police_shootings.main() | |
print('police_shootings.main() π') | |
else: | |
pass | |
if 9 <= this_hour <= 18: | |
from scheduled_tweets import scheduled_tweets | |
scheduled_tweets.main() | |
print('scheduled_tweets π') | |
else: | |
print('not my tweeting hours') | |
print("πΎ Cheers! π₯π₯π₯") | |
if __name__ == "__main__": | |
main() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment