Skip to content

Instantly share code, notes, and snippets.

@oktomus
Last active October 23, 2020 20:41
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save oktomus/bd88007e8cc8aee2aaab7ea364fb8300 to your computer and use it in GitHub Desktop.
Save oktomus/bd88007e8cc8aee2aaab7ea364fb8300 to your computer and use it in GitHub Desktop.
Tell me the difference
import requests
import urllib.request
import time
from bs4 import BeautifulSoup
import difflib
import os
import smtplib
import email.message
import io
from requests_html import HTMLSession
import sys
things_to_test = [
#{
#'name': 'clock',
#'url': 'https://free-onlineclock.com/online-clock',
#'div': 'vue_clock'
#},
]
local_mail = 'localhost@localhost'
script_folder = os.path.dirname(os.path.realpath(__file__))
def send_self_local_mail(subject, body):
sender_mail = local_mail
receivers_mail = [local_mail]
m = email.message.Message()
m['From'] = local_mail
m['To'] = local_mail
m['Subject'] = subject
m.set_payload(body);
m = m.as_string()
try:
smtpObj = smtplib.SMTP('localhost')
smtpObj.sendmail(sender_mail, receivers_mail, m)
print("Successfully sent email")
except Exception as e:
print("Error: unable to send email")
print(str(e))
_, _, stack = sys.exc_info()
print(str(stack))
def tell_me_the_difference(url, div_id, name):
print('Tell me the difference\nname={}\nurl={}\nid={}'.format(name, url, div_id))
session = HTMLSession()
try:
r = session.get(url)
try:
r.raise_for_status()
except ValueError as e:
raise('Dead link')
r.html.render(sleep = 2, timeout = 20) # this call executes the js in the page
#soup = BeautifulSoup(response.text, 'html.parser')
details = r.html.find('#{}'.format(div_id))[0].text.replace('\t', ' ').splitlines()
#details = str(soup.findAll(id=div_id)[0]).replace('\t', ' ').splitlines()
file_path = os.path.join(script_folder, name)
# Write it in a file the first time.
if os.path.exists(file_path):
previous_details = None
with io.open(file_path, 'r', encoding='utf-8') as file:
previous_details = file.read().replace('\t', ' ').splitlines()
diff = difflib.ndiff(details, previous_details)
changes = [l for l in diff if l.startswith('+ ') or l.startswith('- ')]
has_changes = len(changes) > 0
pretty_changes = '\n'.join(changes)
print('DIFF?: ', has_changes)
if has_changes:
send_self_local_mail('{} diff'.format(name), '{}\n\n{}'.format(url, pretty_changes))
with io.open(file_path, 'w', encoding='utf-8') as file:
file.write('\n'.join(details))
except Exception as e:
print('FAILURE')
print(str(e))
_, _, stack = sys.exc_info()
send_self_local_mail('{} failed'.format(name), '{}\n\n{}\n\n{}'.format(url, str(e), stack))
if __name__ == '__main__':
for thing in things_to_test:
url = thing['url']
div_id = thing['div']
name = thing['name']
tell_me_the_difference(url, div_id, name)
time.sleep(10)
sys.exit(0)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment