MGS_scraper.py

from bs4 import BeautifulSoup
import csv
from datetime import datetime
import html
import requests
import sqlite3
from time import time, sleep

# README
# pip install beautifulsoup4
# python MGS_scraper.py

ips = ["10.3.0.78", "10.3.0.121"]

def scrape_data():
    try:
        con = sqlite3.connect('MGS.db')
        for ip in ips:
            try: 
                timestamp = datetime.now()
                url = f'http://{ip}'
                req = requests.get(url)
                raw_html = html.unescape(req.text)
                soup = BeautifulSoup(raw_html, 'html.parser')
                forms = soup.find("form")
                form_data = []
                labels = {}
                label_number = 1
                for label in forms.find_all("p"):
                    if label.text:
                        labels[f'{label_number}'] = label.text
                        label_number += 1

                for input_tag in forms.find_all("input"):
                    input_name = input_tag.attrs.get("name")
                    input_value =input_tag.attrs.get("value", "")
                    form_data.append({"timestamp": timestamp, "name": input_name, "value": input_value})
                keys = form_data[0].keys()
                save_to_db(con, form_data, labels, ip)
            except:
                print(f'An error occurred fetching ip: {ip}')
    except:
        print("An exception occurred in scrate_data()")
    finally:
        con.close()

def create_db():
    try:
        con = sqlite3.connect('MGS.db')
        cur = con.cursor()
        cur.execute("CREATE TABLE IF NOT EXISTS mgs (timestamp text, location text, name text, value text)")
        con.close()
    except:
        print("An exception occurred in create_db")

def save_to_db(con, form_data, labels, ip):
    try:
        cur = con.cursor()
        for row in form_data:
            timestamp = row["timestamp"]
            name = labels[row["name"]]
            value = row["value"]
            cur.execute(f"INSERT INTO mgs VALUES ('{timestamp}', '{ip}', '{name}','{value}')")
        con.commit()

        ###
        # debug statements
        cur.execute("SELECT COUNT(*) FROM mgs;")
        count = cur.fetchone()[0]
        print(f'Total rows: ', count)
        ###
    except:
        print("An exception occurred in save_to_db")

def run():
    frequency_in_seconds = 2
    print(f'Waiting {frequency_in_seconds} seconds')
    try:
        while True:
            sleep(frequency_in_seconds)
            scrape_data()
    except:
        print("An exception occurred in run()")

create_db()
run()