Dualis Grades Webscraper

diff.sh

#!/usr/bin/env bash
python3 dualis.py it11111@lehre.dhbw-stuttgart.de 'pa$$w0rd' > new || exit 1
if [ ! -f old ]; then
    cp new old
fi
diff new old
cp new old

dualis.py

#!/usr/bin/env python3
import requests
from lxml import html
import pandas as pd
import sys

user_agent = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
}

base = "https://dualis.dhbw.de"
results = "/scripts/mgrqcgi?APPNAME=CampusNet&PRGNAME=COURSERESULTS&ARGUMENTS=-N187631396865802,-N000307,"

def get_session_cookie():
    home = "/scripts/mgrqcgi?APPNAME=CampusNet&PRGNAME=EXTERNALPAGES&ARGUMENTS=-N000000000000001,-N000324,-Awelcome"
    r = requests.get(base + home)
    return r.cookies

def login(session_cookie, login_data):
    form = "/scripts/mgrqcgi"
    hidden_fields = {
      "APPNAME"   :"CampusNet",
      "PRGNAME"   :"LOGINCHECK",
      "ARGUMENTS" :"clino,usrname,pass,menuno,menu_type,browser,platform",
      "clino"     :"000000000000001",
      "menuno"    :"000324",
      "menu_type" :"classic",
      "browser"   :"",
      "platform"  :""
    }
    data = {**hidden_fields, **login_data, **user_agent}
    r = requests.post(base + form, data=data, cookies=session_cookie)
    refresh = r.headers["refresh"][7:]
    return refresh

def print_r(r):
    print(r.text)
    print("")
    print(r.status_code)
    print("")
    print(r.headers)
    print("")
    print(r.cookies)

def follow_redirects(session_cookie, refresh):
    r = requests.get(base + refresh, cookies=session_cookie)
    tree = html.fromstring(r.content)
    refresh = tree.xpath('//meta[@http-equiv="refresh"]')[0].get("content")[6:]
    r = requests.get(base + refresh, cookies=session_cookie)
    return r

def get_pruefungsergebnisse(session_cookie, home_page):
        tree = html.fromstring(home_page.content)
        pruefungsergebnisse = tree.cssselect("li#link000307 > a")[0].get("href")
        r = requests.get(base + pruefungsergebnisse, cookies=session_cookie)
        return r

def get_totals(r):
    tables = pd.read_html(r.content)
    endnoten = tables[0][["Name", "Endnote"]]
    return endnoten.to_string(index=False, header=False)


def get_individual_results(session_cookie, r):
    pruefungsergebnisse = html.fromstring(r.content)
    urls = ([pruefungsergebnisse.cssselect("a#Popup_details000{}".format(i))[0].get("href") for i in range(1, 9)])
    for url in urls:
        r = requests.get(base + url, cookies=session_cookie)

        tree = html.fromstring(r.content)
        title = tree.xpath('//h1/text()')[0]
        print(title)

        tables = pd.read_html(r.content)
        grades = tables[0][[1, 3]]
        grades = grades.dropna()
        print(grades.to_string(index=False, header=False))


def main():
    login_data = {
        "usrname": sys.argv[1],
        "pass": sys.argv[2]
    }

    # GET the main site to acquire a valid session cookie
    session_cookie = get_session_cookie()
    # POST the login request
    refresh = login(session_cookie, login_data)
    # follow all redirects to get to the welcome page
    home_page = follow_redirects(session_cookie, refresh)
    # GET the results overview
    r = get_pruefungsergebnisse(session_cookie, home_page)
    # print the total of each course
    print("Overview:")
    print(get_totals(r))
    print(get_individual_results(session_cookie, r))

if __name__ == "__main__":
    main()