Scrape OpnSense DHCP Leases Status Page and Export Results as List

scrape_opnsense_dhcp_leases.py

#!/usr/bin/env python3

#
# This python script provides a function to query the opnSense (v21.3 - v23.1.x) dhcp leases status page and 
# return a list of tuples including ip, hostname, and mac address. This script will not work with version 23.7.x+ 
# due to changes in the DHCP lease page.
# See comment from dheldt on modified version that works using api searchLeases page. 
# To use the original script, ensure LXML is installed via package manager or via pip. 
#
# 27-Mar-2021 - Original release
# 17-Jul-2022 - Fix url in scrape function. Add error trapping for case where user/pass are not set up correctly.
# 24-Sep-2023 - Add note about 23.7.x incompatibility.
#


import sys
import requests
from lxml import html
import re

url  = "http://192.168.1.1/status_dhcp_leases.php" #change url to match your opnSense machine address. Note http or https!
user = 'your_username'  #Username for opnSense login (default is 'root')
password = 'your_password' #Password for opnSense login

def scrape_opnsense_dhcp(url, user, password):
   ip = []
   mac = []
   hostname = []

   s = requests.session()
   r = s.get(url,verify = False)

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))

   payload = {
'login' : 'Login',
'usernamefld' : user,
'passwordfld' : password
}

   r = s.post(url,data=payload,verify = False,headers={"X-CSRFToken":csrf.group(1)})
   r = s.get(url,verify = False)
   tree = html.fromstring(r.content)
   tr_elements = tree.xpath('//tr')
   try:
      headers = [header.text for header in tr_elements[0]]
   except IndexError:
      print("Error retrieving lease list. Are you sure username and password were set up in script?")
      quit()

   ip.extend(tree.xpath('//table[@class="table table-striped"]//tbody//tr//td[' + str(headers.index('IP address') + 1) +']//text()'))

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('MAC address') + 1) +']'):
      if bool(re.search(r'([0-9a-f]{2}(?::[0-9a-f]{2}){5})', node.text)):
         mac.append(node.text)

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Hostname') + 1) +']'):
      if node.text is None:
         hostname.append('no_hostname')
      else:
         hostname.append(node.text)

   for i in range(len(mac)):
      mac[i] = mac[i].strip()

   return(list(zip(ip, mac, hostname)))

if __name__ == "__main__":     
    dhcp_list = scrape_opnsense_dhcp(url, user, password)

    for entry in dhcp_list:
            print(entry)

Yes, this is broken as of 23.7.x due to changes in the lease page. I haven't dug into trying to fix yet.

I wasn't aware of the searchlease endpoint but it does indeed seem like an opportunity to simplify. Will take a look when I get a little time in the next few days.

If you adept as follows, it does (currently) work:

mainUrl = "https://192.168.1.1/"
url  = "https://192.168.1.1/api/dhcpv4/leases/searchlease

   s = requests.session()
   r = s.get(mainUrl,verify = False)

   if not(r == 200):
   	print("failed to connect")

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))
   csrf_token = csrf.group(1)
   
   matchme = 'input type="hidden" name="([^"]*)" value="([^"]*)"'
   hidden_field = re.search(matchme,str(r.text))
   hidden_field_id = hidden_field.group(1)
   hidden_field_value = hidden_field.group(2)

   payload = {
'login' : '1',
'usernamefld' : user,
'passwordfld' : password
}

   payload[hidden_field_id] = hidden_field_value
   r = s.post(mainUrl,data=payload,verify = False,headers={"X-CSRFToken":csrf_token})
  
   if not(r == 200):
   	print("failed to log in")
   	
   r = s.get(url,verify = False)
   
   return r.json()

Note that you need to use two different urls now, because the log in does not work for the rest endpoint.

regards,

daniel

Thank you for working through this!

It turns out it is simpler to set up an api access key as described here:
https://docs.opnsense.org/development/how-tos/api.html.

The code to retrieve the leases can then simply be reduced to a single GET request.

import json
import requests

# define endpoint and credentials
api_key = '---------'   #use your unique key
api_secret = '-----------------'  #use your unique secret
url = 'http://opnsense.home/api/dhcpv4/leases/searchlease'
r = requests.get(url,
                 verify='OPNsense.pem',
                 auth=(api_key, api_secret))

if r.status_code == 200:
    return r.json()
else:
    print ('Connection / Authentication issue, response received:')