Skip to content

Instantly share code, notes, and snippets.

@pletch
Last active September 24, 2023 13:16
Show Gist options
  • Save pletch/c9b208c1bbc1bd33de19d0ad03f96f82 to your computer and use it in GitHub Desktop.
Save pletch/c9b208c1bbc1bd33de19d0ad03f96f82 to your computer and use it in GitHub Desktop.
Scrape OpnSense DHCP Leases Status Page and Export Results as List
#!/usr/bin/env python3
#
# This python script provides a function to query the opnSense (v21.3 - v23.1.x) dhcp leases status page and
# return a list of tuples including ip, hostname, and mac address. This script will not work with version 23.7.x+
# due to changes in the DHCP lease page.
# See comment from dheldt on modified version that works using api searchLeases page.
# To use the original script, ensure LXML is installed via package manager or via pip.
#
# 27-Mar-2021 - Original release
# 17-Jul-2022 - Fix url in scrape function. Add error trapping for case where user/pass are not set up correctly.
# 24-Sep-2023 - Add note about 23.7.x incompatibility.
#
import sys
import requests
from lxml import html
import re
url = "http://192.168.1.1/status_dhcp_leases.php" #change url to match your opnSense machine address. Note http or https!
user = 'your_username' #Username for opnSense login (default is 'root')
password = 'your_password' #Password for opnSense login
def scrape_opnsense_dhcp(url, user, password):
ip = []
mac = []
hostname = []
s = requests.session()
r = s.get(url,verify = False)
matchme = '"X-CSRFToken", "(.*)" \);'
csrf = re.search(matchme,str(r.text))
payload = {
'login' : 'Login',
'usernamefld' : user,
'passwordfld' : password
}
r = s.post(url,data=payload,verify = False,headers={"X-CSRFToken":csrf.group(1)})
r = s.get(url,verify = False)
tree = html.fromstring(r.content)
tr_elements = tree.xpath('//tr')
try:
headers = [header.text for header in tr_elements[0]]
except IndexError:
print("Error retrieving lease list. Are you sure username and password were set up in script?")
quit()
ip.extend(tree.xpath('//table[@class="table table-striped"]//tbody//tr//td[' + str(headers.index('IP address') + 1) +']//text()'))
for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('MAC address') + 1) +']'):
if bool(re.search(r'([0-9a-f]{2}(?::[0-9a-f]{2}){5})', node.text)):
mac.append(node.text)
for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Hostname') + 1) +']'):
if node.text is None:
hostname.append('no_hostname')
else:
hostname.append(node.text)
for i in range(len(mac)):
mac[i] = mac[i].strip()
return(list(zip(ip, mac, hostname)))
if __name__ == "__main__":
dhcp_list = scrape_opnsense_dhcp(url, user, password)
for entry in dhcp_list:
print(entry)
@smithj33
Copy link

Any way we can get an updated version? It's throwing all kinds of errors. I assume OPNsense changed some of the elements. Thanks

@pletch
Copy link
Author

pletch commented Jul 17, 2022

Let me have a look.

@pletch
Copy link
Author

pletch commented Jul 17, 2022

@smithj33, there was indeed an error in the script version that I uploaded but you are the first to mention. Can you try the version there now and let me know if you are still having issues? I just tried it with my own installation on OpnSense 22.1.10 and it works fine via http access.

@smithj33
Copy link

smithj33 commented Jul 17, 2022

Works perfect! Thank you for taking time to update it. I used it with https and no issues besides the cert warning.

@gsmA42
Copy link

gsmA42 commented Feb 7, 2023

This is awesome. Thank you!

@kellerassel007
Copy link

kellerassel007 commented Jun 13, 2023

I updated the code to work with latest OPNSense version 23.1.9. It seems that it required an additional field in POST request.
Also I included the description of each device from DHCP overview.

Usage:
OPNSENSE_USERNAME=root OPNSENSE_PASSWORD='abc' python3 opnsense-dhcp-ip-overview.py

#!/usr/bin/env python3

#
# # This python script provides a function to query the opnSense (+v21.3) dhcp leases status page and return a list of tuples including 
# ip, hostname, and mac address. To use, ensure LXML is installed via package manager or via pip.
#
# 27-Mar-2021 - Original release
# 17-Jul-2022 - Fix url in scrape function. Add error trapping for case where user/pass are not set up correctly.
# 13-Jun-2023 - Fix missing field in POST request for new OPNSense version. Added field Description.
#

# Use:
# OPNSENSE_USERNAME=marcel OPNSENSE_PASSWORD='abc' python3 opnsense-dhcp-ip-overview.py

import os
import sys
import requests
import urllib3
import re
from lxml import html

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url  = "https://192.168.1.1/status_dhcp_leases.php" #change url to match your opnSense machine address. Note http or https!
user = os.getenv('OPNSENSE_USERNAME')
password = os.getenv('OPNSENSE_PASSWORD')

def scrape_opnsense_dhcp(url, user, password):
   ip = []
   mac = []
   hostname = []
   description = []

   s = requests.session()
   r = s.get(url,verify = False)

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))
   csrf_token = csrf.group(1)
   
   matchme = 'input type="hidden" name="([^"]*)" value="([^"]*)"'
   hidden_field = re.search(matchme,str(r.text))
   hidden_field_id = hidden_field.group(1)
   hidden_field_value = hidden_field.group(2)

   payload = {
'login' : '1',
'usernamefld' : user,
'passwordfld' : password
}

   payload[hidden_field_id] = hidden_field_value

   r = s.post(url,data=payload,verify = False,headers={"X-CSRFToken":csrf_token})
   r = s.get(url,verify = False)
   tree = html.fromstring(r.content)
   tr_elements = tree.xpath('//tr')
   try:
      headers = [header.text for header in tr_elements[0]]
   except IndexError:
      print("Error retrieving lease list. Are you sure username and password were set up in script?")
      quit()

   ip.extend(tree.xpath('//table[@class="table table-striped"]//tbody//tr//td[' + str(headers.index('IP address') + 1) +']//text()'))

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('MAC address') + 1) +']'):
      if bool(re.search(r'([0-9a-f]{2}(?::[0-9a-f]{2}){5})', node.text)):
         mac.append(node.text)

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Hostname') + 1) +']'):
      if node.text is None:
         hostname.append('no_hostname')
      else:
         hostname.append(node.text)
         
   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Description') + 1) +']'):
      if node.text is None:
         description.append('no_description')
      else:
         description.append(node.text)

   for i in range(len(mac)):
      mac[i] = mac[i].strip()

   return(list(zip(ip, mac, hostname, description)))

if __name__ == "__main__":     
    dhcp_list = scrape_opnsense_dhcp(url, user, password)

    for entry in dhcp_list:
            print(entry)

@dheldt
Copy link

dheldt commented Sep 19, 2023

Hi there,

for me, neither version works (having an OPNsense 23.7.3-amd64 running).
First of all because on my system there is no status_dhcp_leases.php present. Instead I get a "Page not found"-Error.

But I stumplet across /api/dhcpv4/leases/searchlease, which does work on my system. A simple get Request there (in a browser where I am logged in) gives a json object of the form:

{"total":18,"rowCount":18,"current":1,"rows":[{"address":"192.168.x.y","type":"static","mac":"SomeMac","starts":"","ends":"","hostname":"firstHost","descr":"","if_descr":"default","if":"opt8","state":"active","status":"offline","man":"manufacturer"},
....

]

so ... maybe this is the right way to go and fix instead of parsing html?

And, thank you a lot for your work and script, which has led me to the get-endpoint in the first place!

@pletch
Copy link
Author

pletch commented Sep 19, 2023

Yes, this is broken as of 23.7.x due to changes in the lease page. I haven't dug into trying to fix yet.

I wasn't aware of the searchlease endpoint but it does indeed seem like an opportunity to simplify. Will take a look when I get a little time in the next few days.

@dheldt
Copy link

dheldt commented Sep 19, 2023

If you adept as follows, it does (currently) work:

mainUrl = "https://192.168.1.1/"
url  = "https://192.168.1.1/api/dhcpv4/leases/searchlease

   s = requests.session()
   r = s.get(mainUrl,verify = False)

   if not(r == 200):
   	print("failed to connect")

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))
   csrf_token = csrf.group(1)
   
   matchme = 'input type="hidden" name="([^"]*)" value="([^"]*)"'
   hidden_field = re.search(matchme,str(r.text))
   hidden_field_id = hidden_field.group(1)
   hidden_field_value = hidden_field.group(2)

   payload = {
'login' : '1',
'usernamefld' : user,
'passwordfld' : password
}

   payload[hidden_field_id] = hidden_field_value
   r = s.post(mainUrl,data=payload,verify = False,headers={"X-CSRFToken":csrf_token})
  
   if not(r == 200):
   	print("failed to log in")
   	
   r = s.get(url,verify = False)
   
   return r.json()

Note that you need to use two different urls now, because the log in does not work for the rest endpoint.

regards,

daniel

@pletch
Copy link
Author

pletch commented Sep 19, 2023

Thank you for working through this!

It turns out it is simpler to set up an api access key as described here:
https://docs.opnsense.org/development/how-tos/api.html.

The code to retrieve the leases can then simply be reduced to a single GET request.

import json
import requests

# define endpoint and credentials
api_key = '---------'   #use your unique key
api_secret = '-----------------'  #use your unique secret
url = 'http://opnsense.home/api/dhcpv4/leases/searchlease'
r = requests.get(url,
                 verify='OPNsense.pem',
                 auth=(api_key, api_secret))

if r.status_code == 200:
    return r.json()
else:
    print ('Connection / Authentication issue, response received:')

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment