URL Extractor: Sniffing HTTP and HTTPS Traffic with Scapy

main.py

from scapy.all import sniff, IP, TCP, Raw


def extract_http_url(packet):
    """Extracts and prints HTTP URL from the packet"""
    if packet.haslayer(Raw):
        try:
            http_payload = packet[Raw].load.decode('utf-8')
            if "Host:" in http_payload:
                headers = http_payload.split("\r\n")
                host = None
                path = None
                for header in headers:
                    if header.startswith("Host:"):
                        host = header.split(" ")[1]
                    if "GET " in header or "POST " in header:
                        path = header.split(" ")[1]
                if host and path:
                    url = f"http://{host}{path}"
                    print(f"HTTP Request: {url}")
        except UnicodeDecodeError:
            pass

def extract_https_url(packet):
    """Extracts and prints HTTPS URL from the packet"""
    if packet.haslayer(TCP) and packet.haslayer(Raw):
        try:
            payload = packet[Raw].load
            
            # Check if it is a TLS Client Hello
            if payload[0] == 0x16 and payload[1:3] == b'\x03\x01':  # TLS handshake, version TLS 1.0
                # print(payload)
                if payload[5] == 0x01:  # Client Hello
                    # Get SNI (Server Name Indication)
                    sni_start = payload.find(b'\x00\x00') + 8
                    sni_length = payload[sni_start]
                    sni = payload[sni_start + 1:sni_start + 1 + sni_length].decode('utf-8')
                    url = f"https://{sni}"
                    print(f"HTTPS Request: {url}")
        except Exception as e:
            pass

def packet_callback(packet):
    """Callback function for packet sniffing"""
    if packet.haslayer(IP) and packet.haslayer(TCP):
        extract_http_url(packet)
        extract_https_url(packet)

def main():
    print("Starting packet capture...")
    # Capture packets with a filter to include only HTTP and HTTPS traffic
    sniff(prn=packet_callback, store=0)

if __name__ == "__main__":
    main()