script to find out ICMP bookkeeping overhead.

icmp-skbuf-test.py

#!/usr/bin/env python

import sys
import re
import os
import socket as s
import time
import struct

# Per packet rx_queue & tx_queue (SO_RCVBUF, SO_SNDBUF) utilisation according to /proc/net/raw{,6}
# 2.6.32-45-generic  IPv4  RX/TX 296/296
# 2.6.32-45-generic  IPv6  RX/TX 360/360
# 2.6.32-39-server   IPv4  RX/TX 424/296
# 2.6.32-39-server   IPv6  RX/TX 424/360
# 3.2.0-33-generic   IPv4  RX/TX 768/768
# 3.2.0-33-generic   IPv6  RX/TX 768/768
# (IP_RECVERR and SO_TIMESTAMP do not matter)

IP_RECVERR = 11 # Linux
SO_TIMESTAMP = 29 # Linux

def carry_around_add(a, b):
    c = a + b
    return (c & 0xffff) + (c >> 16)

def checksum(msg):
    s = 0
    for i in range(0, len(msg), 2):
        w = ord(msg[i]) + (ord(msg[i+1]) << 8)
        s = carry_around_add(s, w)
    return ~s & 0xffff

def gen_ping(af, seq):
    if af == s.AF_INET:
        msg = struct.pack(">BBHHH", 8, 0, 0, os.getpid(), seq) # type, code, checksum, id, seq
        cs = checksum(msg)
        msg = struct.pack(">BBHHH", 8, 0, s.ntohs(cs), os.getpid(), seq) # type, code, checksum, id, seq
    elif af == s.AF_INET6:
        msg = struct.pack(">BBHHH", 128, 0, 0, os.getpid(), seq)
    return msg

def parse_proc_net(af):
    fname = {s.AF_INET:  "raw", s.AF_INET6: "raw6"}[af]
    retval = {}
    with open("/proc/net/" + fname) as fd:
        fd.next() # skip first line
        for lineno, line in enumerate(fd):
            # sl local_address(:port) remote_address(:port?) st(ate) tx_queue rx_queue tr=0 tm->when=0 retrnsmt=0 uid timeout=0 inode ref pointer drops
            chunks = re.split("[: ]*", line.strip())
            asis = {
                "sl": int(chunks[0]),
                "tx_queue": int(chunks[6], 16),
                "rx_queue":  int(chunks[7], 16),
                "inode": int(chunks[13]), # to track fd changes
                "ref": int(chunks[14]),
                "drops": int(chunks[16]),
            }
            assert asis["inode"] not in retval
            retval[asis["inode"]] = asis
    return retval

def make_fd_with_inode(af):
    proc_net_before = parse_proc_net(af)
    ipproto = {s.AF_INET: s.IPPROTO_ICMP, s.AF_INET6: s.IPPROTO_ICMPV6}[af]
    fd = s.socket(af, s.SOCK_RAW, ipproto)
    proc_net_after = parse_proc_net(af)
    new_nodes = set(proc_net_after.keys()).difference(set(proc_net_before.keys()))
    assert len(new_nodes) == 1
    return (fd, new_nodes.pop())

def main():
    # IP_RECVERR, SO_TIMESTAMP, O_NONBLOCK, ICMP_FILTER - are omited
    af = {
        "v4": s.AF_INET,
        "v6": s.AF_INET6,
    }[sys.argv[1]]
    fd, inode = make_fd_with_inode(af)

    dst = {
        # mirror.yandex.ru - hardcoded
        s.AF_INET: "213.180.204.183",
        s.AF_INET6: "2a02:6b8:0:201::1",
    }[af]

    if len(sys.argv) > 3:
        fd.setsockopt(s.SOL_SOCKET, s.SO_RCVBUF, int(sys.argv[2]))
        fd.setsockopt(s.SOL_SOCKET, s.SO_SNDBUF, int(sys.argv[3]))

    #fd.setsockopt(s.SOL_IP, IP_RECVERR, 1)
    #fd.setsockopt(s.SOL_SOCKET, SO_TIMESTAMP, 1)

    seq = 0
    line = parse_proc_net(af)[inode]
    tx_per_pkt = float("+inf")
    rx_per_pkt = float("+inf")
    while True:
        print ". %4d %6d %s" % (seq, line["rx_queue"], line)
        fd.sendto(gen_ping(af, seq), (dst, 0))
        while True:
            newline = parse_proc_net(af)[inode]
            if newline["tx_queue"] == 0:
                break
            else:
                tx_per_pkt = min(newline["tx_queue"], tx_per_pkt)
                time.sleep(0.01)
        while True:
            newline = parse_proc_net(af)[inode]
            if newline["rx_queue"] > line["rx_queue"]:
                rx_per_pkt = min(newline["rx_queue"] - line["rx_queue"], rx_per_pkt)
                break
            elif newline["drops"] > 0 :
                break
            else :
                time.sleep(0.01)
        if newline["drops"] > 0 :
            print "! %4d %6d %s" % (seq, line["rx_queue"], newline)
            print 1.0 * line["rx_queue"] / seq
            print "rx_per_pkt:", rx_per_pkt
            print "tx_per_pkt:", tx_per_pkt
            print "SO_RCVBUF:", fd.getsockopt(s.SOL_SOCKET, s.SO_RCVBUF)
            print "SO_SNDBUF:", fd.getsockopt(s.SOL_SOCKET, s.SO_SNDBUF)
            break
        line = newline
        time.sleep(0.2)
        seq += 1

if __name__ == "__main__":
    main()