Collect Network IO Statistic using psutil

collect_net_io.py

#!/usr/bin/env python

#-#  Copyright 2015 Karlsruhe Institute of Technology
#-#
#-#  Licensed under the Apache License, Version 2.0 (the "License");
#-#  you may not use this file except in compliance with the License.
#-#  You may obtain a copy of the License at
#-#
#-#      http://www.apache.org/licenses/LICENSE-2.0
#-#
#-#  Unless required by applicable law or agreed to in writing, software
#-#  distributed under the License is distributed on an "AS IS" BASIS,
#-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-#  See the License for the specific language governing permissions and
#-#  limitations under the License.

__author__ = 'Manuel Giffels'
__email__ = 'giffels@gmail.com'

from optparse import OptionParser
try:
    from psutil import net_io_counters
except ImportError:
    #psutil is distributed under BSD license reproduced below.
    #
    #Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola'
    #All rights reserved.
    #
    #Redistribution and use in source and binary forms, with or without modification,
    #are permitted provided that the following conditions are met:
    #
    # * Redistributions of source code must retain the above copyright notice, this
    #   list of conditions and the following disclaimer.
    # * Redistributions in binary form must reproduce the above copyright notice,
    #   this list of conditions and the following disclaimer in the documentation
    #   and/or other materials provided with the distribution.
    # * Neither the name of the psutil authors nor the names of its contributors
    #   may be used to endorse or promote products derived from this software without
    #   specific prior written permission.
    #
    #THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
    #ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    #WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
    #DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
    #ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
    #(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
    #LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
    #ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    #(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
    #SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    from collections import namedtuple
    _nt_net_iostat = namedtuple('iostat','bytes_sent bytes_recv packets_sent packets_recv errin errout dropin dropout')
    def _net_io_counters():
        """Return network I/O statistics for every network interface
        installed on the system as a dict of raw tuples.
        """
        f = open("/proc/net/dev", "r")
        try:
            lines = f.readlines()
        finally:
            f.close()

        retdict = {}
        for line in lines[2:]:
            colon = line.rfind(':')
            assert colon > 0, repr(line)
            name = line[:colon].strip()
            fields = line[colon + 1:].strip().split()
            bytes_recv = int(fields[0])
            packets_recv = int(fields[1])
            errin = int(fields[2])
            dropin = int(fields[3])
            bytes_sent = int(fields[8])
            packets_sent = int(fields[9])
            errout = int(fields[10])
            dropout = int(fields[11])
            retdict[name] = (bytes_sent, bytes_recv, packets_sent, packets_recv,
                             errin, errout, dropin, dropout)
        return retdict

    def net_io_counters(pernic=False):
        """Return network I/O statistics as a namedtuple including
        the following attributes:
        - bytes_sent:   number of bytes sent
        - bytes_recv:   number of bytes received
        - packets_sent: number of packets sent
        - packets_recv: number of packets received
        - errin:        total number of errors while receiving
        - errout:       total number of errors while sending
        - dropin:       total number of incoming packets which were dropped
        - dropout:      total number of outgoing packets which were dropped
        (always 0 on OSX and BSD)
        If pernic is True return the same information for every
        network interface installed on the system as a dictionary
        with network interface names as the keys and the namedtuple
        described above as the values.
        """
        rawdict = _net_io_counters()
        if not rawdict:
            raise RuntimeError("couldn't find any network interface")
        if pernic:
            for nic, fields in rawdict.items():
                rawdict[nic] = _nt_net_iostat(*fields)
            return rawdict
        else:
            return _nt_net_iostat(*[sum(x) for x in zip(*rawdict.values())])

import sqlite3 as sqlite
import signal
import socket
import time


class SQLiteNetIoStats(object):
    def __init__(self, dbfile):
        self.conn = sqlite.connect(dbfile)

        #create db
        with self.conn:
            cur = self.conn.cursor()
            ### Performance Tuning
            cur.execute("PRAGMA cache_size=200000")
            cur.execute("PRAGMA synchronous = 0")
            cur.execute("CREATE TABLE IF NOT EXISTS MetaData(Id INTEGER PRIMARY KEY, hostname TEXT,\
                         UNIQUE(Id, hostname))")
            cur.execute("INSERT OR IGNORE INTO MetaData(hostname) VALUES ('%s')" % socket.gethostname())
            cur.execute("CREATE TABLE IF NOT EXISTS NetIoStat(Id INTEGER PRIMARY KEY, interface TEXT, bytes_sent\
                         INTEGER, bytes_recv INTEGER, packets_sent INTEGER, packets_recv INTEGER, timestamp INTEGER)")

    def add_net_io_stats(self, net_io_stats):
        values = []
        for interface, io_stat in net_io_stats.iteritems():
            values.append((str(interface),
                           long(io_stat['bytes_sent']),
                           long(io_stat['bytes_recv']),
                           long(io_stat['packets_sent']),
                           long(io_stat['packets_recv']),
                           long(io_stat['time_stamp'])))
        with self.conn:
            cur = self.conn.cursor()
            cur.executemany('INSERT INTO NetIoStat(interface, bytes_sent, bytes_recv, packets_sent, packets_recv,\
                             timestamp) VALUES(?,?,?,?,?,?)', values)


def get_net_io_stat(pernic=True):
    net_io_stats = net_io_counters(pernic=pernic)
    time_stamp = int(time.time())
    if pernic:
        net_io_stats_dict = {}
        for interface, snetio in net_io_stats.iteritems():
            net_io_stats_dict.update({interface: dict(snetio._asdict(), time_stamp=time_stamp)})
        return net_io_stats_dict
    return {'all': dict(net_io_stats._asdict(), time_stamp=time_stamp)}


def get_command_line_options():
    parser = OptionParser(usage="usage: %prog [options]")
    parser.add_option("-o", "--out", type="string", dest="output", help="Output DB File")
    parser.add_option("-i", "--interval", type="int", dest="interval",
                      help="Measurement Interval (Default: 1s)", default=1)

    options, args = parser.parse_args()

    if not options.output:
        parser.print_help()
        parser.error("You need to provide following options, --out=OutputDB.db")

    return options, args


class NetIoStatDaemon(object):
    def __init__(self, options, args):
        self.run = True
        self.interval = options.interval
        self.sql_net_io_stats = SQLiteNetIoStats(options.output)

    def register_signal(self, signum):
        signal.signal(signum, self.signal_handler)

    def signal_handler(self, signum, frame):
        print "Caught signal", signum
        print "Shutdown..."
        self.run = False

    def serve_forever(self):
        while self.run:
            net_io_stats = get_net_io_stat(pernic=True)
            self.sql_net_io_stats.add_net_io_stats(net_io_stats)
            time.sleep(self.interval)

if __name__ == '__main__':
    net_io_stat_daemon = NetIoStatDaemon(*get_command_line_options())
    net_io_stat_daemon.register_signal(signal.SIGHUP)
    net_io_stat_daemon.register_signal(signal.SIGINT)
    net_io_stat_daemon.serve_forever()

net_io_analysis.ipynb