Generate usage charts from squid logs

proxy_analyzer.py

#!/usr/bin/env python
from __future__ import print_function
import argparse
import os
import progressbar
import numpy as np
from plotly import tools
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot

class Analyzer(object):
	def __init__(self, months, file, graphs):
		self.months = months
		self.file = file
		self.graphs = graphs
		self.checkArgs()
		self.time = Time(self.setInitialTimestamp(), months)

	def checkArgs(self):
		if not self.months > 0 :
			print('months: Value of months should be greater than 0.')
			exit(1)
		if not os.path.isfile(self.file):
			print(self.file + ": File doesn't exist")
			exit(2)

	def setInitialTimestamp(self):
		with open(self.file) as logs:
			initial_timestamp = logs.readline().split()
			initial_timestamp = float(initial_timestamp[0])
		return initial_timestamp

	def dataUsageCharts(self):
		daily_usage = np.zeros(self.time.expected['days'])
		weekly_usage = np.zeros(self.time.expected['weeks'])
		monthly_usage = np.zeros(self.time.expected['months'])
		total_usage = 0

		with open(self.file) as logfile:
			log = logfile.readlines()
			bar = progressbar.ProgressBar(
				term_width = 56,
				max_value = self.time.expected['days'],
				widgets = [
					progressbar.Counter(format='%(value)05d/%(max_value)05d'),
					' ',
					progressbar.Bar('=', '[', ']', '.'),
					' ',
					progressbar.ETA()
					])

			for request in log:
				request = request.split()
				current_time = float(request[0])
				data_usage = int(request[4])
				try:
					daily_usage[self.time.days(current_time)] += data_usage
					weekly_usage[self.time.weeks(current_time)] += data_usage
					monthly_usage[self.time.months(current_time)] += data_usage
					total_usage += data_usage
					bar.update(self.time.days(current_time))
				except Exception as e:
					break
		bar.update(self.time.expected['days'])
		print('\nUsed values that fit for 6 months')
		
		daily_average_usage = np.average(daily_usage)
		weekly_average_usage = np.average(weekly_usage)
		monthly_average_usage = np.average(monthly_usage)
		print('Daily Average Usage:', getHumanReadableSize(daily_average_usage))
		print('Weekly Average Usage:', getHumanReadableSize(weekly_average_usage))
		print('Monthly Average Usage:', getHumanReadableSize(monthly_average_usage))
		print('Total Usage in '+ str(self.months) +' months:', getHumanReadableSize(total_usage))

		if not graphs:
			daily_trace = go.Scatter(
				x = np.arange(self.time.expected['days']),
				y = daily_usage,
				mode = 'lines+markers',
				name = 'Daily Usage'
			)

			weekly_trace = go.Scatter(
				x = np.arange(self.time.expected['weeks']),
				y = weekly_usage,
				mode = 'lines+markers',
				name = 'Weekly Usage'
			)

			monthly_trace = go.Scatter(
				x = np.arange(self.time.expected['months']),
				y = monthly_usage,
				mode = 'lines+markers',
				name = 'Monthly Usage'
			)

			fig = tools.make_subplots(rows=2, cols=2, specs=[[{'colspan': 2}, None], [{}, {}]], subplot_titles=('Daily', 'Weekly', 'Monthly'))
			fig.append_trace(daily_trace, 1, 1)
			fig.append_trace(weekly_trace, 2, 1)
			fig.append_trace(monthly_trace, 2, 2)
			fig['layout'].update(title='Usage Stats')
			# data = [ daily_trace, weekly_trace, monthly_trace ]
			plot(fig)

			# plot(data)

class Time(object):
	def __init__(self, initial_timestamp, months):
		self.hour = 60 * 60
		self.day = self.hour * 24
		self.week = self.day * 7
		self.month = self.day * 30
		self.start = initial_timestamp
		self.end = self.start + months * self.month
		self.expected = { 'days': months * 30, 'months': months, 'weeks': round(months * 30 / 7) }

	def days(self, target):
		return int((target - self.start ) / self.day)
	
	def weeks(self, target):
		return int((target - self.start ) / self.week)

	def months(self, target):
		return int((target - self.start ) / self.month)

	def next(self, target):
		return target < self.end

def getHumanReadableSize(bytes):
	size = ['Bytes', 'KB', 'MB', 'GB']
	counter = 0
	while counter < 3 and (bytes / 1024) > 1:
		bytes /= 1024
		counter += 1
	return str(round(bytes, 2)) + ' ' + size[counter]

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('-m', '--months', help = 'Number of months for analysis', required = True, type = int)
	parser.add_argument('-q', '--no-graphs', help = 'Disable graph outputs', action = 'store_true')
	parser.add_argument('-f', '--file', help = 'Squid log file to analyze', required = True)
	args = parser.parse_args()
	log = Analyzer(args.months, args.file, args.no_graphs)
	log.dataUsageCharts()

if __name__ == '__main__':
	main()
  

requirements.txt

certifi==2017.11.5
chardet==3.0.4
decorator==4.1.2
idna==2.6
ipython-genutils==0.2.0
jsonschema==2.6.0
jupyter-core==4.4.0
nbformat==4.4.0
numpy==1.13.3
plotly==2.2.3
progressbar2==3.34.3
python-utils==2.2.0
pytz==2017.3
requests==2.18.4
six==1.11.0
traitlets==4.3.2
urllib3==1.22