logScript.py

# -*- coding: utf-8 -*-
import sys
import datetime
import time
import calendar
import operator
from prettytable import PrettyTable

def isMonitorRequest(requestList):
        if "/monitor/l7check" in requestList:
                return True
        else:
                return False

def isResourceRequest(requestList):
        if "/resources/" in requestList[4]:
                return True
        else:
                return False

def isCookieRequest(requestList):
        if "/cookie" in requestList[4]:
                return True
        else:
                return False
        
        
def isIncludeIP(requestList):
        arr = requestList[0].split(".")
        if len(arr) == 4:
                return True
        else:
                return False
 

def makeLogLineToList(line):
        requestList = filter(None, line.replace("-"," ").replace("\""," ").replace("["," ").replace("]", " ").split(" "));
        if isIncludeIP(requestList) == True and isMonitorRequest(requestList) == False and isResourceRequest(requestList) == False and isCookieRequest(requestList)==False:
                return requestList

def getDatetimeForm(date, period):
        dateList = date.split("/")
        periodList = period.split(":")

        dateList = list(map(int, dateList))
        periodList = list(map(int, periodList))

        retDatetime = datetime.datetime(dateList[0], dateList[1], dateList[2], periodList[0], periodList[1])
        return retDatetime

def getMonthFromAbbr(abbr):
        for monthIdx in range(1, 13):
                if calendar.month_abbr[monthIdx] == abbr:
                        return monthIdx

def isCoveredPeriod(reqList, date, period):
        reqDateList = reqList[1].split(":")
        reqDateList[0] = reqDateList[0].split("/")

        reqDateList[0][1] = str(getMonthFromAbbr(reqDateList[0][1]))
        reqDate = getDatetimeForm(reqDateList[0][2]+"/"+reqDateList[0][1]+"/"+reqDateList[0][0], reqDateList[1]+":"+reqDateList[2])

        stdDate = getDatetimeForm(date,period)
        stdDateTimestamp = time.mktime(time.strptime(str(stdDate), "%Y-%m-%d %H:%M:%S"))

        maxDateTimestamp = int(stdDateTimestamp) + 300
        minDateTimestamp = int(stdDateTimestamp) - 300
        reqDateTimestamp = int(time.mktime(time.strptime(str(reqDate), "%Y-%m-%d %H:%M:%S")))

        if reqDateTimestamp >= minDateTimestamp and reqDateTimestamp <= maxDateTimestamp:
                return True
        else:
                return False

def getUserIPRanking(reqList):
        ipRanking = {}
        if len(reqList) == 0:
                print "해당 기간 자료가 없습니다\n"
                return
        for item in reqList:
                if item[0] in ipRanking:
                        ipRanking[item[0]] += 1
                else:
                        ipRanking[item[0]] = 1
        ipRanking_x = sorted(ipRanking.iteritems(), key=operator.itemgetter(1), reverse=True)

        print "<접속횟수 별 IP>"
        pt = PrettyTable()
        pt.field_names = ['순위','IP', '횟수', '비율']
        for index, item in enumerate(ipRanking_x):
                pt.add_row([index+1, item[0], item[1], str(round(float(item[1])/float(len(reqList))*100,2))+"%"])
        print pt.get_string(title="접속횟수 별 IP")

def getURLRanking(reqList):
        urlRanking = {}
        if len(reqList) == 0: return
        for item in reqList:
                if item[4] in urlRanking:
                        urlRanking[item[4]] += 1
                else:
                        urlRanking[item[4]] = 1
        urlRanking_x = sorted(urlRanking.iteritems(), key=operator.itemgetter(1), reverse=True)
        pt = PrettyTable()
        print "<접속횟수 별 URL>"
        pt.field_names=['순위','URL','횟수', '비율']
        for index, item in enumerate(urlRanking_x):
                pt.add_row([index+1, item[0], item[1], str(round(float(item[1])/float(len(reqList))*100,2))+"%"])
        print pt.get_string(title="접속횟수 별 URL")

def main():
        print """
****************************************************************************
# Usage
* get ip, user ranking: input> date period (ex>20180227 11:11:11)
* exit: input> exit
****************************************************************************
"""

        while True:
                input = raw_input("input> ")
                argumentArr = input.split(" ")
                if(input == "exit"):
                        print "bye!"
                        break
                if(len(argumentArr)<2): continue
                date = argumentArr[0]
                period = argumentArr[1]

                # 적절한 argument form인지 체크필요

                try:
                        f = open("/home1/irteam/logs/apache/apache-2.4.29/access.log."+date, 'r');
                except IOError as e:
                        print("해당 기간의 파일이 존재하지 않습니다.")
                date = date[0:4]+"/"+date[4:6]+"/"+date[6:8]

                coveredPeriodReqList = []
                while True:
                        line = f.readline()
                        if not line: break
                        reqList = makeLogLineToList(line);

                        if reqList == None: continue

                        if isCoveredPeriod(reqList, date, period) == True:
                                coveredPeriodReqList.append(reqList)
                f.close();

                getUserIPRanking(coveredPeriodReqList)
                getURLRanking(coveredPeriodReqList)

main()