AccessLogAnalyser.java

package br.com.willianantunes.analyser;

import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.*;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Checks the requests status available in an access.log.
 * The following pattern is expected: '%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-Agent}i" %Dms'
 * @author Willian Antunes
 */
public class AccessLogAnalyser {
	public static void main(String args[]) throws URISyntaxException, IOException {
		String myRegex = "\\/edg-api-web\\/services\\/rest\\/usuarios\\/[1-9A-Za-z].*\\/autorizacao-acesso\\?servicoid=6434";
		Pattern myPattern = Pattern.compile(myRegex);
		Integer timeOut = 2500;
	
		System.out.format("Pattern (regex) used: %s\r\n\r\n", myPattern.toString());

		showAccessLogDetails(Paths.get(ClassLoader.getSystemResource("access_log.2017-03-09").toURI()), myPattern, timeOut);
		
		/**
		 * OUTPUT SAMPLE:

#### File: access_log.2017-03-16
Number of cases: 130389
Cases that have more than 2500ms: 163 which is 0.12501055 percent of the total
Latest request: 25555ms at 15:07:09
Fastest request: 26ms at 06:10:30
Average time: 254ms
00:00 - 02:59 => 1 which is 0.61349696 percent. 
	- Average: 3425ms
	- Latest request: 3425ms at 01:14:48
	- Fastest request: 3425ms at 01:14:48
06:00 - 08:59 => 29 which is 17.79141 percent. 
	- Average: 9685ms
	- Latest request: 19962ms at 08:07:39
	- Fastest request: 3004ms at 08:07:30
09:00 - 11:59 => 3 which is 1.8404908 percent. 
	- Average: 3221ms
	- Latest request: 3533ms at 10:22:25
	- Fastest request: 3060ms at 10:51:44
12:00 - 14:59 => 28 which is 17.177914 percent. 
	- Average: 4857ms
	- Latest request: 17103ms at 14:11:28
	- Fastest request: 2588ms at 13:03:18
15:00 - 17:59 => 101 which is 61.963192 percent. 
	- Average: 7992ms
	- Latest request: 25555ms at 15:07:09
	- Fastest request: 2849ms at 15:04:52
18:00 - 20:59 => 1 which is 0.61349696 percent. 
	- Average: 2598ms
	- Latest request: 2598ms at 18:12:59
	- Fastest request: 2598ms at 18:12:59
HTTP 200: 130341
HTTP 404: 48

		 */
	}
	
	private static void showAccessLogDetails(Path path, Pattern pattern, Integer timeOut) throws IOException {		
		Map<Boolean, List<String>> myConsumedLines = groupMyLinesByTimeOut(path, pattern, timeOut);
		
		List<String> myMatchedLines = new ArrayList<>(myConsumedLines.get(true));
		myMatchedLines.addAll(myConsumedLines.get(false));
		
		// Number of lines which matches the REGEX		
		Integer countOfLines = myMatchedLines.size();
		
		// Number of lines which delays more than {Integer timeOut} to be answered
		Integer countOfTimeOutCases = myConsumedLines.get(true).size();

		//  Summary statistics regarding time taken to process the request
		IntSummaryStatistics summaryStatistics = myMatchedLines.parallelStream()
			.mapToInt(l -> {
				String[] myProperties = l.split(" ");
				return Integer.parseInt(myProperties[myProperties.length-1].substring(0, myProperties[myProperties.length-1].length()-2));
			}).summaryStatistics();
		
		System.out.println(String.format("#### File: %s\r\n"
				+ "Number of cases: %s\r\n"
				+ "Cases that have more than %sms: %s which is %s percent of the total\r\n"
				+ "Latest request: %sms at %s\r\n"
				+ "Fastest request: %sms at %s\r\n"
				+ "Average time: %sms", 
				path.getFileName(), countOfLines, timeOut, countOfTimeOutCases, 
				((countOfTimeOutCases.floatValue()*100)/countOfLines.floatValue()),
				summaryStatistics.getMax(), queryByRequestTime(myMatchedLines, summaryStatistics.getMax()).get(0).substring(13, 21), 
				summaryStatistics.getMin(), queryByRequestTime(myMatchedLines, summaryStatistics.getMin()).get(0).substring(13, 21), 
				Double.valueOf(summaryStatistics.getAverage()).intValue()));
		
		Map<String, List<String>> myCompiledMap = groupByHourPeriod(myConsumedLines.get(true));
		for (String myKey: myCompiledMap.keySet()) {
			//  Summary statistics regarding a specific period
			IntSummaryStatistics myPeriodSummaryStatistics = myCompiledMap.get(myKey).parallelStream()
					.mapToInt(l -> {
						String[] myProperties = l.split(" ");
						return Integer.parseInt(myProperties[myProperties.length-1].substring(0, myProperties[myProperties.length-1].length()-2));
					}).summaryStatistics();
			
			System.out.format("%s => %s which is %s percent. \r\n"
					+ "\t- Average: %sms\r\n"
					+ "\t- Latest request: %sms at %s\r\n"
					+ "\t- Fastest request: %sms at %s\r\n", 
					myKey, myCompiledMap.get(myKey).size(),
					(Float.valueOf(myCompiledMap.get(myKey).size())*100/countOfTimeOutCases.floatValue()),
					Double.valueOf(myPeriodSummaryStatistics.getAverage()).intValue(),
					myPeriodSummaryStatistics.getMax(), queryByRequestTime(myCompiledMap.get(myKey), myPeriodSummaryStatistics.getMax()).get(0).substring(13, 21),
					myPeriodSummaryStatistics.getMin(), queryByRequestTime(myCompiledMap.get(myKey), myPeriodSummaryStatistics.getMin()).get(0).substring(13, 21));	
		}
		
		Map<String, List<String>> myMap = groupMyLinesHttpStatusCode(myMatchedLines);
		myMap.keySet().stream().forEach(k -> System.out.format("HTTP %s: %s\r\n", k, myMap.get(k).size()));
	}
	
	private static Map<String, List<String>> groupByHourPeriod(List<String> lines) {
		DateTimeFormatter myFormat = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss");
		
		return lines.parallelStream() 
			.collect(Collectors.groupingBy(l -> {
				String[] myProperties = l.split(" ");
				LocalDateTime myDateTime = LocalDateTime.parse(myProperties[3].substring(1).toLowerCase(), myFormat);
				switch (myDateTime.getHour()) {
				case 0:case 1:case 2:
					return "00:00 - 02:59";
				case 3:case 4:case 5:
					return "03:00 - 05:59";
				case 6:case 7:case 8:
					return "06:00 - 08:59";
				case 9:case 10:case 11:
					return "09:00 - 11:59";
				case 12:case 13:case 14:
					return "12:00 - 14:59";
				case 15:case 16:case 17:
					return "15:00 - 17:59";
				case 18:case 19:case 20:
					return "18:00 - 20:59";
				case 21:case 22:case 23:
					return "21:00 - 23:59";
				default:
					throw new RuntimeException("A strange thing has happened!");
				}
			}, TreeMap::new, Collectors.toList()));
	}
	
	private static List<String> queryByRequestTime(List<String> lines, Integer desiredTime) throws IOException {
		return lines.parallelStream()
		.filter(l -> {
			String[] myProperties = l.split(" ");
			Integer myTime = Integer.parseInt(myProperties[myProperties.length-1].substring(0, myProperties[myProperties.length-1].length()-2));
			return myTime.equals(desiredTime);
		})
		.map(l -> {
			String[] myProperties = l.split(" ");			
			return myProperties[3] + myProperties[4];
		}).collect(Collectors.toList());
	}
	
	private static Map<Boolean, List<String>> groupMyLinesByTimeOut(Path path, Pattern pattern, Integer timeOut) throws IOException {
		// The inverse sorted request list by request time out 
		// I didn't use parallelStream because the order would be affected
		Map<Boolean, List<String>> myTimeOutRequest = Files.lines(path)
			.filter(l -> pattern.matcher(l).find())
			.sorted((p1, p2) -> {
				String[] myPropertiesP1 = p1.split(" ");
				Integer myTimeP1 = Integer.parseInt(myPropertiesP1[myPropertiesP1.length-1].substring(0, myPropertiesP1[myPropertiesP1.length-1].length()-2));
				
				String[] myPropertiesP2 = p2.split(" ");
				Integer myTimeP2 = Integer.parseInt(myPropertiesP2[myPropertiesP2.length-1].substring(0, myPropertiesP2[myPropertiesP2.length-1].length()-2));
				
				return myTimeP2.compareTo(myTimeP1);				
			}).collect(Collectors.partitioningBy(l -> {
				String[] myProperties = l.split(" ");
				return Integer.parseInt(myProperties[myProperties.length-1].substring(0, myProperties[myProperties.length-1].length()-2)) >= timeOut;
			}, Collectors.toList()));
		
		return myTimeOutRequest;
	}
	
	private static Map<String, List<String>> groupMyLinesHttpStatusCode(List<String> lines) throws IOException {
		Map<String, List<String>> myTimeOutRequest = lines.parallelStream()
			.collect(Collectors.groupingBy(l -> {
				String[] myProperties = l.split(" ");
				return myProperties[8];
			}, TreeMap::new, Collectors.toList()));
		
		return myTimeOutRequest;
	}
}