apacheLogParser.go

// -*- mode:go;mode:go-playground -*-
// snippet of code @ 2019-03-01 15:48:42

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Apache2 sample of access.log
const aLog = `65.55.147.227 - - [15/Oct/2009:02:00:24 +0000] "GET /index.html HTTP/1.1" 200 21878
65.55.86.34 - - [15/Oct/2009:02:00:58 +0000] "GET /index.html HTTP/1.1" 200 1416
148.188.55.88 - - [15/Oct/2009:02:01:41 +0000] "GET /faq.html HTTP/1.1" 200 10946
72.30.57.238 - - [15/Oct/2009:02:01:59 +0000] "GET /contribute.txt HTTP/1.0" 200 39943
66.249.139.233 - - [15/Oct/2009:02:02:09 +0000] "GET /faq.html HTTP/1.1" 200 17247
72.30.50.248 - - [15/Oct/2009:02:02:13 +0000] "GET /index.html HTTP/1.0" 200 7883
216.129.13.4 - - [15/Oct/2009:02:02:37 +0000] "GET /contribute.txt HTTP/1.0" 200 18119
66.249.61.232 - - [15/Oct/2009:02:02:39 +0000] "GET /contribute.txt HTTP/1.1" 200 10946
65.55.80.97 - - [15/Oct/2009:02:02:51 +0000] "GET /index.html HTTP/1.1" 200 1416
65.55.161.41 - - [15/Oct/2009:02:02:54 +0000] "GET /index.html HTTP/1.1" 200 37122`

// Regular expression breaking access.log records into the corresponding fields.
const regexApacheAccessLogDefaultFormat = `^(?P<IP>[\d{1-3}\.]+)` + // IP address
	`\s+\S+\s+\S+\s+` + // skipping two fields
	`\[(?P<date>\d+\/\w{3}\/\d{4}):` + // date
	`(?P<time>\d{2}:\d{2}:\d{2} [+-]\d{4})\]\s+` + // time
	`"(?P<method>\w+)\s+` + // method
	`(?P<URI>[[:graph:]]+)\s+` + // URI
	`(?P<proto>[[:graph:]]+)"\s+` + // proto
	`(?P<status>\d+)\s+` + // response status code
	`(?P<size>\d+)` // returned content size

// Utility function parsing out access.log and returning the total amount of
// transferred data.
func sumTransferApacheAccessLog(apacheAccessLogPath string) (sum int) {
	apacheLog, err := os.Open(apacheAccessLogPath)
	if err != nil {
		log.Fatalf("failed to open Apache access log: %v", err)
	}
	defer apacheLog.Close()

	return sumTransfer(apacheLog)
}

// sumTransfer scans access log reader and returns total size of transferred
// content.
func sumTransfer(apacheAccessLogReader io.Reader) (sum int) {
	re := regexp.MustCompile(regexApacheAccessLogDefaultFormat)
	scanner := bufio.NewScanner(apacheAccessLogReader)
	for scanner.Scan() {
		n, _ := strconv.Atoi(re.FindAllStringSubmatch(scanner.Text(), -1)[0][8]) // returned content size
		sum += n
	}
	return
}

func main() {
	fmt.Println(sumTransfer(strings.NewReader(aLog))/1024, "Kb")
	fmt.Println(sumTransferApacheAccessLog("/var/log/apache2/access.log"), "bytes") // /tmp/apache.log
}