Created
October 24, 2014 22:34
-
-
Save jdevillard/bee7742b59f6f724d7c4 to your computer and use it in GitHub Desktop.
Azure Web Job - Parsing IIS Log Blob to Table
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#inspired from https://code.msdn.microsoft.com/windowsazure/Cloud-Service-Fundamentals-4ca72649 | |
public class WebLogEntry | |
{ | |
public String RowKey { get; set; } | |
public String PartitionKey { get; set; } | |
/* | |
date 2012-06-20 | |
time 13:00:06 | |
s-sitename W3SVC472036332 | |
s-computername RD00155D3AB0F9 | |
s-ip 10.119.166.64 | |
cs-method GET | |
cs-uri-stem /en/common-room/5/2/3 | |
cs-uri-query 1340197206&ajax=ajax&_=1340197205660 | |
s-port 80 | |
cs-username SpiritAuror180 | |
c-ip 195.10.36.9 | |
cs-version HTTP/1.1 | |
cs(User-Agent) Mozilla/5.0+(Windows+NT+6.0;+rv:12.0)+Gecko/20100101+Firefox/12.0 | |
cs(Cookie) SessionCookie=ljm1uqofn2a1zvdgll12lpk0;+__Req********************* | |
cs(Referer) http://jeremiedevillard.wordpress.com | |
cs-host www.pottermore.com | |
sc-status 200 | |
sc-substatus 0 | |
sc-win32-status 0 | |
sc-bytes 3343 | |
cs-bytes 2082 | |
time-taken 62 | |
*/ | |
public DateTime Date{ get; set; } | |
public DateTime Time{ get; set; } | |
public string SiteName{ get; set; } | |
public string ComputerName{ get; set; } | |
public string RoleInstance{ get; set; } | |
public string ServerIpAddress{ get; set; } | |
public string HttpMethod{ get; set; } | |
public string UriStem{ get; set; } | |
public string UriQuery{ get; set; } | |
public int Port{ get; set; } | |
public string UserName{ get; set; } | |
public string ClientIpAddress{ get; set; } | |
public string Version{ get; set; } | |
public string UserAgent{ get; set; } | |
public string Cookie{ get; set; } | |
public string Referer{ get; set; } | |
public string Host{ get; set; } | |
public int Status{ get; set; } | |
public int SubStatus{ get; set; } | |
public int Win32Status{ get; set; } | |
public int ScBytes{ get; set; } | |
public int CsBytes{ get; set; } | |
public int TimeTaken{ get; set; } | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static void IISLogParserToTable([BlobTrigger(@"wad-iis-logfiles/{name}")] Stream input,string name, | |
[Table("wadiislogs")] ICollector<WebLogEntry> tableBinding) | |
{ | |
//blob name of type : wad-iis-logfiles/5d8a5e565a4544ecb4fcf12b2d533ed6/Router/Router_IN_0/Web/W3SVC1273337584 | |
using (var reader = new StreamReader(input)) | |
{ | |
foreach (var line in reader.GetSplittedLines(" ").Skip(4)) | |
{ | |
//To allow descending entries in the table | |
var inverseTimeKey = DateTime | |
.MaxValue | |
.Subtract(DateTime.UtcNow) | |
.TotalMilliseconds | |
.ToString(CultureInfo.InvariantCulture); | |
var entry = new WebLogEntry | |
{ | |
PartitionKey = String.Empty, | |
RowKey = string.Format("{0}-{1}",inverseTimeKey,Guid.NewGuid()), | |
RoleInstance = name.Split(new[] {'/'})[2], | |
Date = Convert.ToDateTime(line[0]), | |
Time = Convert.ToDateTime(line[1]), | |
SiteName = line[2], | |
ComputerName = line[3], | |
ServerIpAddress = line[4], | |
HttpMethod = line[5], | |
UriStem = line[6], | |
UriQuery = line[7], | |
Port = Convert.ToInt32(line[8]), | |
UserName = line[9], | |
ClientIpAddress = line[10], | |
Version = line[11], | |
UserAgent = line[12], | |
Cookie = line[13], | |
Referer = line[14], | |
Host = line[15], | |
Status = Convert.ToInt32(line[16]), | |
SubStatus = Convert.ToInt32(line[17]), | |
Win32Status = Convert.ToInt32(line[18]), | |
ScBytes = Convert.ToInt32(line[19]), | |
CsBytes = Convert.ToInt32(line[20]), | |
TimeTaken = Convert.ToInt32(line[21]) | |
}; | |
tableBinding.Add(entry); | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#used in https://code.msdn.microsoft.com/windowsazure/Cloud-Service-Fundamentals-4ca72649 | |
public static class LinqToTextReader | |
{ | |
public static IEnumerable<string[]> GetSplittedLines(this TextReader reader, string[] delimiters) | |
{ | |
using (var parser = new TextFieldParser(reader)) | |
{ | |
parser.Delimiters = delimiters; | |
string[] fields; | |
while ((fields = parser.ReadFields()) != null) | |
{ | |
yield return fields; | |
} | |
} | |
} | |
public static IEnumerable<string[]> GetSplittedLines(this TextReader reader, string delimiter) | |
{ | |
return GetSplittedLines(reader, new[] { delimiter }); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment