Skip to content

Instantly share code, notes, and snippets.

@jdevillard
Created October 24, 2014 22:34
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jdevillard/bee7742b59f6f724d7c4 to your computer and use it in GitHub Desktop.
Save jdevillard/bee7742b59f6f724d7c4 to your computer and use it in GitHub Desktop.
Azure Web Job - Parsing IIS Log Blob to Table
#inspired from https://code.msdn.microsoft.com/windowsazure/Cloud-Service-Fundamentals-4ca72649
public class WebLogEntry
{
public String RowKey { get; set; }
public String PartitionKey { get; set; }
/*
date 2012-06-20
time 13:00:06
s-sitename W3SVC472036332
s-computername RD00155D3AB0F9
s-ip 10.119.166.64
cs-method GET
cs-uri-stem /en/common-room/5/2/3
cs-uri-query 1340197206&ajax=ajax&_=1340197205660
s-port 80
cs-username SpiritAuror180
c-ip 195.10.36.9
cs-version HTTP/1.1
cs(User-Agent) Mozilla/5.0+(Windows+NT+6.0;+rv:12.0)+Gecko/20100101+Firefox/12.0
cs(Cookie) SessionCookie=ljm1uqofn2a1zvdgll12lpk0;+__Req*********************
cs(Referer) http://jeremiedevillard.wordpress.com
cs-host www.pottermore.com
sc-status 200
sc-substatus 0
sc-win32-status 0
sc-bytes 3343
cs-bytes 2082
time-taken 62
*/
public DateTime Date{ get; set; }
public DateTime Time{ get; set; }
public string SiteName{ get; set; }
public string ComputerName{ get; set; }
public string RoleInstance{ get; set; }
public string ServerIpAddress{ get; set; }
public string HttpMethod{ get; set; }
public string UriStem{ get; set; }
public string UriQuery{ get; set; }
public int Port{ get; set; }
public string UserName{ get; set; }
public string ClientIpAddress{ get; set; }
public string Version{ get; set; }
public string UserAgent{ get; set; }
public string Cookie{ get; set; }
public string Referer{ get; set; }
public string Host{ get; set; }
public int Status{ get; set; }
public int SubStatus{ get; set; }
public int Win32Status{ get; set; }
public int ScBytes{ get; set; }
public int CsBytes{ get; set; }
public int TimeTaken{ get; set; }
}
public static void IISLogParserToTable([BlobTrigger(@"wad-iis-logfiles/{name}")] Stream input,string name,
[Table("wadiislogs")] ICollector<WebLogEntry> tableBinding)
{
//blob name of type : wad-iis-logfiles/5d8a5e565a4544ecb4fcf12b2d533ed6/Router/Router_IN_0/Web/W3SVC1273337584
using (var reader = new StreamReader(input))
{
foreach (var line in reader.GetSplittedLines(" ").Skip(4))
{
//To allow descending entries in the table
var inverseTimeKey = DateTime
.MaxValue
.Subtract(DateTime.UtcNow)
.TotalMilliseconds
.ToString(CultureInfo.InvariantCulture);
var entry = new WebLogEntry
{
PartitionKey = String.Empty,
RowKey = string.Format("{0}-{1}",inverseTimeKey,Guid.NewGuid()),
RoleInstance = name.Split(new[] {'/'})[2],
Date = Convert.ToDateTime(line[0]),
Time = Convert.ToDateTime(line[1]),
SiteName = line[2],
ComputerName = line[3],
ServerIpAddress = line[4],
HttpMethod = line[5],
UriStem = line[6],
UriQuery = line[7],
Port = Convert.ToInt32(line[8]),
UserName = line[9],
ClientIpAddress = line[10],
Version = line[11],
UserAgent = line[12],
Cookie = line[13],
Referer = line[14],
Host = line[15],
Status = Convert.ToInt32(line[16]),
SubStatus = Convert.ToInt32(line[17]),
Win32Status = Convert.ToInt32(line[18]),
ScBytes = Convert.ToInt32(line[19]),
CsBytes = Convert.ToInt32(line[20]),
TimeTaken = Convert.ToInt32(line[21])
};
tableBinding.Add(entry);
}
}
}
#used in https://code.msdn.microsoft.com/windowsazure/Cloud-Service-Fundamentals-4ca72649
public static class LinqToTextReader
{
public static IEnumerable<string[]> GetSplittedLines(this TextReader reader, string[] delimiters)
{
using (var parser = new TextFieldParser(reader))
{
parser.Delimiters = delimiters;
string[] fields;
while ((fields = parser.ReadFields()) != null)
{
yield return fields;
}
}
}
public static IEnumerable<string[]> GetSplittedLines(this TextReader reader, string delimiter)
{
return GetSplittedLines(reader, new[] { delimiter });
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment