Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Convert multiple storage analytical logs into a single csv file for easy analysis
# NOTE : This sample source code is provided "AS IS", with no warranties of any sort.
# Save as XLog2CSV.ps1 and execute as follows
# XLog2CSV.ps1 -inputFolder <input_folder_path> -outputFile <full_path_and_filename_of_csv>
# Eg:-
# XLog2CSV.ps1 -inputFolder c:\cases\Logs\2015\08\*.log -outputFile c:\cases\Logs\all_logs.csv
#
# 2019-03-26 : ajithkr : Update column list to account for v2 logging
# 2015-08-21 : ajithkr : authored
#
[CmdletBinding()]
Param(
[ValidateScript ({Test-Path $_})][Parameter (Mandatory=$True,Position=1)] [string] $inputFolder,
[ValidateScript ({-not(Test-Path $_ -Pathtype Container)})][Parameter (Mandatory=$True,Position=2)] [string] $outputFile
)
# Perf measurement (uncomment below if needed)
#Measure-Command {
#
try {
# Clean up the input variables
# Strip off any leading ..\ from the file
$outputfile = $outputfile -ireplace "^\.\.\\",((Split-Path (Get-Location)).ToString()+"\")
# Strip off any leading .\ from the file
$outputfile = $outputfile -ireplace "^\.\\",((Get-Location).ToString()+"\")
# Account for variable paths
if (!([System.IO.Path]::IsPathRooted($outputFile) ))
{
$outputFile = ((Get-Location).ToString() + "\" + $outputFile) -ireplace "\\\\","\"
}
# Create the output file
$writer = [System.IO.StreamWriter] $outputFile
# Write out the log header
$writer.WriteLine('"Log Version","Transaction Start Time","REST Operation Type","Request Status","HTTP Status Code","E2E Latency","Server Latency","Authentication type","Requestor Account Name","Owner Account Name","Service Type","Request URL","Object Key","Request ID","Operation Number","Client IP","Request Version","Request Header Size","Request Packet Size","Response Header Size","Response Packet Size","Request Content Length","Request MD5","Server MD5","ETag","Last Modified Time","ConditionsUsed","User Agent","Referrer","Client Request ID","User Object ID","Tenant ID","Application ID","Audience","Issuer","User Principal Name","Unused Field","Authorization Detail"')
#Process each file - excluding the output file (in case the output file happens to be inside the input folder)
#TBD Add safety check for the file type.
foreach ($inputFile in (Get-ChildItem -Path $inputFolder -File -Recurse| Where-Object {$_.FullName -ne $outputFile}))
{
$srcFile=$inputFile.FullName
$reader = [System.IO.File]::OpenText($srcFile)
Write-Host "Processing File : ",$srcFile -NoNewline
$execTime=Measure-Command {
try {
for(;;) {
$line = $reader.ReadLine()
if ($line -eq $null) { break }
# process the line
$line = $line -creplace '^([^"])','"$1' -creplace ";(?=(?:[^`"]*`"[^`"]*`")*[^`"]*$)",'","' -creplace '([^,])"",','$1",' -creplace ',""([^,])',',"$1' -creplace ',"$',',""'
# write it out
$writer.WriteLine($line)
}
}
finally {
$reader.Close()
$writer.Flush()
}
}
Write-Host " => Completed in ", $execTime.ToString("hh\:mm\:ss")
}
}finally {
$writer.Close()
}
#}
# Perf measurement
@ajith-k
Copy link
Author

ajith-k commented Mar 26, 2019

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment