Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
Convert multiple storage analytical logs into a single csv file for easy analysis
# NOTE : This sample source code is provided "AS IS", with no warranties of any sort.
# Save as XLog2CSV.ps1 and execute as follows
# XLog2CSV.ps1 -inputFolder <input_folder_path> -outputFile <full_path_and_filename_of_csv>
# Eg:-
# XLog2CSV.ps1 -inputFolder c:\cases\Logs\2015\08\*.log -outputFile c:\cases\Logs\all_logs.csv
# 2019-03-26 : ajithkr : Update column list to account for v2 logging
# 2015-08-21 : ajithkr : authored
[ValidateScript ({Test-Path $_})][Parameter (Mandatory=$True,Position=1)] [string] $inputFolder,
[ValidateScript ({-not(Test-Path $_ -Pathtype Container)})][Parameter (Mandatory=$True,Position=2)] [string] $outputFile
# Perf measurement (uncomment below if needed)
#Measure-Command {
try {
# Clean up the input variables
# Strip off any leading ..\ from the file
$outputfile = $outputfile -ireplace "^\.\.\\",((Split-Path (Get-Location)).ToString()+"\")
# Strip off any leading .\ from the file
$outputfile = $outputfile -ireplace "^\.\\",((Get-Location).ToString()+"\")
# Account for variable paths
if (!([System.IO.Path]::IsPathRooted($outputFile) ))
$outputFile = ((Get-Location).ToString() + "\" + $outputFile) -ireplace "\\\\","\"
# Create the output file
$writer = [System.IO.StreamWriter] $outputFile
# Write out the log header
$writer.WriteLine('"Log Version","Transaction Start Time","REST Operation Type","Request Status","HTTP Status Code","E2E Latency","Server Latency","Authentication type","Requestor Account Name","Owner Account Name","Service Type","Request URL","Object Key","Request ID","Operation Number","Client IP","Request Version","Request Header Size","Request Packet Size","Response Header Size","Response Packet Size","Request Content Length","Request MD5","Server MD5","ETag","Last Modified Time","ConditionsUsed","User Agent","Referrer","Client Request ID","User Object ID","Tenant ID","Application ID","Audience","Issuer","User Principal Name","Unused Field","Authorization Detail"')
#Process each file - excluding the output file (in case the output file happens to be inside the input folder)
#TBD Add safety check for the file type.
foreach ($inputFile in (Get-ChildItem -Path $inputFolder -File -Recurse| Where-Object {$_.FullName -ne $outputFile}))
$reader = [System.IO.File]::OpenText($srcFile)
Write-Host "Processing File : ",$srcFile -NoNewline
$execTime=Measure-Command {
try {
for(;;) {
$line = $reader.ReadLine()
if ($line -eq $null) { break }
# process the line
$line = $line -creplace '^([^"])','"$1' -creplace ";(?=(?:[^`"]*`"[^`"]*`")*[^`"]*$)",'","' -creplace '([^,])"",','$1",' -creplace ',""([^,])',',"$1' -creplace ',"$',',""'
# write it out
finally {
Write-Host " => Completed in ", $execTime.ToString("hh\:mm\:ss")
}finally {
# Perf measurement
Copy link

ajith-k commented Mar 26, 2019

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment