Skip to content

Instantly share code, notes, and snippets.

@mw2q
Created January 3, 2020 00:28
Show Gist options
  • Save mw2q/380129282504682629e1c458a43ac5f4 to your computer and use it in GitHub Desktop.
Save mw2q/380129282504682629e1c458a43ac5f4 to your computer and use it in GitHub Desktop.
post processing dbt2 mix.log with julia and data frames
#!/usr/bin/env julia
using CSV
using DataFrames
using Printf
using Statistics
function displayreport(df::DataFrame)
errors = count(y -> (y .== "E"), skipmissing(df.code))
StartTime = maximum(df[df.transaction .== "START", :ctime])
TerminatedTime = minimum(df[df.transaction .== "TERMINATED", :ctime])
SteadyStateTime = TerminatedTime - StartTime
ctime0 = df[[1], [1]].ctime[1]
RampupTime = StartTime - ctime0
df = df[(df.ctime .> StartTime) .& (df.ctime .< TerminatedTime),
[:transaction, :code, :response_time]]
TotalTransactions = nrow(df)
summary = by(df,
[:transaction, :code],
length = :transaction => length,
AverageResponseTime = :response_time => mean,
Decile9 = :response_time => x -> quantile(x, .90)
)
TransactionNames = Dict(
"d" => "Delivery",
"n" => "New Order",
"o" => "Order Status",
"p" => "Payment",
"s" => "Stock Level"
)
@printf(" Response Time (s)\n")
@printf(" Transaction %% Average : 90th %% Total")
@printf(" Rollbacks %%\n")
@printf("------------ ----- --------------------- ------------- ")
@printf("------------- -----\n")
rollbacks = 0
for i in ["d", "n", "o", "p", "s"]
row = summary[(summary.transaction .== i) .& (summary.code .== "C"), :]
try
rollbacks = summary[(summary.transaction .== i) .&
(summary.code .== "R"), :].length[1]
catch
rollbacks = 0
end
@printf("%12s %5.2f %9.3f : %9.3f %13d %13d %5.2f\n",
TransactionNames[i],
(row.length[1] + rollbacks) / TotalTransactions * 100,
row.AverageResponseTime[1], row.Decile9[1], row.length[1],
rollbacks, rollbacks / (row.length[1] + rollbacks) * 100)
end
@printf("------------ ----- --------------------- ------------- ")
@printf("------------- -----\n")
@printf("%0.2f new-order transactions per minute (NOTPM)\n",
summary[summary.transaction .== "n", :].length[1] /
(SteadyStateTime / 60))
@printf("%0.1f minute duration\n", SteadyStateTime / 60)
@printf("%d total unknown errors\n", errors)
@printf("%0.1f seconds(s) ramping up\n", RampupTime)
end
mixfile = ARGS[1]
df = CSV.read(mixfile, header = ["ctime", "transaction", "code",
"response_time", "pthread"])
displayreport(df)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment