Effect size when you first "check if it's real"

testingforreal.jl

using Distributions, HypothesisTests, DataFrames, DataFramesMeta, StatsPlots

function dowork(effsizes,reps)
    res = typeof((effsize=1.0,meanval=1.0,pval=0.1))[]
    for s in effsizes
        for r in 1:reps
            sam = rand(Normal(s,1.0),15)
            tt = OneSampleTTest(sam)
            push!(res,(effsize=s,meanval=mean(sam),pval=pvalue(tt)))
        end
    end
    res
end

results = DataFrame(dowork([0.05,0.1,0.2,0.5,0.8,1.0,1.5,2.0],100))


realeffects = @subset(results,:pval .< 0.05)

@df realeffects histogram(:meanval./:effsize; label="Effect Estimate/Reality",normalize=true, title="Histogram of the Relative Effect Size Estimate\nfor \"real\" effects")
p = plot!([1.0,1.0],[0.0,3.0]; color=:red,label="1.0")
display(p)

p = @df realeffects scatter(:effsize, :meanval ./ :effsize; title="Relative Effect Size\nAs Function of Real size")

display(p)