Brand Messaging experiment power analysis code

brand_power.r

effect_sizes <- (1:200*0.001+1)
av.monthly.impressions <- 23742/2
av.monthly.click_throughs <- 8367/2

baseline_ctr <- av.monthly.click_throughs/av.monthly.impressions
baseline_signup_rate <- 0.0212 * baseline_ctr

n_samples_ctr <- double(length(effect_sizes))
n_samples_signup_rate <- double(length(effect_sizes))

for (i in 1:length(effect_sizes)) {
    e <- effect_sizes[i]
    n_samples_ctr[i] <- power.prop.test(p1=baseline_ctr,p2=e*baseline_ctr,power=0.8,sig.level=0.05)$n
}

for (i in 1:length(effect_sizes)) {
    e <- effect_sizes[i]
    n_samples_signup_rate[i] <- power.prop.test(p1=baseline_signup_rate,p2=e*baseline_signup_rate,
                                        power=0.8,sig.level=0.05)$n
}

plot(effect_sizes,n_samples_ctr/av.monthly.impressions,type="l",lty="dotted", log='y', yaxt='n',
         xlab="Effect Size as multiples of the baseline",ylab="Number of Months to Detect",
        main="Number of Months of Data required for Detection\n(as a function of Effect Size)\nPower: 80%, Confidence: 95%")
lines(effect_sizes,n_samples_signup_rate/av.monthly.impressions,type="l",lty="dashed")

axis(side=2, at=c(0.1,1,10,100,1000), labels=c("0.1","1","10","100","1000"))

legend(1.1,1000,legend=c("Click Through Rate","Signup Conversion Rate"),lty=c("dotted","dashed"))

plot(effect_sizes,30*n_samples_ctr/av.monthly.impressions,type="l",lty="dotted",
     xlim=c(1.05,1.15), ylim=c(0,30),
     xlab="Effect Size as multiples of the baseline",ylab="Number of Days to Detect",
     main="Number of Days of Data required for Detection\n(as a function of Effect Size)\nPower: 80%, Confidence: 95%")
legend(1.1,30,legend=c("Click Through Rate"),lty=c("dotted"))