Bayesian A/B testing of WhatsApp Messages

1_Bayesian AB Testing WhatsApp Messages.md

Bayesian A/B Testing WhatsApp Messages

eda.py

# Output the total number of conversions
print("\nTotal Conversions:", df["result"].sum())

# Output the overall conversion rate of our sample
print("Overall conversion rate:", df["result"].mean())

# Get the conversion rate by HSM
conversion_rates = [df[f"hsm_template_{i+1}_result"].mean() for i in range(3)]

# Print the conversion rate for each HSM
for i, cr in enumerate(conversion_rates):
    print(f"Conversion rate HSM {i+1}:", cr)

get_message_result.py

# Get the name of the column with the result
df["hsm_assign_name"] = df["hsm_assign_n"].apply(lambda v: f"hsm_template_{v}_result")

# Put all the results in a same column
df["result"] = df.apply(lambda row: row[row["hsm_assign_name"]], axis=1)

model_1.py

import pymc3 as pm

# Get the observed results
obs_hsm_2 = df[df["hsm_assign"] == 2]["result"].values
obs_hsm_3 = df[df["hsm_assign"] == 3]["result"].values

with pm.Model() as model_1:

    # Assume uniform priors for the conversion rates
    p_1 = pm.Uniform("p_1", 0, 1)
    p_2 = pm.Uniform("p_2", 0, 1)

    # Define a deterministic delta function
    delta = pm.Deterministic("delta", p_1 - p_2)

    # Set of observations
    obs_1 = pm.Bernoulli("obs_1", p_1, observed=obs_hsm_2)
    obs_2 = pm.Bernoulli("obs_2", p_2, observed=obs_hsm_3)

    # Start the sampling process
    trace_1 = pm.sample()

template_assignment.py

# Generate a uniform random number from 1 to 3 for each message
df["hsm_assign_n"] = np.random.randint(1, 4, size=len(df))