Table of Contents¶
Part I - Probability¶
In [1]:
import pandas as pd
import numpy as np
import random
import matplotlib.pyplot as plt
from scipy import stats
%matplotlib inline
random.seed(42)
In [2]:
df = pd.read_csv('ab_data.csv')
df.head()
Out[2]:
In [3]:
def proportions_ztest(count, nobs, value=None, alternative='two-sided',
prop_var=False):
count = np.asarray(count)
nobs = np.asarray(nobs)
if nobs.size == 1:
nobs = nobs * np.ones_like(count)
print('nobs.size == 1')
prop = count * 1. / nobs
print('prop: ' + str(prop))
k_sample = np.size(prop)
print('k_sample: ' + str(k_sample))
if value is None:
if k_sample == 1:
raise ValueError('value must be provided for a 1-sample test')
value = 0
if k_sample == 1:
diff = prop - value
print('k_sample == 1 and diff: ' + str(diff))
elif k_sample == 2:
diff = prop[0] - prop[1] - value
print('k_sample == 2 and diff: ' + str(diff))
else:
msg = 'more than two samples are not implemented yet'
raise NotImplementedError(msg)
p_pooled = np.sum(count) * 1. / np.sum(nobs)
print('p_pooled: ' + str(p_pooled))
nobs_fact = np.sum(1. / nobs)
print('nobs_fact: ' + str(nobs_fact))
print('prop_var: ' + str(prop_var))
if prop_var:
p_pooled = prop_var
var_ = p_pooled * (1 - p_pooled) * nobs_fact
print('var_: ' + str(var_))
std_diff = np.sqrt(var_)
print('std_diff: ' + str(std_diff))
return _zstat_generic2(diff, std_diff, alternative)
In [4]:
def _zstat_generic2(value, std_diff, alternative):
'''generic (normal) z-test to save typing
can be used as ztest based on summary statistics
'''
zstat = value / std_diff
print('zstat: ' + str(zstat))
if alternative in ['two-sided', '2-sided', '2s']:
pvalue = stats.norm.sf(np.abs(zstat))*2
print('alternative is two-sided.')
print('pvalue: ' + str(pvalue))
elif alternative in ['larger', 'l']:
pvalue = stats.norm.sf(zstat)
elif alternative in ['smaller', 's']:
pvalue = stats.norm.cdf(zstat)
else:
raise ValueError('invalid alternative')
return zstat, pvalue
In [5]:
df['group'].unique()
Out[5]:
In [6]:
df['landing_page'].unique()
Out[6]:
In [7]:
len(df)
Out[7]:
In [8]:
df['user_id'].nunique()
Out[8]:
In [9]:
sum(df['converted'] == 1) / len(df['converted'])
Out[9]:
In [10]:
treatment_wrong = sum((df['group'] == 'treatment') & (df['landing_page'] != 'new_page'))
control_wrong = sum((df['group'] == 'control') & (df['landing_page'] == 'new_page'))
total_wrong = treatment_wrong + control_wrong
total_wrong
Out[10]:
In [11]:
df.info()
No rows appear to be missing any values.
In [12]:
df2 = df
df2.info()
In [13]:
df2 = df2.loc[~((df2['group'] == 'treatment') & (df2['landing_page'] != 'new_page'))]
df2 = df2.loc[~((df2['group'] == 'control') & (df2['landing_page'] == 'new_page'))]
df2.shape
Out[13]:
In [14]:
df2[((df2['group'] == 'treatment') == (df2['landing_page'] == 'new_page')) == False].shape[0]
Out[14]:
In [15]:
df2['user_id'].nunique()
Out[15]:
In [16]:
df2.loc[df2['user_id'].duplicated()]
Out[16]:
In [17]:
df2.loc[df2['user_id'].duplicated()]
Out[17]:
In [18]:
df2.drop(2893, inplace = True);
In [19]:
df2['user_id'].duplicated().value_counts()
Out[19]:
In [20]:
conv_prob = sum(df2['converted'] == 1) / len(df2['converted'])
conv_prob
Out[20]:
In [21]:
control_group = df2.loc[df2['group'] == 'control']
control_conv_prob = sum(control_group['converted'] == 1) / len(control_group['converted'])
control_conv_prob
Out[21]:
In [22]:
treatment_group = df2.loc[df2['group'] == 'treatment']
treatment_conv_prob = sum(treatment_group['converted'] == 1) / len(treatment_group['converted'])
treatment_conv_prob
Out[22]:
In [23]:
len(treatment_group) / len(df2)
Out[23]:
In [24]:
actual_prob_diff = control_conv_prob - treatment_conv_prob
actual_prob_diff
Out[24]:
There appears to be a 0.00158 higher probability of a conversion from a control page than a conversion from a treatment page in this dataset.
Part II - A/B Test¶
$H_{0}: P_{new} - P_{old} \leq 0$
$H_{1}: P_{new} - P_{old} > 0$
$\alpha: 0.05$
In [25]:
conv_prob
Out[25]:
In [26]:
n_new = len(treatment_group)
n_new
Out[26]:
In [27]:
n_old = len(control_group)
n_old
Out[27]:
In [28]:
new_data_sample = treatment_group.sample(n_new, replace = True)
new_page_converted = new_data_sample['converted'].tolist()
p_new_conv_rate = sum(new_page_converted) / len(new_page_converted)
p_new_conv_rate
Out[28]:
In [29]:
old_data_sample = control_group.sample(n_old)
old_page_converted = old_data_sample['converted'].tolist()
p_old_conv_rate = sum(old_page_converted) / len(old_page_converted)
p_old_conv_rate
Out[29]:
In [30]:
ef_diff = p_new_conv_rate - p_old_conv_rate
ef_diff
Out[30]:
In [31]:
p_diffs = np.empty(0)
for x in range(10000):
new_data_sample = treatment_group.sample(1000, replace = True)
new_page_converted = new_data_sample['converted'].tolist()
new_conv_prob = sum(new_page_converted) / len(new_page_converted)
old_data_sample = control_group.sample(1000, replace = True)
old_page_converted = old_data_sample['converted'].tolist()
old_conv_prob = sum(old_page_converted) / len(old_page_converted)
difference = new_conv_prob - old_conv_prob
p_diffs = np.append(p_diffs, difference)
In [32]:
plt.hist(p_diffs)
Out[32]:
In [33]:
greater = [i for i in p_diffs if i > actual_prob_diff]
print('len(greater): ' + str(len(greater)))
print('len(p_diffs): ' + str(len(p_diffs)))
greater_prop = len(greater) / len(p_diffs)
greater_prop
Out[33]:
The p-value is the probability of observing the statistic, or one that is more extreme in favor of the alternative, if the null hypothesis is true. That means that if the p-value is smaller than $\alpha$, then the alternative hypothesis is more likely, and if the p-value is larger than $\alpha$, then the null hypothesis is more likely.
In [34]:
import statsmodels.api as sm
convert_old = sum(control_group['converted'] == 1)
convert_new = sum(treatment_group['converted'] == 1)
n_old = n_old
n_new = n_new
conv_diff = (convert_old / n_old) - (convert_new / n_new)
num_diff = n_new - n_old
conv_diff, num_diff
Out[34]:
In [35]:
z_score, p_value = proportions_ztest([convert_new, convert_old], [n_new, n_old])
z_score, p_value
Out[35]:
The p-value is greater than the alpha of 0.05, so we fail to reject the null hypothesis according to this.
This does not agree with the findings of the earlier findings.
Part III - A Regression Approach¶
In [36]:
df2['intercept'] = 1
df2['ab_page'] = 0
df2.loc[df2['group'] == 'treatment', 'ab_page'] = 1
In [37]:
df2.head(20)
Out[37]:
In [38]:
log_mod = sm.Logit(df2['converted'], df2[['intercept', 'ab_page']])
results = log_mod.fit()
In [39]:
results.summary()
Out[39]:
P-value: 0.190 It appears to be rounded to the thousandth place.
In [40]:
countries_df = pd.read_csv('countries.csv')
countries_df.head()
Out[40]:
In [41]:
df3 = pd.DataFrame.join(df2, countries_df['country'])
df3.head()
Out[41]:
In [42]:
df3['country'].value_counts()
Out[42]:
In [43]:
df3[['CA','UK', 'US']] = pd.get_dummies(df3['country'])
df3 = df3.drop('US', axis=1)
df3.head()
Out[43]:
In [44]:
log_mod2 = sm.Logit(df3['converted'], df3[['intercept', 'ab_page', 'CA', 'UK']])
results2 = log_mod2.fit()
In [45]:
results2.summary()
Out[45]:
Given that the country p-values are higher than 0.05, the country does not appear to have an impact of conversion.