python3 — панды определяют, являются ли события статистически значимыми

Загрузить пакеты, установить глобальные переменные, создать данные.

import scipy.stats as stats
import numpy as np

n = 60
stat_sig_thresh = 0.05

event_perc = pd.DataFrame({"event occurs": np.random.choice([True,False],n),
                          "percent change": [i*.1 for i in np.random.randint(1,1000,n)]})

Определите, является ли распределение нормальным

stat_sig = event_perc.groupby("event occurs").apply(lambda x: stats.normaltest(x))
stat_sig = pd.DataFrame(stat_sig)
stat_sig = pd.DataFrame(stat_sig[0].values.tolist(), index=stat_sig.index).reset_index()
stat_sig.loc[(stat_sig.pvalue <= stat_sig_thresh), "Normal"] = False
stat_sig["Normal"].fillna("True",inplace=True)

>>>stat_sig

    event occurs  statistic             pvalue                  Normal
0   False         [2.9171920993203915]  [0.23256255191146755]   True
1   True          [2.938332679486047]   [0.23011724484588764]   True

Определить статистическую значимость

normal = [bool(i) for i in stat_sig.Normal.unique().tolist()]

rvs1 = event_perc["percent change"][event_perc["event occurs"] == True]
rvs2 = event_perc["percent change"][event_perc["event occurs"] == False]

if (len(normal) == 1) & (normal[0] == True):
    print("the distributions are normal")
    if stats.ttest_ind(rvs1,rvs2).pvalue >= stat_sig_thresh:
        # we cannot reject the null hypothesis of identical average scores
        print("we can't say whether there is statistically significant difference")
    else:
        # we reject the null hypothesis of equal averages
        print("there is a statisically significant difference")

elif (len(normal) == 1) & (normal[0] == False):
    print("the distributions are not normal")
    if stats.wilcoxon(rvs1,rvs2).pvalue >= stat_sig_thresh:
        # we cannot reject the null hypothesis of identical average scores
        print("we can't say whether there is statistically significant difference")
    else:
        # we reject the null hypothesis of equal averages
        print("there is a statisically significant difference")
else:
    print("samples are drawn from different distributions")

the distributions are normal
we can't say whether there is statistically significant difference

Спасибо @DarkDrassher34 и @ChrisDanger. Я собрал этот пример кода из различных источников, первоначально из ответа Дарка, а затем просмотрел его после сообщения Криса. Мысли?

corr_data = df[['event occurs','percent change']]
cat1 = corr_data[corr_data['event occurs']==True]
cat2 = corr_data[corr_data['event occurs']==False]


#----------------------
# is the sample normal / gaussian
#----------------------
from scipy.stats import shapiro # test for normalcy in small samples
from scipy.stats import normaltest

if (len(cat1['percent change'].index) <= 20 ):
    stat1, p1 = shapiro(cat1['percent change'])
else:
    stat1, p1 = normaltest(cat1['percent change'])

if (len(cat2['percent change'].index) <= 20 ):
    stat2, p2 = shapiro(cat2['percent change'])
else:
    stat2, p2 = normaltest(cat2['percent change'])


alpha = 0.05 # stat threshold
# both groups are normal
if ((p1 > alpha) and (p2 > alpha)):
    print('Samples looks Gaussian (fail to reject H0)')

    #----------------------
    # if normal / gaussian run these tests
    #----------------------
    from scipy.stats import ttest_ind
    stat, p = ttest_ind(cat1['percent change'], cat2['percent change'])
    print('Statistics=%.3f, p=%.3f' % (stat, p))
    if p > alpha:
        print('Same distribution (fail to reject H0)')
    else:
        print('Different distribution (reject H0)')


else:
    print('Samples do not look Gaussian (reject H0)')
    #----------------------
    # if not normal / gaussian run these tests
    #----------------------
    from scipy.stats import mannwhitneyu
    stat, p = mannwhitneyu(cat1['percent change'], cat2['percent change'])
    print('Statistics=%.3f, p=%.3f' % (stat, p))
    if p > alpha:
        print('Same distribution (fail to reject H0)')
    else:
        print('Different distribution (reject H0)')