Hypothesis Testing in Python – t-tests, ANOVA & Chi-Square 2026

Hypothesis testing is the statistical framework for making data-driven decisions. Is the difference between two groups real or just noise? Did the product change actually improve conversion? Is this feature correlated with the outcome? Hypothesis tests give you mathematically rigorous answers. This guide covers every test a data scientist needs, with Python code and plain-English interpretation.

The Framework: Null and Alternative Hypotheses

Every hypothesis test starts with two hypotheses. The null hypothesis (H₀) is the “nothing happening” baseline — no difference, no effect, no relationship. The alternative hypothesis (H₁) is what you want to prove. You never “prove” H₁ directly; instead, you calculate how unlikely the observed data would be if H₀ were true. If that probability (the p-value) is below a threshold (α, usually 0.05), you reject H₀. A p-value of 0.03 means “if H₀ is true, there is only a 3% chance of seeing data this extreme.”

One-Sample t-test

import numpy as np
from scipy import stats

# Question: Is the average order value significantly different from ₹500?
order_values = np.array([520, 485, 610, 490, 530, 515, 480, 605, 495, 525,
                          540, 475, 560, 510, 498, 623, 487, 512, 534, 501])

t_stat, p_value = stats.ttest_1samp(order_values, popmean=500)
ci = stats.t.interval(0.95, df=len(order_values)-1,
                       loc=np.mean(order_values),
                       scale=stats.sem(order_values))

print(f'Sample mean:  ₹{np.mean(order_values):.2f}')
print(f't-statistic:  {t_stat:.4f}')
print(f'p-value:      {p_value:.4f}')
print(f'95% CI:       ₹{ci[0]:.2f} to ₹{ci[1]:.2f}')

if p_value < 0.05:
    print('✅ Reject H₀ — mean is significantly different from ₹500')
else:
    print('❌ Fail to reject H₀ — no significant difference from ₹500')

Two-Sample t-test (A/B Test)

import numpy as np
from scipy import stats

np.random.seed(42)
# Control: mean=100, Treatment: mean=108
control   = np.random.normal(loc=100, scale=15, size=200)
treatment = np.random.normal(loc=108, scale=15, size=200)

# Test normality first (Shapiro-Wilk)
_, p_norm_ctrl = stats.shapiro(control[:50])   # Shapiro works best on n<50
_, p_norm_trt  = stats.shapiro(treatment[:50])
print(f'Normality p-values: control={p_norm_ctrl:.3f}, treatment={p_norm_trt:.3f}')

# Levene test for equal variance
_, p_var = stats.levene(control, treatment)
equal_var = p_var > 0.05
print(f'Equal variance: {equal_var} (p={p_var:.3f})')

# Independent samples t-test
t_stat, p_value = stats.ttest_ind(control, treatment,
                                   equal_var=equal_var)
effect_size = (treatment.mean() - control.mean()) /               np.sqrt((control.std()**2 + treatment.std()**2) / 2)

print(f'
Control mean:   {control.mean():.2f}')
print(f'Treatment mean: {treatment.mean():.2f}')
print(f'Lift:           {treatment.mean() - control.mean():.2f} '
      f'({(treatment.mean()/control.mean()-1)*100:.1f}%)')
print(f't-statistic:    {t_stat:.4f}')
print(f'p-value:        {p_value:.4f}')
print(f"Cohen's d:      {effect_size:.4f}")

Paired t-test

# Use when same subjects are measured twice (before/after)
before = np.array([72, 68, 75, 80, 65, 71, 69, 77, 73, 70])
after  = np.array([68, 65, 70, 76, 61, 67, 66, 72, 69, 66])

t_stat, p_value = stats.ttest_rel(before, after)
mean_diff = np.mean(after - before)
print(f'Mean difference: {mean_diff:.2f}')
print(f'p-value: {p_value:.4f}')

One-Way ANOVA (3+ Groups)

# Question: Do revenue means differ across 4 regions?
north = np.random.normal(500, 50, 100)
south = np.random.normal(520, 55, 100)
east  = np.random.normal(490, 45, 100)
west  = np.random.normal(510, 50, 100)

f_stat, p_value = stats.f_oneway(north, south, east, west)
print(f'F-statistic: {f_stat:.4f}')
print(f'p-value:     {p_value:.4f}')

if p_value < 0.05:
    print('✅ At least one region differs significantly')

    # Post-hoc: Tukey HSD to identify which pairs differ
    from statsmodels.stats.multicomp import pairwise_tukeyhsd
    import pandas as pd

    data   = np.concatenate([north, south, east, west])
    groups = ['North']*100 + ['South']*100 + ['East']*100 + ['West']*100

    tukey = pairwise_tukeyhsd(data, groups, alpha=0.05)
    print(tukey.summary())

Chi-Square Test (Categorical Variables)

import pandas as pd
from scipy.stats import chi2_contingency

# Question: Is device type associated with purchase behaviour?
contingency = pd.DataFrame({
    'Purchased':     [320, 180, 80],
    'Not Purchased': [480, 420, 120]
}, index=['Mobile', 'Desktop', 'Tablet'])

print(contingency)

chi2, p_value, dof, expected = chi2_contingency(contingency)

print(f'
Chi-square: {chi2:.4f}')
print(f'p-value:    {p_value:.4f}')
print(f'DOF:        {dof}')

# Cramér's V — effect size for chi-square
n = contingency.values.sum()
cramer_v = np.sqrt(chi2 / (n * (min(contingency.shape) - 1)))
print(f"Cramér's V: {cramer_v:.4f} "
      f"({'strong' if cramer_v > 0.3 else 'moderate' if cramer_v > 0.1 else 'weak'} association)")

Non-Parametric Tests

# Mann-Whitney U — non-parametric alternative to t-test (no normality assumption)
stat, p = stats.mannwhitneyu(control, treatment, alternative='two-sided')
print(f'Mann-Whitney: stat={stat:.0f}, p={p:.4f}')

# Kruskal-Wallis — non-parametric alternative to ANOVA
stat, p = stats.kruskal(north, south, east, west)
print(f'Kruskal-Wallis: stat={stat:.4f}, p={p:.4f}')

# Wilcoxon signed-rank — non-parametric paired test
stat, p = stats.wilcoxon(before, after)
print(f'Wilcoxon: stat={stat:.0f}, p={p:.4f}')

Multiple Testing Correction

from statsmodels.stats.multitest import multipletests

# Running 20 t-tests — expect ~1 false positive at α=0.05 by chance alone
p_values = [0.04, 0.001, 0.08, 0.03, 0.7, 0.002, 0.04, 0.9,
            0.03, 0.05, 0.6, 0.01, 0.04, 0.8, 0.002, 0.03,
            0.5, 0.04, 0.1, 0.03]

# Bonferroni (conservative — multiply p by n_tests)
reject_bon, p_bon, _, _ = multipletests(p_values, alpha=0.05,
                                         method='bonferroni')

# Benjamini-Hochberg FDR (less conservative, controls false discovery rate)
reject_bh,  p_bh,  _, _ = multipletests(p_values, alpha=0.05,
                                          method='fdr_bh')

print(f'Uncorrected rejections: {sum(p < 0.05 for p in p_values)}')
print(f'Bonferroni rejections:  {sum(reject_bon)}')
print(f'BH FDR rejections:      {sum(reject_bh)}')

Conclusion

Choose your test based on the data type and number of groups: t-test for two numeric groups, ANOVA for three or more, chi-square for categorical associations. Always check assumptions (normality, equal variance) and switch to non-parametric alternatives when they are violated. Correct for multiple comparisons whenever you run more than one test — the Benjamini-Hochberg procedure is the practical default. And remember: statistical significance is not business significance — a tiny p-value with a tiny effect size may not justify any action.