Statistical Learning Theory – Bias-Variance, Overfitting & Regularisation 2026

Understanding why models fail is as important as building models that work. The bias-variance tradeoff is the central tension in machine learning: a model that is too simple (high bias) misses real patterns; a model that is too complex (high variance) memorises noise. Regularisation is the toolkit for navigating this tradeoff. This guide builds genuine intuition for these concepts with Python code and visualisations.

The Bias-Variance Decomposition

The expected prediction error of any model decomposes into three parts: Bias² (error from wrong assumptions — how far the model’s average prediction is from the truth), Variance (error from sensitivity to training data fluctuations — how much predictions vary across different training sets), and Irreducible Noise (the randomness in the data itself, which no model can remove). Total error = Bias² + Variance + Noise. You cannot reduce all three simultaneously — reducing bias (using a more complex model) typically increases variance, and vice versa.

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline

np.random.seed(42)

def true_function(x):
    return np.sin(2 * np.pi * x)

def generate_data(n=20, noise=0.3):
    x = np.sort(np.random.uniform(0, 1, n))
    y = true_function(x) + np.random.normal(0, noise, n)
    return x, y

x_test = np.linspace(0, 1, 200)

fig, axes = plt.subplots(1, 3, figsize=(15, 5))
degrees = [1, 4, 15]   # underfit, good, overfit

for ax, degree in zip(axes, degrees):
    all_preds = []
    for _ in range(50):
        x_tr, y_tr = generate_data()
        model = make_pipeline(PolynomialFeatures(degree), LinearRegression())
        model.fit(x_tr.reshape(-1, 1), y_tr)
        all_preds.append(model.predict(x_test.reshape(-1, 1)))

    all_preds = np.array(all_preds)
    mean_pred = all_preds.mean(axis=0)
    ax.plot(x_test, true_function(x_test), 'k-', lw=2, label='True')
    for pred in all_preds:
        ax.plot(x_test, pred, 'b-', alpha=0.1)
    ax.plot(x_test, mean_pred, 'r-', lw=2, label='Mean prediction')
    bias    = np.mean((mean_pred - true_function(x_test))**2)
    variance = np.mean(all_preds.var(axis=0))
    ax.set_title(f'Degree {degree}
Bias²={bias:.3f} Var={variance:.3f}')
    ax.legend()

plt.tight_layout()
plt.show()

Diagnosing Overfitting with Learning Curves

from sklearn.model_selection import learning_curve
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import numpy as np
import matplotlib.pyplot as plt

X, y = make_classification(n_samples=1000, n_features=20,
                             n_informative=10, random_state=42)

model = RandomForestClassifier(n_estimators=100, random_state=42)

train_sizes, train_scores, val_scores = learning_curve(
    model, X, y,
    train_sizes=np.linspace(0.1, 1.0, 10),
    cv=5, scoring='accuracy', n_jobs=-1)

train_mean = train_scores.mean(axis=1)
val_mean   = val_scores.mean(axis=1)
train_std  = train_scores.std(axis=1)
val_std    = val_scores.std(axis=1)

plt.figure(figsize=(8, 5))
plt.plot(train_sizes, train_mean, 'o-', label='Train accuracy')
plt.plot(train_sizes, val_mean,   'o-', label='Val accuracy')
plt.fill_between(train_sizes, train_mean - train_std,
                 train_mean + train_std, alpha=0.15)
plt.fill_between(train_sizes, val_mean - val_std,
                 val_mean + val_std, alpha=0.15)
plt.xlabel('Training set size')
plt.ylabel('Accuracy')
plt.title('Learning Curve')
plt.legend()
plt.grid(True)
plt.show()
# Large gap between train and val = overfitting
# Both curves low = underfitting
# Curves converging high = good fit

L1 and L2 Regularisation

from sklearn.linear_model import Ridge, Lasso, ElasticNet
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
import numpy as np

# L2 (Ridge) — penalises sum of squared coefficients
# Shrinks all coefficients toward zero but rarely to exactly zero
ridge = Pipeline([('scaler', StandardScaler()),
                  ('model', Ridge(alpha=1.0))])

# L1 (Lasso) — penalises sum of absolute coefficients
# Drives some coefficients to exactly zero → automatic feature selection
lasso = Pipeline([('scaler', StandardScaler()),
                  ('model', Lasso(alpha=0.01))])

# ElasticNet — combines L1 and L2
enet  = Pipeline([('scaler', StandardScaler()),
                  ('model', ElasticNet(alpha=0.01, l1_ratio=0.5))])

for name, pipe in [('Ridge', ridge), ('Lasso', lasso), ('ElasticNet', enet)]:
    scores = cross_val_score(pipe, X, y, cv=5, scoring='accuracy')
    print(f'{name}: {scores.mean():.4f} ± {scores.std():.4f}')

# Choose alpha with cross-validation
from sklearn.linear_model import RidgeCV, LassoCV
ridge_cv = RidgeCV(alphas=[0.01, 0.1, 1.0, 10.0, 100.0], cv=5)
ridge_cv.fit(X, y)
print(f'Best alpha: {ridge_cv.alpha_}')

Dropout Regularisation (Neural Networks)

import torch
import torch.nn as nn

class RegularisedNet(nn.Module):
    def __init__(self, input_dim, dropout_rate=0.3, weight_decay=1e-4):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, 256),
            nn.BatchNorm1d(256),
            nn.ReLU(),
            nn.Dropout(dropout_rate),      # randomly zero 30% of activations
            nn.Linear(256, 128),
            nn.BatchNorm1d(128),
            nn.ReLU(),
            nn.Dropout(dropout_rate),
            nn.Linear(128, 1)
        )

    def forward(self, x):
        return self.net(x)

model = RegularisedNet(input_dim=20, dropout_rate=0.4)

# Pass weight_decay to optimizer for L2 regularisation on all weights
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)

# During inference: disable dropout
model.eval()   # dropout inactive
# During training: enable dropout
model.train()  # dropout active

Early Stopping

class EarlyStopping:
    def __init__(self, patience=10, min_delta=0.001):
        self.patience  = patience
        self.min_delta = min_delta
        self.counter   = 0
        self.best_loss = None
        self.stop      = False

    def __call__(self, val_loss, model):
        if self.best_loss is None:
            self.best_loss = val_loss
            torch.save(model.state_dict(), 'checkpoint.pt')
        elif val_loss < self.best_loss - self.min_delta:
            self.best_loss = val_loss
            torch.save(model.state_dict(), 'checkpoint.pt')
            self.counter = 0
        else:
            self.counter += 1
            if self.counter >= self.patience:
                self.stop = True

es = EarlyStopping(patience=15, min_delta=0.001)
for epoch in range(500):
    train_loss = train_epoch(model, train_loader, optimizer)
    val_loss   = evaluate(model, val_loader)
    es(val_loss, model)
    if es.stop:
        print(f'Early stopping at epoch {epoch}')
        break

model.load_state_dict(torch.load('checkpoint.pt'))

Cross-Validation Strategies

from sklearn.model_selection import (KFold, StratifiedKFold,
                                      TimeSeriesSplit, cross_val_score)

# Standard K-fold (regression)
kf  = KFold(n_splits=5, shuffle=True, random_state=42)

# Stratified K-fold (classification — preserves class ratios)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Time series split — no data leakage from future
tss = TimeSeriesSplit(n_splits=5)

model = RandomForestClassifier(n_estimators=100, random_state=42)
for cv_name, cv in [('KFold', kf), ('Stratified', skf)]:
    scores = cross_val_score(model, X, y, cv=cv, scoring='roc_auc')
    print(f'{cv_name}: {scores.mean():.4f} ± {scores.std():.4f}')

Conclusion

The bias-variance tradeoff is not a problem to solve — it is a reality to navigate. Underfitting models need more capacity or better features. Overfitting models need regularisation: L1 for feature selection, L2 for shrinkage, dropout for neural networks, and early stopping everywhere. Learning curves tell you which regime you are in. Cross-validation tells you how well your regularisation generalises. Together, these diagnostic tools and techniques are the foundation of building models that work not just on training data, but in production.