Data Cleaning in Python – Handling Missing Values, Outliers & Duplicates 2026

Data scientists spend 60-80% of their time cleaning data. Garbage in, garbage out — a model trained on dirty data will produce confident wrong predictions. This guide covers every data cleaning technique you need, with practical pandas code for handling the messy real-world data that appears in every data science project.

Getting a Data Quality Overview

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

df = pd.read_csv('raw_data.csv')

# Quick overview
print(df.shape)
print(df.dtypes)
print(df.head())

# Missing value audit
missing = pd.DataFrame({
    'count':   df.isnull().sum(),
    'percent': df.isnull().mean() * 100
}).sort_values('percent', ascending=False)
print(missing[missing['count'] > 0])

# Unique value counts (spot constant / near-constant columns)
for col in df.columns:
    n_unique = df[col].nunique()
    print(f'{col}: {n_unique} unique values')

# Basic stats including min/max to spot impossible values
print(df.describe(include='all'))

Handling Missing Values

# ── Strategy depends on: why is data missing? ───────────────

# 1. Drop rows with missing target variable (never impute the target)
df = df.dropna(subset=['target'])

# 2. Drop columns with >50% missing (usually not recoverable)
threshold = 0.5
df = df.loc[:, df.isnull().mean() < threshold]

# 3. Simple imputation for numeric columns
from sklearn.impute import SimpleImputer
num_cols = df.select_dtypes(include='number').columns.tolist()

# Mean imputation (use when data is roughly normal, no outliers)
df[num_cols] = df[num_cols].fillna(df[num_cols].mean())

# Median imputation (use when data is skewed or has outliers)
df[num_cols] = df[num_cols].fillna(df[num_cols].median())

# 4. Mode imputation for categorical
cat_cols = df.select_dtypes(include='object').columns.tolist()
df[cat_cols] = df[cat_cols].fillna(df[cat_cols].mode().iloc[0])

# 5. Forward/backward fill for time series
df = df.sort_values('date')
df[num_cols] = df[num_cols].fillna(method='ffill').fillna(method='bfill')

# 6. KNN imputation (best for numeric, preserves correlations)
from sklearn.impute import KNNImputer
imputer    = KNNImputer(n_neighbors=5)
df[num_cols] = imputer.fit_transform(df[num_cols])

# 7. Add missingness indicator (can be predictive)
for col in df.columns:
    if df[col].isnull().any():
        df[f'{col}_was_missing'] = df[col].isnull().astype(int)

Fixing Data Types

# Dates stored as strings
df['date']          = pd.to_datetime(df['date'], errors='coerce')
df['created_at']    = pd.to_datetime(df['created_at'], format='%Y-%m-%d %H:%M:%S')

# Numbers stored as strings (commas, currency symbols)
df['revenue'] = df['revenue'].astype(str)                               .str.replace(',', '')                               .str.replace('$', '')                               .str.strip()
df['revenue'] = pd.to_numeric(df['revenue'], errors='coerce')

# Boolean stored as Yes/No strings
df['is_active'] = df['is_active'].map({'Yes': True, 'No': False,
                                       'yes': True, 'no': False,
                                       '1': True, '0': False})

# Categorical — saves memory, speeds groupby
df['category'] = df['category'].astype('category')

# Check memory before/after
print(f'Before: {df.memory_usage(deep=True).sum() / 1e6:.1f} MB')
df = df.infer_objects()
print(f'After:  {df.memory_usage(deep=True).sum() / 1e6:.1f} MB')

Detecting and Treating Outliers

# ── Method 1: IQR (Interquartile Range) ──────────────────────
def iqr_bounds(series, factor=1.5):
    Q1, Q3 = series.quantile([0.25, 0.75])
    IQR    = Q3 - Q1
    return Q1 - factor * IQR, Q3 + factor * IQR

for col in num_cols:
    lo, hi = iqr_bounds(df[col])
    n_out  = ((df[col] < lo) | (df[col] > hi)).sum()
    print(f'{col}: {n_out} outliers ({n_out/len(df):.1%})')

# Remove outliers (use when they are data errors)
for col in num_cols:
    lo, hi = iqr_bounds(df[col])
    df = df[(df[col] >= lo) & (df[col] <= hi)]

# Clip outliers (use when they are real but extreme)
for col in num_cols:
    lo, hi = iqr_bounds(df[col], factor=3)
    df[col] = df[col].clip(lo, hi)

# ── Method 2: Z-score ────────────────────────────────────────
from scipy import stats
z_scores    = np.abs(stats.zscore(df[num_cols].dropna()))
df_no_out   = df[(z_scores < 3).all(axis=1)]

# ── Method 3: Isolation Forest (multivariate outliers) ───────
from sklearn.ensemble import IsolationForest
iso = IsolationForest(contamination=0.05, random_state=42)
df['is_outlier'] = iso.fit_predict(df[num_cols].fillna(0))
df_clean = df[df['is_outlier'] == 1].drop('is_outlier', axis=1)
print(f'Removed {(df["is_outlier"] == -1).sum()} multivariate outliers')

Removing Duplicates

# Exact duplicates
print(f'Exact duplicates: {df.duplicated().sum()}')
df = df.drop_duplicates()

# Duplicates on key columns (keep most recent)
df = df.sort_values('updated_at', ascending=False)
df = df.drop_duplicates(subset=['user_id', 'product_id'], keep='first')

# Fuzzy duplicates in text (company names etc.)
from thefuzz import fuzz, process

names   = df['company'].unique().tolist()
matches = []
for name in names:
    similar = process.extractBests(name, names, scorer=fuzz.token_sort_ratio,
                                   score_cutoff=90, limit=5)
    if len(similar) > 1:
        matches.append({'name': name, 'similar': similar})

for m in matches[:5]:
    print(f'{m["name"]}: {m["similar"]}')

Standardising Text and Categories

# Clean string columns
df['name'] = (df['name']
    .str.strip()
    .str.lower()
    .str.replace(r'\s+', ' ', regex=True)
    .str.title())

# Standardise known aliases
STATUS_MAP = {
    'active':   'Active', 'ACTIVE':   'Active', 'yes':    'Active',
    'inactive': 'Inactive', 'INACTIVE': 'Inactive', 'no':   'Inactive',
    'pending':  'Pending', 'PENDING':  'Pending'
}
df['status'] = df['status'].map(STATUS_MAP).fillna(df['status'])

# Extract structure from messy strings
import re
df['phone'] = df['phone'].str.replace(r'[^\d+]', '', regex=True)
df['email'] = df['email'].str.lower().str.strip()
df['valid_email'] = df['email'].str.match(r'^[\w.+-]+@[\w-]+\.[a-z]{2,}$')

print(f'Invalid emails: {(~df["valid_email"]).sum()}')

Data Validation Pipeline

def validate_dataframe(df: pd.DataFrame) -> dict:
    issues = {}

    # Check expected columns
    required = ['id', 'user_id', 'amount', 'date']
    missing  = [c for c in required if c not in df.columns]
    if missing: issues['missing_columns'] = missing

    # Check no nulls in required fields
    null_required = {c: int(df[c].isnull().sum())
                     for c in required if c in df.columns
                     and df[c].isnull().any()}
    if null_required: issues['null_required'] = null_required

    # Business rule checks
    if (df['amount'] < 0).any():
        issues['negative_amounts'] = int((df['amount'] < 0).sum())

    if (df['date'] > pd.Timestamp.now()).any():
        issues['future_dates'] = int((df['date'] > pd.Timestamp.now()).sum())

    # Uniqueness
    if not df['id'].is_unique:
        issues['duplicate_ids'] = int(df['id'].duplicated().sum())

    if not issues:
        print('✅ Validation passed')
    else:
        print(f'⚠️  {len(issues)} validation issues found:')
        for k, v in issues.items():
            print(f'   {k}: {v}')
    return issues

issues = validate_dataframe(df)

Conclusion

Data cleaning is not glamorous, but it is where good data science is built or broken. Always start with a missing value audit and data type check before any analysis. Build your cleaning steps as reproducible functions — not one-off manual edits — so you can rerun them when new data arrives. Document every cleaning decision: why you dropped those rows, why you chose median over mean, why you capped that column at the 99th percentile. Undocumented data cleaning is technical debt that bites you six months later.