Pandas Performance – Speed Up Your Data Analysis 10x 2026

Slow pandas code is one of the most common performance bottlenecks in data science workflows. A notebook that takes 20 minutes to run often has simple fixes that bring it down to 2 minutes. This guide covers every major pandas optimisation technique — from the basics of avoiding iterrows to Polars and Dask for when pandas is not enough.

Profile Before Optimising

import pandas as pd
import numpy as np
import time

# Generate sample data
df = pd.DataFrame({
    'user_id':  np.random.randint(1, 10000, 1_000_000),
    'amount':   np.random.uniform(10, 1000, 1_000_000),
    'category': np.random.choice(['A', 'B', 'C', 'D'], 1_000_000),
    'date':     pd.date_range('2024-01-01', periods=1_000_000, freq='1s')
})

# Simple timing
start = time.perf_counter()
result = df['amount'].mean()
print(f'{time.perf_counter() - start:.4f}s')

# %timeit in Jupyter
# %timeit df['amount'].mean()

# Memory usage
print(df.info(memory_usage='deep'))
print(f'Total: {df.memory_usage(deep=True).sum() / 1e6:.1f} MB')

Avoid iterrows — Use Vectorisation

import timeit

# ❌ SLOW: iterrows (100-1000x slower than vectorised)
def slow_apply(df):
    results = []
    for _, row in df.iterrows():
        if row['amount'] > 500:
            results.append(row['amount'] * 1.1)
        else:
            results.append(row['amount'] * 0.9)
    return results

# ❌ Better but still slow: apply
def medium_apply(df):
    return df['amount'].apply(lambda x: x * 1.1 if x > 500 else x * 0.9)

# ✅ FAST: vectorised with numpy (10-100x faster than apply)
def fast_vectorised(df):
    return np.where(df['amount'] > 500,
                    df['amount'] * 1.1,
                    df['amount'] * 0.9)

# Benchmark
df_small = df.head(10_000)
t_slow   = timeit.timeit(lambda: slow_apply(df_small), number=1)
t_medium = timeit.timeit(lambda: medium_apply(df_small), number=1)
t_fast   = timeit.timeit(lambda: fast_vectorised(df_small), number=3) / 3

print(f'iterrows: {t_slow:.3f}s')
print(f'apply:    {t_medium:.3f}s')
print(f'vectorised: {t_fast:.3f}s')
print(f'Speedup: {t_slow/t_fast:.0f}x')

Optimise Data Types

# ── Reduce memory, increase speed ────────────────────────────

# Before optimisation
print(f'Before: {df.memory_usage(deep=True).sum() / 1e6:.1f} MB')

# Downcast numeric types
df['user_id']  = pd.to_numeric(df['user_id'],  downcast='integer')   # int64→int16
df['amount']   = pd.to_numeric(df['amount'],   downcast='float')     # float64→float32

# Category dtype for low-cardinality strings (saves 5-10x memory)
df['category'] = df['category'].astype('category')

# After optimisation
print(f'After: {df.memory_usage(deep=True).sum() / 1e6:.1f} MB')

# Automated optimisation
def optimise_dtypes(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()
    for col in df.select_dtypes(include='integer').columns:
        df[col] = pd.to_numeric(df[col], downcast='integer')
    for col in df.select_dtypes(include='float').columns:
        df[col] = pd.to_numeric(df[col], downcast='float')
    for col in df.select_dtypes(include='object').columns:
        if df[col].nunique() / len(df) < 0.5:
            df[col] = df[col].astype('category')
    return df

df_opt = optimise_dtypes(df)
print(f'Optimised: {df_opt.memory_usage(deep=True).sum() / 1e6:.1f} MB')

Efficient Groupby and Aggregation

# ✅ GroupBy is already vectorised — use it over manual loops
agg = df.groupby('category').agg(
    total_amount  = ('amount', 'sum'),
    avg_amount    = ('amount', 'mean'),
    count         = ('amount', 'count'),
    max_amount    = ('amount', 'max'),
).reset_index()

# ✅ Named aggregations (pandas 0.25+) — more readable
agg2 = (df
    .groupby(['category', df['date'].dt.month])
    .agg(revenue=('amount', 'sum'), n_orders=('amount', 'count'))
    .reset_index())

# ✅ transform() for broadcasting group stats back to original df
df['category_mean'] = df.groupby('category')['amount'].transform('mean')
df['pct_of_cat']    = df['amount'] / df['category_mean']

Process Large Files in Chunks

# For files that don't fit in RAM
chunk_size = 100_000
results    = []

for chunk in pd.read_csv('large_file.csv', chunksize=chunk_size):
    # Process each chunk
    chunk_result = (chunk
        .query('amount > 100')
        .groupby('category')['amount']
        .agg(['sum', 'count']))
    results.append(chunk_result)

final = pd.concat(results).groupby(level=0).sum()
print(final)

# Or use with read_parquet + filters (much faster for parquet)
df_filtered = pd.read_parquet('data.parquet',
                               columns=['user_id', 'amount', 'category'],
                               filters=[('amount', '>', 100)])

Polars – The Faster Alternative

pip install polars

import polars as pl

# Polars is 5-20x faster than pandas for most operations
df_pl = pl.read_csv('large_file.csv')

# Lazy evaluation — query plan optimised before execution
result = (
    pl.scan_csv('large_file.csv')
    .filter(pl.col('amount') > 100)
    .groupby('category')
    .agg([
        pl.col('amount').sum().alias('total'),
        pl.col('amount').mean().alias('avg'),
        pl.col('amount').count().alias('n'),
    ])
    .sort('total', descending=True)
    .collect()   # execute the optimised plan
)
print(result)

# Convert between pandas and polars
df_from_pandas = pl.from_pandas(df)
df_to_pandas   = result.to_pandas()

Dask for Distributed Processing

pip install dask[complete]

import dask.dataframe as dd

# Read files too large for RAM
ddf = dd.read_parquet('data/*.parquet')

# Same API as pandas — lazily evaluated
result = (ddf
    .query('amount > 100')
    .groupby('category')['amount']
    .agg(['sum', 'mean', 'count'])
    .compute())   # triggers actual computation

# Process in parallel
from dask import delayed

@delayed
def process_file(path):
    return pd.read_parquet(path).query('amount > 100')

paths   = ['file1.parquet', 'file2.parquet', 'file3.parquet']
results = [process_file(p) for p in paths]
final   = dd.from_delayed(results).compute()

Quick Wins Checklist

Replace iterrows() with vectorised operations or numpy.where() — 10-1000x speedup. Use category dtype for string columns with under 50% unique values — 5-10x memory reduction. Read parquet instead of CSV — 5-10x faster to read. Specify usecols when reading files — avoid loading columns you don't need. Use query() for filtering — often faster than boolean indexing on large DataFrames. For 100M+ rows, switch to Polars. For data that won't fit in RAM across multiple machines, use Dask.

Conclusion

Pandas performance optimisation follows a clear priority order: first eliminate iterrows and apply loops through vectorisation, then reduce memory with dtype optimisation, then profile with timeit to find the remaining bottlenecks. For data that fits in RAM and you need maximum speed, Polars is the future — its query optimiser and Rust backend deliver consistent 5-20x speedups over pandas. For data that exceeds RAM, Dask extends the pandas API to distributed computing. These tools together mean you almost never need to move to Spark just for performance.