Building a machine learning model is 20% of the work. Getting it to production reliably, keeping it accurate over time, and retraining it automatically when performance degrades — that is MLOps. This guide covers the full MLOps lifecycle: experiment tracking, model registry, CI/CD pipelines, and production monitoring.
Why MLOps?
ML models are not static software. Data distributions shift, features get deprecated, and model accuracy degrades silently. Without MLOps, teams spend hours manually retraining and redeploying models, experiments are unreproducible, and production failures go undetected. MLOps applies DevOps principles to ML: automate everything, version everything, monitor everything.
Experiment Tracking with MLflow
pip install mlflow scikit-learn
import mlflow
import mlflow.sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
import numpy as np
mlflow.set_tracking_uri('sqlite:///mlflow.db')
mlflow.set_experiment('breast-cancer-classifier')
X, y = load_breast_cancer(return_X_y=True)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=42)
# Grid of hyperparameters to try
configs = [
{'n_estimators': 50, 'max_depth': 3},
{'n_estimators': 100, 'max_depth': 5},
{'n_estimators': 200, 'max_depth': 7},
]
for cfg in configs:
with mlflow.start_run():
mlflow.log_params(cfg)
model = RandomForestClassifier(**cfg, random_state=42, n_jobs=-1)
cv_auc = cross_val_score(model, X_tr, y_tr, cv=5,
scoring='roc_auc').mean()
model.fit(X_tr, y_tr)
test_acc = model.score(X_te, y_te)
mlflow.log_metric('cv_auc', cv_auc)
mlflow.log_metric('test_acc', test_acc)
mlflow.sklearn.log_model(model, 'model',
registered_model_name='BreastCancerRF')
print(f'n_est={cfg["n_estimators"]} depth={cfg["max_depth"]} '
f'AUC={cv_auc:.4f} Acc={test_acc:.4f}')
# Launch the MLflow UI
# mlflow ui --backend-store-uri sqlite:///mlflow.db
Model Registry and Staging
from mlflow.tracking import MlflowClient
client = MlflowClient('sqlite:///mlflow.db')
# Find best run by CV AUC
exp = client.get_experiment_by_name('breast-cancer-classifier')
runs = client.search_runs(exp.experiment_id,
order_by=['metrics.cv_auc DESC'])
best = runs[0]
print(f'Best run: {best.info.run_id}')
print(f'Best AUC: {best.data.metrics["cv_auc"]:.4f}')
# Promote best model to Production
model_versions = client.search_model_versions("name='BreastCancerRF'")
best_version = sorted(model_versions,
key=lambda v: float(v.tags.get('cv_auc', 0)),
reverse=True)[0]
client.transition_model_version_stage(
name='BreastCancerRF',
version=best_version.version,
stage='Production',
archive_existing_versions=True
)
print(f'Version {best_version.version} promoted to Production')
# Load production model anywhere
prod_model = mlflow.sklearn.load_model(
'models:/BreastCancerRF/Production')
CI/CD Pipeline with GitHub Actions
# .github/workflows/ml-pipeline.yml
name: ML CI/CD Pipeline
on:
push:
branches: [main]
schedule:
- cron: '0 6 * * 1' # weekly retraining every Monday 6am
jobs:
train-and-evaluate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run data validation
run: python src/validate_data.py
- name: Train model
run: python src/train.py
env:
MLFLOW_TRACKING_URI: ${{ secrets.MLFLOW_URI }}
- name: Evaluate model
run: python src/evaluate.py --threshold 0.92
- name: Deploy if metrics pass
if: success()
run: python src/deploy.py
env:
WP_DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
Model Monitoring and Drift Detection
from evidently.report import Report
from evidently.metric_preset import DataDriftPreset, ClassificationPreset
from evidently.metrics import *
import pandas as pd
# Reference data (training distribution)
ref_data = pd.read_parquet('data/train.parquet')
# Current production data (last 7 days)
curr_data = pd.read_parquet('data/production_last_7d.parquet')
# Data drift report
drift_report = Report(metrics=[DataDriftPreset()])
drift_report.run(reference_data=ref_data,
current_data=curr_data)
drift_report.save_html('reports/drift_report.html')
# Check if drift is significant
result = drift_report.as_dict()
drifted_cols = [col for col, info in
result['metrics'][0]['result']['drift_by_columns'].items()
if info['drift_detected']]
if len(drifted_cols) > 0:
print(f'⚠️ Drift detected in: {drifted_cols}')
# Trigger retraining via webhook or CI/CD
# Model performance monitoring
perf_report = Report(metrics=[
ClassificationPreset(),
ColumnDriftMetric(column_name='prediction'),
])
perf_report.run(reference_data=ref_data.assign(prediction=ref_preds),
current_data=curr_data.assign(prediction=curr_preds))
perf_report.save_html('reports/performance_report.html')
Automated Retraining Pipeline
import schedule
import time
import logging
from datetime import datetime
logger = logging.getLogger(__name__)
def check_and_retrain():
logger.info('Running drift check...')
drift_score = compute_drift_score() # your drift metric
curr_auc = compute_current_auc() # live model performance
DRIFT_THRESHOLD = 0.15
AUC_THRESHOLD = 0.90
if drift_score > DRIFT_THRESHOLD or curr_auc < AUC_THRESHOLD:
logger.warning(f'Retraining triggered: drift={drift_score:.3f} '
f'auc={curr_auc:.3f}')
retrain_and_deploy()
else:
logger.info(f'Model healthy: drift={drift_score:.3f} '
f'auc={curr_auc:.3f}')
def retrain_and_deploy():
import subprocess
result = subprocess.run(['python', 'src/train.py', '--auto'],
capture_output=True, text=True)
if result.returncode == 0:
logger.info('Retraining complete — deploying new model')
subprocess.run(['python', 'src/deploy.py'])
else:
logger.error(f'Retraining failed: {result.stderr}')
# Schedule weekly check
schedule.every().monday.at('06:00').do(check_and_retrain)
schedule.every().day.at('08:00').do(lambda: log_daily_metrics())
while True:
schedule.run_pending()
time.sleep(60)
Conclusion
MLOps transforms ML from one-off experiments into reliable production systems. Start with MLflow for experiment tracking — it requires minimal setup and pays dividends immediately. Add a model registry to control what goes to production. Build a CI/CD pipeline that runs your training and evaluation automatically on every code push. And implement drift monitoring so you know when your model needs retraining before your users notice. These four practices alone will make your ML systems dramatically more robust.
