Natural Language Processing (NLP) with Python – Complete Guide 2026

Natural Language Processing (NLP) is the branch of AI that gives computers the ability to understand, interpret, and generate human language. From sentiment analysis and chatbots to document classification and information extraction, NLP powers some of the most valuable AI applications. This guide covers NLP fundamentals through modern transformer-based approaches using Python.

Text Preprocessing

import re
import nltk
import spacy
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt')

nlp       = spacy.load('en_core_web_sm')
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))

def preprocess(text: str) -> str:
    # Lowercase
    text = text.lower()
    # Remove URLs
    text = re.sub(r'http\S+|www\S+', '', text)
    # Remove special characters
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # Tokenise and clean
    tokens = nltk.word_tokenize(text)
    tokens = [lemmatizer.lemmatize(t) for t in tokens
              if t not in stop_words and len(t) > 2]
    return ' '.join(tokens)

texts = [
    'Machine learning is revolutionising data science in 2026!',
    'Python NLP libraries make text analysis incredibly easy.',
]
clean = [preprocess(t) for t in texts]
print(clean)

Feature Extraction

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

# TF-IDF (Term Frequency-Inverse Document Frequency)
tfidf = TfidfVectorizer(max_features=5000, ngram_range=(1, 2),
                        min_df=2, max_df=0.95)
X_tfidf = tfidf.fit_transform(clean)
print(f'TF-IDF matrix shape: {X_tfidf.shape}')

# Top terms per document
feature_names = tfidf.get_feature_names_out()
for i, doc in enumerate(clean):
    scores = X_tfidf[i].toarray().flatten()
    top    = sorted(zip(feature_names, scores),
                    key=lambda x: -x[1])[:5]
    print(f'Doc {i}: {top}')

Sentiment Analysis

from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

# Rule-based — fast and requires no training
vader = SentimentIntensityAnalyzer()

reviews = [
    'This product is absolutely amazing! Best purchase ever.',
    'Terrible quality. Total waste of money.',
    'It is okay, nothing special but not bad either.'
]

for review in reviews:
    scores  = vader.polarity_scores(review)
    blob    = TextBlob(review)
    label   = 'POSITIVE' if scores['compound'] > 0.05 else               'NEGATIVE' if scores['compound'] < -0.05 else 'NEUTRAL'
    print(f'{label} (compound={scores["compound"]:.3f}, '
          f'subjectivity={blob.sentiment.subjectivity:.2f}): {review[:50]}')

Named Entity Recognition (NER)

import spacy

nlp = spacy.load('en_core_web_lg')

text = '''Apple CEO Tim Cook announced a new AI partnership with
          Microsoft in San Francisco on Tuesday, with a deal worth
          $5 billion over three years.'''

doc = nlp(text)

print('Entities found:')
for ent in doc.ents:
    print(f'  {ent.text:25s} | {ent.label_:10s} | {spacy.explain(ent.label_)}')

# Custom NER with training data
# Use spaCy's training pipeline for domain-specific entities

Transformers with HuggingFace

from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
import torch

# Zero-shot classification — no fine-tuning needed
classifier = pipeline('zero-shot-classification',
                       model='facebook/bart-large-mnli')

text   = 'The quarterly revenue grew by 35% driven by cloud services.'
labels = ['finance', 'technology', 'sports', 'politics']
result = classifier(text, candidate_labels=labels)
print(result['labels'][0], result['scores'][0])

# Sentiment with BERT
sentiment = pipeline('sentiment-analysis',
                     model='distilbert-base-uncased-finetuned-sst-2-english')
print(sentiment('The new Python 4.0 features are groundbreaking!'))

# Text generation
generator = pipeline('text-generation', model='gpt2')
output    = generator('Data science in 2026 is characterised by',
                       max_length=100, num_return_sequences=1,
                       temperature=0.7)
print(output[0]['generated_text'])

Fine-tuning BERT for Text Classification

from transformers import (AutoTokenizer, AutoModelForSequenceClassification,
                          TrainingArguments, Trainer)
from datasets import Dataset
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

MODEL_NAME = 'distilbert-base-uncased'
tokenizer  = AutoTokenizer.from_pretrained(MODEL_NAME)
model      = AutoModelForSequenceClassification.from_pretrained(
    MODEL_NAME, num_labels=3)

def tokenize(examples):
    return tokenizer(examples['text'], truncation=True,
                     padding='max_length', max_length=128)

# Prepare datasets
train_ds = Dataset.from_pandas(train_df)
test_ds  = Dataset.from_pandas(test_df)
train_ds = train_ds.map(tokenize, batched=True)
test_ds  = test_ds.map(tokenize, batched=True)

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return {'accuracy': accuracy_score(labels, preds),
            'f1': f1_score(labels, preds, average='weighted')}

args = TrainingArguments(
    output_dir='./bert-finetuned',
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    warmup_steps=100,
    weight_decay=0.01,
    evaluation_strategy='epoch',
    save_strategy='epoch',
    load_best_model_at_end=True,
)

trainer = Trainer(model=model, args=args,
                  train_dataset=train_ds, eval_dataset=test_ds,
                  compute_metrics=compute_metrics)
trainer.train()

Topic Modelling with LDA

from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer

count_vect = CountVectorizer(max_features=1000, min_df=3, max_df=0.9)
X_counts   = count_vect.fit_transform(clean_texts)

lda = LatentDirichletAllocation(n_components=5, random_state=42,
                                 max_iter=20, learning_method='online')
lda.fit(X_counts)

feature_names = count_vect.get_feature_names_out()
for topic_idx, topic in enumerate(lda.components_):
    top_words = [feature_names[i] for i in topic.argsort()[:-11:-1]]
    print(f'Topic {topic_idx}: {", ".join(top_words)}')

Conclusion

NLP in 2026 operates at two speeds: fast rule-based preprocessing and VADER for quick sentiment tasks, and transformer-based models for anything requiring deep language understanding. Start with the HuggingFace pipeline API — it gives you state-of-the-art models in three lines of code. Fine-tune only when you have domain-specific data and the zero-shot approach underperforms. The investment in understanding text preprocessing fundamentals pays dividends even in the transformer era — garbage in, garbage out applies to LLMs just as much as to traditional ML.