Web Scraping with Python – BeautifulSoup, Scrapy & Playwright 2026

Web scraping is a critical data collection skill — public data on prices, job listings, news, reviews, and social signals is often only accessible through scraping. This guide covers three tools that together handle every scraping scenario: BeautifulSoup for quick static scraping, Scrapy for large-scale crawling, and Playwright for JavaScript-rendered sites.

Ethics and Legality

Always check the site’s robots.txt before scraping (e.g., https://example.com/robots.txt). Respect the Crawl-delay directive. Never scrape personal data without legal basis. Do not overload servers — add delays between requests. Check the site’s Terms of Service. Public data is generally scrape-able; authenticated user data and copyrighted full-text content require more care. When an API exists, use it instead of scraping.

BeautifulSoup for Static Pages

pip install requests beautifulsoup4 lxml

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/120.0.0.0 Safari/537.36'
}

def scrape_page(url: str) -> BeautifulSoup:
    r = requests.get(url, headers=headers, timeout=15)
    r.raise_for_status()
    return BeautifulSoup(r.text, 'lxml')

# Scrape Hacker News front page
soup  = scrape_page('https://news.ycombinator.com')
items = soup.select('.athing')

stories = []
for item in items:
    title_el = item.select_one('.titleline > a')
    score_el = item.find_next_sibling('tr').select_one('.score')
    stories.append({
        'title':  title_el.text if title_el else '',
        'url':    title_el.get('href', '') if title_el else '',
        'score':  score_el.text if score_el else '0 points',
    })

df = pd.DataFrame(stories)
print(df.head(10))

# ── Multi-page scraping ───────────────────────────────────────
all_data = []
for page in range(1, 6):
    url  = f'https://news.ycombinator.com/?p={page}'
    soup = scrape_page(url)
    # ... extract data ...
    all_data.extend(stories)
    time.sleep(random.uniform(1.5, 3.0))   # polite delay

Parsing Techniques

soup = scrape_page('https://example-ecommerce.com/products')

# CSS selectors (most reliable)
prices = soup.select('.product-card .price')
names  = soup.select('h2.product-name')

# By tag + attribute
links  = soup.find_all('a', class_='product-link')
img    = soup.find('img', attrs={'data-testid': 'hero-image'})

# By text content
heading = soup.find('h1', string=lambda t: t and 'Sale' in t)

# Navigate the tree
parent    = soup.find('div', class_='product-info').parent
siblings  = soup.find('h2').find_next_siblings('p')
first_li  = soup.find('ul').find('li')

# Extract attributes
for a in soup.find_all('a', href=True):
    if 'product' in a['href']:
        print(a['href'], a.get_text(strip=True))

# Handle pagination
next_btn = soup.select_one('a[rel="next"]')
next_url = next_btn['href'] if next_btn else None

Scrapy for Large-Scale Crawling

pip install scrapy

# Create a Scrapy project
# scrapy startproject myproject
# cd myproject && scrapy genspider quotes quotes.toscrape.com

# spiders/quotes_spider.py
import scrapy

class QuotesSpider(scrapy.Spider):
    name         = 'quotes'
    start_urls   = ['https://quotes.toscrape.com/']
    custom_settings = {
        'DOWNLOAD_DELAY': 1,
        'RANDOMIZE_DOWNLOAD_DELAY': True,
        'CONCURRENT_REQUESTS': 4,
        'ROBOTSTXT_OBEY': True,
    }

    def parse(self, response):
        # Extract items from the page
        for quote in response.css('div.quote'):
            yield {
                'text':   quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
                'tags':   quote.css('a.tag::text').getall(),
            }

        # Follow pagination
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

# Run: scrapy crawl quotes -o quotes.csv
# Run: scrapy crawl quotes -o quotes.json

# Scrapy with Item Pipeline for cleaning
# items.py
import scrapy

class QuoteItem(scrapy.Item):
    text   = scrapy.Field()
    author = scrapy.Field()
    tags   = scrapy.Field()

# pipelines.py
class CleaningPipeline:
    def process_item(self, item, spider):
        item['text'] = item['text'].strip('"').strip('"')
        item['tags'] = [t.lower() for t in item['tags']]
        return item

# settings.py
ITEM_PIPELINES = {'myproject.pipelines.CleaningPipeline': 300}

Playwright for JavaScript Pages

pip install playwright
python -m playwright install chromium

import asyncio
from playwright.async_api import async_playwright
import pandas as pd

async def scrape_js_page():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page    = await browser.new_page()

        # Set realistic headers
        await page.set_extra_http_headers({
            'Accept-Language': 'en-US,en;q=0.9'
        })

        # Navigate and wait for content to load
        await page.goto('https://example-spa.com/listings',
                        wait_until='networkidle')

        # Scroll to load lazy content
        for _ in range(5):
            await page.evaluate('window.scrollBy(0, window.innerHeight)')
            await page.wait_for_timeout(1000)

        # Extract data via CSS selectors
        items = await page.query_selector_all('.listing-card')

        results = []
        for item in items:
            title = await item.query_selector('.title')
            price = await item.query_selector('.price')
            results.append({
                'title': await title.inner_text() if title else '',
                'price': await price.inner_text() if price else '',
            })

        await browser.close()
        return pd.DataFrame(results)

df = asyncio.run(scrape_js_page())
print(df.head())

Handling Anti-Scraping Measures

import time, random
from itertools import cycle

# Rotate user agents
USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36...',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36...',
]
ua_cycle = cycle(USER_AGENTS)

# Use session with retries
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

session = requests.Session()
retry   = Retry(total=3, backoff_factor=1,
                status_forcelist=[429, 500, 502, 503, 504])
session.mount('http://', HTTPAdapter(max_retries=retry))

def polite_get(url):
    time.sleep(random.uniform(1.0, 3.0))
    headers = {'User-Agent': next(ua_cycle)}
    return session.get(url, headers=headers, timeout=15)

Conclusion

BeautifulSoup is your first tool for any scraping job — fast to set up, easy to debug, perfect for static pages. When you need to scrape thousands of pages systematically, Scrapy’s pipeline architecture and built-in rate limiting make it the right choice. When the target site renders content with JavaScript, Playwright handles what static scrapers cannot. Always scrape responsibly: respect robots.txt, add delays, and prefer official APIs when they exist.