Web Scraping with Python – BeautifulSoup & Scrapy Complete Guide

Web scraping lets you collect data from websites that don’t offer an API — product prices, news articles, job listings, stock data, research papers. Python has the best ecosystem for this: BeautifulSoup for parsing HTML, Scrapy for large-scale crawls, and Selenium/Playwright for JavaScript-heavy pages. This guide covers all three with practical examples.

Ethics and Legality

Before scraping, always check the website’s robots.txt file and Terms of Service. Many sites explicitly prohibit scraping. Always add delays between requests to avoid overloading servers, identify your scraper with a User-Agent header, and prefer official APIs when available. Never scrape personal or private data, and check the legal jurisdiction — data scraping laws vary by country.

BeautifulSoup – Simple HTML Parsing

pip install requests beautifulsoup4 lxml

import requests
from bs4 import BeautifulSoup
import time

headers = {'User-Agent': 'Mozilla/5.0 (DataResearch Bot 1.0)'}

def scrape_page(url):
    response = requests.get(url, headers=headers, timeout=10)
    response.raise_for_status()
    return BeautifulSoup(response.text, 'lxml')

# Example: scrape job listings
soup = scrape_page("https://example-jobs.com/data-science")

jobs = []
for card in soup.select('div.job-card'):
    title    = card.select_one('h2.job-title')
    company  = card.select_one('span.company-name')
    location = card.select_one('span.location')
    salary   = card.select_one('span.salary')

    jobs.append({
        'title':    title.text.strip()    if title    else None,
        'company':  company.text.strip()  if company  else None,
        'location': location.text.strip() if location else None,
        'salary':   salary.text.strip()   if salary   else None,
    })

import pandas as pd
df = pd.DataFrame(jobs)
print(df.head())

Navigating the DOM

# CSS selectors (most flexible)
soup.select('div.article p')          # all 
 inside div.article
soup.select_one('h1#main-title')      # first h1 with id="main-title"
soup.select('a[href^="https"]')       # anchors where href starts with https

# Direct navigation
soup.find('table', {'class': 'data-table'})
soup.find_all('tr')[1:]               # all rows except header
tag.get('href')                        # get attribute value
tag.text.strip()                       # get inner text, stripped

# Parent/sibling navigation
tag.parent
tag.find_next_sibling('td')
tag.find_previous('h2')

Scraping Multiple Pages

import time, random

BASE_URL = "https://example.com/articles?page={}"
all_articles = []

for page in range(1, 11):
    url  = BASE_URL.format(page)
    soup = scrape_page(url)

    articles = soup.select('article.post')
    if not articles:
        break

    for art in articles:
        all_articles.append({
            'title': art.select_one('h2').text.strip(),
            'date':  art.select_one('time')['datetime'],
            'url':   art.select_one('a')['href'],
        })

    # Polite delay: 1-3 seconds between pages
    time.sleep(random.uniform(1, 3))

print(f"Scraped {len(all_articles)} articles")

Scrapy – Production-Scale Web Crawling

pip install scrapy
scrapy startproject datascraper
cd datascraper

# datascraper/spiders/jobs_spider.py
import scrapy

class JobsSpider(scrapy.Spider):
    name  = "jobs"
    start_urls = ["https://example-jobs.com/data-science"]

    custom_settings = {
        'DOWNLOAD_DELAY': 1.5,          # 1.5s between requests
        'CONCURRENT_REQUESTS': 4,
        'ROBOTSTXT_OBEY': True,
        'USER_AGENT': 'DataResearch Bot 1.0',
    }

    def parse(self, response):
        for card in response.css('div.job-card'):
            yield {
                'title':   card.css('h2.job-title::text').get('').strip(),
                'company': card.css('span.company::text').get('').strip(),
                'url':     card.css('a::attr(href)').get(),
            }

        # Follow pagination
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

# Run spider and save to CSV
scrapy crawl jobs -o jobs.csv
scrapy crawl jobs -o jobs.json

Handling JavaScript-Rendered Pages with Playwright

pip install playwright
playwright install chromium

from playwright.sync_api import sync_playwright
import time

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page    = browser.new_page()

    page.goto("https://javascript-heavy-site.com/data")
    page.wait_for_selector("div.data-table", timeout=10000)

    # Wait for dynamic content to load
    page.wait_for_load_state("networkidle")

    content = page.content()
    browser.close()

soup = BeautifulSoup(content, 'lxml')
# ... parse as normal

Storing Scraped Data

import sqlite3, pandas as pd

# Save to SQLite
conn = sqlite3.connect("scraped_data.db")
df.to_sql("jobs", conn, if_exists="append", index=False)
conn.close()

# Save to CSV with deduplication
existing = pd.read_csv("jobs.csv") if os.path.exists("jobs.csv") else pd.DataFrame()
combined = pd.concat([existing, df]).drop_duplicates(subset=['url'])
combined.to_csv("jobs.csv", index=False)

Conclusion

Web scraping is an essential data collection skill for data scientists. Use BeautifulSoup for simple one-off scrapes, Scrapy for large-scale production crawlers, and Playwright when the target page renders content via JavaScript. Always scrape ethically — respect robots.txt, add delays, and use official APIs when available. Clean, well-structured scraped data can be the foundation of competitive analysis, market research, NLP training datasets, and price monitoring systems.