Reinforcement Learning (RL) is the branch of machine learning where an agent learns by interacting with an environment — taking actions, receiving rewards, and improving its strategy over time. It powers AlphaGo, robotics, trading algorithms, and recommendation systems. This guide builds your understanding from Q-learning fundamentals to Deep Q-Networks with real Python code.
Core RL Concepts
An RL system has four components: an Agent (the learner), an Environment (what the agent interacts with), a State (current situation), and a Reward (feedback signal). The agent’s goal is to learn a Policy — a mapping from states to actions — that maximises cumulative reward over time. The challenge is the exploration-exploitation tradeoff: try new actions to discover better strategies, or exploit known good actions to maximise immediate reward.
Setting Up OpenAI Gymnasium
pip install gymnasium numpy matplotlib torch
import gymnasium as gym
import numpy as np
# Classic CartPole environment — balance a pole on a cart
env = gym.make('CartPole-v1', render_mode='rgb_array')
obs, info = env.reset(seed=42)
print(f'Observation space: {env.observation_space}') # Box(4,) — cart pos, vel, pole angle, vel
print(f'Action space: {env.action_space}') # Discrete(2) — push left or right
# Random agent baseline
total_reward = 0
for _ in range(500):
action = env.action_space.sample() # random action
obs, reward, terminated, truncated, info = env.step(action)
total_reward += reward
if terminated or truncated:
obs, info = env.reset()
print(f'Random agent total reward: {total_reward}')
env.close()
Q-Learning (Tabular)
Q-learning learns a Q-table — a lookup table where Q(state, action) estimates the expected future reward of taking that action in that state. Works well for discrete, small state spaces.
import numpy as np
import gymnasium as gym
env = gym.make('FrozenLake-v1', is_slippery=False)
# Initialise Q-table
n_states = env.observation_space.n # 16 states (4x4 grid)
n_actions = env.action_space.n # 4 actions (L, D, R, U)
Q = np.zeros((n_states, n_actions))
# Hyperparameters
alpha = 0.8 # learning rate
gamma = 0.95 # discount factor (how much future rewards matter)
epsilon = 1.0 # exploration rate (start fully random)
eps_decay = 0.995
eps_min = 0.01
n_episodes = 5000
rewards_per_ep = []
for ep in range(n_episodes):
state, _ = env.reset()
total_reward, done = 0, False
while not done:
# Epsilon-greedy action selection
if np.random.random() < epsilon:
action = env.action_space.sample() # explore
else:
action = np.argmax(Q[state]) # exploit
next_state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
# Q-learning update rule
best_next = np.max(Q[next_state])
Q[state, action] += alpha * (
reward + gamma * best_next * (not done) - Q[state, action])
state = next_state
total_reward += reward
epsilon = max(eps_min, epsilon * eps_decay)
rewards_per_ep.append(total_reward)
# Evaluate trained agent
wins = sum(rewards_per_ep[-100:])
print(f'Win rate (last 100 episodes): {wins}%')
print('Learned Q-table:')
print(Q.reshape(4, 4, 4).round(2))
Deep Q-Network (DQN)
When the state space is too large for a Q-table (images, continuous values), replace the table with a neural network. DQN adds Experience Replay and a Target Network for stable training.
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from collections import deque
import random
class DQN(nn.Module):
def __init__(self, state_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, 128), nn.ReLU(),
nn.Linear(128, 128), nn.ReLU(),
nn.Linear(128, action_dim)
)
def forward(self, x):
return self.net(x)
class ReplayBuffer:
def __init__(self, capacity=10_000):
self.buffer = deque(maxlen=capacity)
def push(self, *transition):
self.buffer.append(transition)
def sample(self, batch_size):
return random.sample(self.buffer, batch_size)
def __len__(self):
return len(self.buffer)
env = gym.make('CartPole-v1')
state_dim = env.observation_space.shape[0] # 4
action_dim = env.action_space.n # 2
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
policy_net = DQN(state_dim, action_dim).to(device)
target_net = DQN(state_dim, action_dim).to(device)
target_net.load_state_dict(policy_net.state_dict())
optimizer = optim.AdamW(policy_net.parameters(), lr=1e-3)
buffer = ReplayBuffer(10_000)
BATCH_SIZE = 64
GAMMA = 0.99
EPS_START = 1.0
EPS_END = 0.05
EPS_DECAY = 500
TARGET_UPDATE = 10
def select_action(state, epsilon):
if random.random() < epsilon:
return env.action_space.sample()
with torch.no_grad():
s = torch.FloatTensor(state).unsqueeze(0).to(device)
return policy_net(s).argmax().item()
def train_step():
if len(buffer) < BATCH_SIZE:
return
batch = buffer.sample(BATCH_SIZE)
states, actions, rewards, next_states, dones = zip(*batch)
states = torch.FloatTensor(states).to(device)
actions = torch.LongTensor(actions).unsqueeze(1).to(device)
rewards = torch.FloatTensor(rewards).to(device)
next_states = torch.FloatTensor(next_states).to(device)
dones = torch.FloatTensor(dones).to(device)
current_q = policy_net(states).gather(1, actions).squeeze()
with torch.no_grad():
next_q = target_net(next_states).max(1)[0]
target = rewards + GAMMA * next_q * (1 - dones)
loss = nn.SmoothL1Loss()(current_q, target)
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(policy_net.parameters(), 1.0)
optimizer.step()
epsilon = EPS_START
ep_rewards = []
for ep in range(600):
state, _ = env.reset()
ep_reward = 0
while True:
action = select_action(state, epsilon)
next_state, reward, term, trunc, _ = env.step(action)
done = term or trunc
buffer.push(state, action, reward, next_state, float(done))
train_step()
state = next_state
ep_reward += reward
if done: break
ep_rewards.append(ep_reward)
epsilon = EPS_END + (EPS_START - EPS_END) * np.exp(-ep / EPS_DECAY)
if ep % TARGET_UPDATE == 0:
target_net.load_state_dict(policy_net.state_dict())
if ep % 50 == 0:
avg = np.mean(ep_rewards[-50:])
print(f'Episode {ep:4d} | Avg reward: {avg:.1f} | Eps: {epsilon:.3f}')
Policy Gradient (REINFORCE)
class PolicyNet(nn.Module):
def __init__(self, state_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, 64), nn.ReLU(),
nn.Linear(64, action_dim), nn.Softmax(dim=-1)
)
def forward(self, x):
return self.net(x)
policy = PolicyNet(state_dim, action_dim).to(device)
optimizer = optim.Adam(policy.parameters(), lr=1e-3)
def reinforce_episode():
state, _ = env.reset()
log_probs, rewards = [], []
while True:
s = torch.FloatTensor(state).unsqueeze(0).to(device)
dist = torch.distributions.Categorical(policy(s))
action = dist.sample()
log_probs.append(dist.log_prob(action))
state, reward, term, trunc, _ = env.step(action.item())
rewards.append(reward)
if term or trunc: break
# Compute discounted returns
returns, G = [], 0
for r in reversed(rewards):
G = r + 0.99 * G
returns.insert(0, G)
returns = torch.FloatTensor(returns).to(device)
returns = (returns - returns.mean()) / (returns.std() + 1e-8)
loss = -sum(lp * ret for lp, ret in zip(log_probs, returns))
optimizer.zero_grad()
loss.backward()
optimizer.step()
return sum(rewards)
for ep in range(1000):
reward = reinforce_episode()
if ep % 100 == 0:
print(f'Episode {ep}: reward = {reward}')
Conclusion
Reinforcement learning is uniquely suited to sequential decision problems where the optimal action depends on long-term consequences. Start with tabular Q-learning on small discrete environments to build intuition, then graduate to DQN for continuous or high-dimensional state spaces. Modern RL libraries like Stable-Baselines3 package DQN, PPO, and SAC with sensible defaults — use them for production, and build from scratch only when you need architectural control.
