跳到主要内容

DQN 改进算法

在 DQN 算法 中,经验回放和目标网络提升了训练稳定性。本篇介绍两种彼此正交的改进:Double DQN 改变 TD 目标的计算方式,Dueling DQN 改变 Q 网络的结构。二者可以同时使用。

1. Double DQN​

1.1 过高估计​

标准 DQN 的 TD 目标为

yDQN=r+γ(1−d)max⁡a′Qθ−(s′,a′),y_{\text{DQN}} = r + \gamma (1-d) \max_{a'} Q_{\theta^-}(s',a'),

其中 dd 表示环境是否真正终止,Qθ−Q_{\theta^-} 是目标网络。最大化操作同时承担动作选择和价值评估;当各动作的估计都带有噪声时,max 更容易选中被高估的动作,使目标值产生正偏差。

若真实动作价值相同,且每个估计误差 ϵa\epsilon_a 独立服从 U[−c,c]U[-c,c],动作数为 mm,则

E[max⁡aϵa]=cm−1m+1.\mathbb{E}\left[\max_a \epsilon_a\right] = c\frac{m-1}{m+1}.

这个简化模型说明:动作越多、估计噪声越大,最大化偏差通常越明显。

1.2 解耦选择与评估​

Double DQN 使用训练网络 QθQ_\theta 选择动作,再使用目标网络 Qθ−Q_{\theta^-} 评估该动作:

a∗=arg⁡max⁡a′Qθ(s′,a′),yDoubleDQN=r+γ(1−d)Qθ−(s′,a∗).\begin{aligned} a^* &= \arg\max_{a'} Q_\theta(s',a'),\\ y_{\text{DoubleDQN}} &= r + \gamma(1-d)Q_{\theta^-}(s',a^*). \end{aligned}

它不需要第三个网络,只改变下一状态价值的计算方式。Double DQN 能减轻过高估计,但不能保证所有任务中的估计都无偏。

2. Dueling DQN​

Dueling DQN 将 Q 网络拆成状态价值分支 V(s)V(s) 和优势分支 A(s,a)A(s,a):

Q(s,a)=V(s)+A(s,a).Q(s,a)=V(s)+A(s,a).

直接相加存在不可辨识性:给 VV 加常数、同时从 AA 减去同一常数,Q 值不会变化。实践中通常对优势做中心化:

Q(s,a)=V(s)+(A(s,a)−1∣A∣∑a′A(s,a′)).Q(s,a)=V(s)+ \left(A(s,a)-\frac{1}{|\mathcal A|} \sum_{a'}A(s,a')\right).

这样,网络可以在多个动作价值相近时重点学习“当前状态本身有多好”,并在优势分支中学习动作间的相对差异。Dueling DQN 改进的是表示方式,并不替代 Double DQN 的目标计算。

3. 自包含实现​

下面的示例只依赖 gymnasium、numpy、torch 和 matplotlib,不依赖项目内的 rl_utils。variant 可设为 "DQN"、"DoubleDQN"、"DuelingDQN" 或 "DuelingDoubleDQN"。示例遵循 Gymnasium API,并区分 terminated 与 truncated:二者都会结束当前回合,但只有 terminated 会阻止 TD 目标自举。

import random
from collections import deque

import gymnasium as gym
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F


SEED = 42
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")


def set_seed(seed):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)


class ReplayBuffer:
def __init__(self, capacity):
self.buffer = deque(maxlen=capacity)

def add(self, state, action, reward, next_state, terminated):
self.buffer.append(
(state, action, reward, next_state, terminated)
)

def sample(self, batch_size):
batch = random.sample(self.buffer, batch_size)
states, actions, rewards, next_states, terminateds = zip(*batch)
return {
"states": np.asarray(states, dtype=np.float32),
"actions": np.asarray(actions, dtype=np.int64),
"rewards": np.asarray(rewards, dtype=np.float32),
"next_states": np.asarray(next_states, dtype=np.float32),
"terminateds": np.asarray(terminateds, dtype=np.float32),
}

def __len__(self):
return len(self.buffer)


class QNetwork(nn.Module):
def __init__(self, state_dim, hidden_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, action_dim),
)

def forward(self, state):
return self.net(state)


class DuelingQNetwork(nn.Module):
def __init__(self, state_dim, hidden_dim, action_dim):
super().__init__()
self.feature = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
)
self.value = nn.Linear(hidden_dim, 1)
self.advantage = nn.Linear(hidden_dim, action_dim)

def forward(self, state):
feature = self.feature(state)
value = self.value(feature)
advantage = self.advantage(feature)
return value + advantage - advantage.mean(dim=1, keepdim=True)


class DQNAgent:
def __init__(
self,
state_dim,
action_dim,
variant="DuelingDoubleDQN",
hidden_dim=128,
learning_rate=1e-3,
gamma=0.99,
epsilon=0.1,
target_update=100,
):
valid_variants = {
"DQN",
"DoubleDQN",
"DuelingDQN",
"DuelingDoubleDQN",
}
if variant not in valid_variants:
raise ValueError(f"variant must be one of {valid_variants}")

network_class = (
DuelingQNetwork if "Dueling" in variant else QNetwork
)
self.online = network_class(
state_dim, hidden_dim, action_dim
).to(DEVICE)
self.target = network_class(
state_dim, hidden_dim, action_dim
).to(DEVICE)
self.target.load_state_dict(self.online.state_dict())
self.optimizer = torch.optim.Adam(
self.online.parameters(), lr=learning_rate
)
self.action_dim = action_dim
self.double = "Double" in variant
self.gamma = gamma
self.epsilon = epsilon
self.target_update = target_update
self.update_count = 0

def take_action(self, state, explore=True):
if explore and random.random() < self.epsilon:
return random.randrange(self.action_dim)
state_tensor = torch.as_tensor(
state, dtype=torch.float32, device=DEVICE
).unsqueeze(0)
with torch.no_grad():
return self.online(state_tensor).argmax(dim=1).item()

def update(self, batch):
states = torch.as_tensor(
batch["states"], dtype=torch.float32, device=DEVICE
)
actions = torch.as_tensor(
batch["actions"], dtype=torch.int64, device=DEVICE
).unsqueeze(1)
rewards = torch.as_tensor(
batch["rewards"], dtype=torch.float32, device=DEVICE
).unsqueeze(1)
next_states = torch.as_tensor(
batch["next_states"], dtype=torch.float32, device=DEVICE
)
terminateds = torch.as_tensor(
batch["terminateds"], dtype=torch.float32, device=DEVICE
).unsqueeze(1)

q_values = self.online(states).gather(1, actions)
with torch.no_grad():
if self.double:
next_actions = self.online(next_states).argmax(
dim=1, keepdim=True
)
next_q_values = self.target(next_states).gather(
1, next_actions
)
else:
next_q_values = self.target(next_states).max(
dim=1, keepdim=True
).values
targets = (
rewards
+ self.gamma * (1.0 - terminateds) * next_q_values
)

loss = F.smooth_l1_loss(q_values, targets)
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(self.online.parameters(), max_norm=10.0)
self.optimizer.step()

self.update_count += 1
if self.update_count % self.target_update == 0:
self.target.load_state_dict(self.online.state_dict())


def train(
variant="DuelingDoubleDQN",
episodes=500,
batch_size=64,
warmup_steps=1000,
):
set_seed(SEED)
env = gym.make("CartPole-v1")
env.action_space.seed(SEED)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = DQNAgent(state_dim, action_dim, variant=variant)
replay_buffer = ReplayBuffer(capacity=20_000)
returns = []

for episode in range(episodes):
state, _ = env.reset(seed=SEED if episode == 0 else None)
episode_return = 0.0

while True:
action = agent.take_action(state)
next_state, reward, terminated, truncated, _ = env.step(
action
)
replay_buffer.add(
state, action, reward, next_state, terminated
)
state = next_state
episode_return += reward

if len(replay_buffer) >= warmup_steps:
agent.update(replay_buffer.sample(batch_size))

if terminated or truncated:
break

returns.append(episode_return)
agent.epsilon = max(0.01, agent.epsilon * 0.995)
if (episode + 1) % 50 == 0:
mean_return = np.mean(returns[-50:])
print(
f"Episode {episode + 1}: "
f"mean return = {mean_return:.1f}"
)

env.close()
return agent, returns


def evaluate(agent, episodes=10):
env = gym.make("CartPole-v1")
scores = []
for episode in range(episodes):
state, _ = env.reset(seed=SEED + 1000 + episode)
score = 0.0
while True:
action = agent.take_action(state, explore=False)
state, reward, terminated, truncated, _ = env.step(action)
score += reward
if terminated or truncated:
break
scores.append(score)
env.close()
return float(np.mean(scores))


if __name__ == "__main__":
agent, returns = train(variant="DuelingDoubleDQN")
print(f"Evaluation mean return: {evaluate(agent):.1f}")
plt.plot(returns)
plt.xlabel("Episode")
plt.ylabel("Return")
plt.title("Dueling Double DQN on CartPole-v1")
plt.show()

固定随机种子有助于复现实验,但 PyTorch、硬件和依赖版本仍可能带来数值差异。评估阶段关闭 ϵ\epsilon-贪婪探索,并使用与训练不同的确定性种子序列。

下一篇将从基于价值的方法转向直接优化策略的 策略梯度与 REINFORCE。