Actor-Critic 算法
1. 从 REINFORCE 到 Actor-Critic
REINFORCE 直接优化随机策略 。它在回合结束后计算蒙特卡洛回报
再用 更新策略。完整回报不依赖价值函数,但方差较大,而且必须等待后续奖励到齐。
Actor-Critic 同时学习两个组件:
- Actor:策略网络 ,负责选择动作并改进策略。
- Critic:价值网络 ,评估当前策略下的状态价值,为 Actor 提供低方差学习信号。
最简单的一步 TD Actor-Critic 用 TD 目标
和 TD 误差
其中 表示 MDP 真正终止。Gymnasium 的时间限制 truncated=True 只结束当前回合,并不代表终止状态,因此价值目标通常仍应从 自举。
2. Actor 与 Critic 的更新
2.1 Critic 更新
Critic 最小化一步 TD 目标与当前估计之间的均方误差:
stopgrad 表示目标分支不参与反向传播。这是半梯度 TD 更新;基础实现不需要额外的目标价值网络。
2.2 Actor 更新
Actor 将停止梯度后的 当作优势函数 的一步估计:
若 ,所选动作的结果好于 Critic 原先的预期,更新会提高该动作的概率;若 ,则降低其概率。常见实现还加入熵正则以避免策略过早变得确定:
3. 与 REINFORCE 的差异
- 回报估计:REINFORCE 使用完整蒙特卡洛回报;一步 Actor-Critic 使用自举的 TD 误差。
- 偏差与方差:蒙特卡洛估计通常无偏但方差高;TD 估计引入价值近似和自举偏差,但方差更低。
- 更新时间:REINFORCE 通常每回合更新;一步 Actor-Critic 每次环境交互后即可更新。
- 模型组成:REINFORCE 只需策略网络;Actor-Critic 还需训练价值网络。
- 数据约束:这里的两者都是同策略(on-policy)方法。基础 Actor-Critic 不能无条件重复使用旧经验,否则数据分布与当前策略不匹配。
4. CartPole-v1 自包含实现
下面代码只依赖 gymnasium、numpy 和 torch,包含训练和评估。训练时按策略分布采样,评估时选择概率最大的动作。所有随机源、环境重置和动作空间都显式设置种子。
import random
import gymnasium as gym
import numpy as np
import torch
import torch.nn as nn
from torch.distributions import Categorical
SEED = 42
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def set_seed(seed):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
class Actor(nn.Module):
def __init__(self, state_dim, action_dim, hidden_dim=128):
super().__init__()
self.network = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, action_dim),
)
def forward(self, state):
return self.network(state) # logits
class Critic(nn.Module):
def __init__(self, state_dim, hidden_dim=128):
super().__init__()
self.network = nn.Sequential(
nn.Linear(state_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, 1),
)
def forward(self, state):
return self.network(state)
class ActorCriticAgent:
def __init__(
self,
state_dim,
action_dim,
actor_lr=1e-3,
critic_lr=2e-3,
gamma=0.99,
entropy_coef=0.01,
):
self.actor = Actor(state_dim, action_dim).to(DEVICE)
self.critic = Critic(state_dim).to(DEVICE)
self.actor_optimizer = torch.optim.Adam(
self.actor.parameters(), lr=actor_lr
)
self.critic_optimizer = torch.optim.Adam(
self.critic.parameters(), lr=critic_lr
)
self.gamma = gamma
self.entropy_coef = entropy_coef
def sample_action(self, state):
state_tensor = torch.as_tensor(
state, dtype=torch.float32, device=DEVICE
).unsqueeze(0)
distribution = Categorical(logits=self.actor(state_tensor))
action = distribution.sample()
return action.item(), distribution.log_prob(
action
), distribution.entropy()
@torch.no_grad()
def greedy_action(self, state):
state_tensor = torch.as_tensor(
state, dtype=torch.float32, device=DEVICE
).unsqueeze(0)
return self.actor(state_tensor).argmax(dim=1).item()
def update(
self,
state,
log_prob,
entropy,
reward,
next_state,
terminated,
):
state_tensor = torch.as_tensor(
state, dtype=torch.float32, device=DEVICE
).unsqueeze(0)
next_state_tensor = torch.as_tensor(
next_state, dtype=torch.float32, device=DEVICE
).unsqueeze(0)
reward_tensor = torch.tensor(
[[reward]], dtype=torch.float32, device=DEVICE
)
terminal_tensor = torch.tensor(
[[float(terminated)]],
dtype=torch.float32,
device=DEVICE,
)
value = self.critic(state_tensor)
with torch.no_grad():
next_value = self.critic(next_state_tensor)
td_target = (
reward_tensor
+ self.gamma
* (1.0 - terminal_tensor)
* next_value
)
td_error = td_target - value
critic_loss = 0.5 * td_error.pow(2).mean()
self.critic_optimizer.zero_grad()
critic_loss.backward()
nn.utils.clip_grad_norm_(
self.critic.parameters(), max_norm=1.0
)
self.critic_optimizer.step()
actor_loss = (
-log_prob * td_error.detach().squeeze(0)
- self.entropy_coef * entropy
).mean()
self.actor_optimizer.zero_grad()
actor_loss.backward()
nn.utils.clip_grad_norm_(
self.actor.parameters(), max_norm=1.0
)
self.actor_optimizer.step()
return actor_loss.item(), critic_loss.item()
def train(episodes=1000):
set_seed(SEED)
env = gym.make("CartPole-v1")
env.action_space.seed(SEED)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = ActorCriticAgent(state_dim, action_dim)
returns = []
for episode in range(episodes):
state, _ = env.reset(seed=SEED if episode == 0 else None)
episode_return = 0.0
while True:
action, log_prob, entropy = agent.sample_action(state)
next_state, reward, terminated, truncated, _ = env.step(
action
)
agent.update(
state,
log_prob,
entropy,
reward,
next_state,
terminated,
)
state = next_state
episode_return += reward
if terminated or truncated:
break
returns.append(episode_return)
if (episode + 1) % 50 == 0:
mean_return = np.mean(returns[-50:])
print(
f"Episode {episode + 1}: "
f"mean return = {mean_return:.1f}"
)
env.close()
return agent, returns
def evaluate(agent, episodes=20):
env = gym.make("CartPole-v1")
scores = []
for episode in range(episodes):
state, _ = env.reset(seed=SEED + 1000 + episode)
episode_return = 0.0
while True:
action = agent.greedy_action(state)
state, reward, terminated, truncated, _ = env.step(
action
)
episode_return += reward
if terminated or truncated:
break
scores.append(episode_return)
env.close()
mean_score = float(np.mean(scores))
std_score = float(np.std(scores))
print(
f"Evaluation over {episodes} episodes: "
f"{mean_score:.1f} ± {std_score:.1f}"
)
return mean_score, std_score
if __name__ == "__main__":
trained_agent, training_returns = train()
evaluate(trained_agent)
固定种子只能提高可复现性,不能保证跨 PyTorch 版本、设备和底层算子得到完全相同的曲线。单步 Actor-Critic 本身也可能对学习率与随机轨迹较敏感;更稳定的实践通常会使用多步回报、优势标准化或并行环境。