【发布时间】：2019-07-17 10:06:46
【问题描述】：
我正在尝试使用 tensorflow 训练一个在线近端策略优化模型，但过了一会儿，tensorflow 会话开始返回 NaN。这会导致我的代理使用这些 nans step，最终整个事情变得一团糟。
来自控制台的短 sn-p：
Action Taken [2. 1.3305835 0.9937418]
Observation [ 0.69689728 -0.46114012 -11.39961704 -0.05004346 -0.05004346
0.74720544 3.49857114 3.05071477 -1.10276782 -9.71530186]
Reward Gained -0.023699851569145534
Action Taken [2. 0.62562937 1.0081608 ]
Observation [ 0.71591491 -0.47488649 11.84026042 -0.05004346 -0.05004346 0.75886336
3.49857114 3.07180685 -1.12458586 -9.84382414]
Reward Gained -0.015462812448075767
Action Taken [nan nan nan]
Observation [ nan nan nan -0.05004346 -0.05004346 nan
nan nan nan nan]
Reward Gained nan
Action Taken [nan nan nan]
Observation [ nan nan nan -0.05004346 -0.05004346 nan
nan nan nan nan]
Reward Gained nan
我的代码 [已更新]：
import gym
import numpy as np
import tensorflow as tf
import rocket_lander_gym
EP_LEN = 200
GAMMA = 0.9
SL_LR = 1e-4
CR_LR = 1e-4
BATCH = 5
ACTOR_UPDATE_STEPS = 20
CRITIC_UPDATE_STEPS = 20
STATE_DIM, ACT_DIM = 10, 3
METHOD = [
dict(name='kl_penalty', kl_target=0.01, lam=0.5),
dict(name='clip', epsilon=0.2),
][1]
PRINT_DEBUG_MSG = True
class PPO:
def __init__(self):
self.tfsess = tf.Session()
self.tf_state = tf.placeholder(tf.float32, [None, STATE_DIM], 'state')
# Critic (value network)
with tf.variable_scope('critic'):
# Layers
l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu)
# Value
self.value = tf.layers.dense(l1, 1)
# Discounted reward: reward in the furture
self.tf_dreward = tf.placeholder(tf.float32, [None, 1], 'discounted_reward')
# Advantage: determine quality of action
self.advantage = self.tf_dreward - self.value
# Loss function: minimize the advantage over time
# The loss function is a mean squared error
self.loss = tf.reduce_mean(tf.square(self.advantage))
# Gradient descent using Adam optimizer
self.train_opt = tf.train.AdamOptimizer(CR_LR)
gradients, variables = zip(*self.train_opt.compute_gradients(self.loss))
gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))
# Actor (policy network)
pi, pi_params = self.tinynn('pi', trainable=True)
old_pi, old_pi_params = self.tinynn('old_pi', trainable=False)
# Sample actions from both the old and the new policy networks
with tf.variable_scope('sample_action'):
# Choose an action from the distribution learnt
self.sample_operation = tf.squeeze(pi.sample(1), axis=0)
with tf.variable_scope('update_old_pi'):
# Choose an action from the distribution learnt
self.update_old_pi_operation = [old_pi.assign(p) for p, old_pi in zip(pi_params, old_pi_params)]
# Placeholder for the action and the advantage
self.tf_action = tf.placeholder(tf.float32, [None, ACT_DIM], 'action')
self.tf_advantage = tf.placeholder(tf.float32, [None, 1], 'advantage')
# Compute loss function
with tf.variable_scope('loss'):
with tf.variable_scope('surrogate'):
ratio = pi.prob(self.tf_advantage) / old_pi.prob(self.tf_advantage)
surrogate = ratio * self.tf_advantage
# KL penalty
if METHOD['name'] == 'kl_penalty':
# Lambda
self.tf_lambda = tf.placeholder(tf.float32, None, 'lambda')
# Compute KL divergence between old and new policy
kl = tf.contrib.distributions.kl_divergence(old_pi, pi)
# Get mean
self.kl_mean = tf.reduce_mean(kl)
# Compute loss using surrogate
self.aloss = -(tf.reduce_mean(surrogate - self.tf_lambda * kl))
else:
self.aloss = -tf.reduce_mean(tf.minimum(surrogate, tf.clip_by_value(ratio, 1.-METHOD['epsilon'], 1.+METHOD['epsilon']) * self.tf_advantage))
# Minimize the loss using gradient descent
with tf.variable_scope('atrain'):
self.atrain_operation = tf.train.AdamOptimizer(SL_LR)
gradients, variables = zip(*self.atrain_operation.compute_gradients(self.aloss))
gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
self.atrain_operation = self.atrain_operation.apply_gradients(zip(gradients, variables))
# Write to disk
tf.summary.FileWriter("log/", self.tfsess.graph)
# Run the session
self.tfsess.run(tf.global_variables_initializer())
def update(self, state, action, reward):
self.tfsess.run(self.update_old_pi_operation)
advantage = self.tfsess.run(self.advantage, {self.tf_state: state, self.tf_dreward: reward})
# Update actor (policy)
if METHOD['name'] == 'kl_penalty':
for _ in range(ACTOR_UPDATE_STEPS):
_, kl = self.tfsess.run([self.atrain_operation, self.kl_mean], {self.tf_state: state, self.tf_action: action, tf_advantage: advantage, self.tf_lambda: METHOD['lam']})
if kl > 4*METHOD['kl_target']:
break
if kl < METHOD['kl_target'] / 1.5:
# Adaptive lambda
METHOD['lam'] /= 2
elif kl > METHOD['kl_target'] * 1.5:
METHOD['lam'] *= 2
# Lambda might explode, we need to clip it
METHOD['lam'] = np.clip(METHOD['lam'], 1e-4, 10)
else:
[self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
# Update critic (value)
[self.tfsess.run(self.train_opt, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]
def tinynn(self, name, trainable):
with tf.variable_scope(name):
l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu, trainable=trainable)
mu = 2 * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, trainable=trainable)
sigma = tf.layers.dense(l1, ACT_DIM, tf.nn.softplus, trainable=trainable)
norm_dist = tf.distributions.Normal(loc=mu, scale=sigma)
params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)
return norm_dist, params
def choose_action(self, state):
state = state[np.newaxis, :]
action = self.tfsess.run(self.sample_operation, {self.tf_state: state})[0]
return np.clip(action, -1, 1)
def get_value(self, state):
if state.ndim < 2: state = state[np.newaxis, :]
return self.tfsess.run(self.value, {self.tf_state: state})[0, 0]
def train(self, env, ppo, epochs, render=True):
# Rewards
all_ep_r = []
# Training loop
for ep in range(epochs):
# Initial state
s = env.reset()
# States, actions and rewards
buffer_s, buffer_a, buffer_r = [], [], []
# Initial reward
ep_r = 0
# For a single episode
for t in range(EP_LEN):
if render:
# Render the environment
env.render()
# Choose best action
a = ppo.choose_action(s)
# State,reward,done,info
s_, r, done, _ = env.step(a)
if PRINT_DEBUG_MSG:
print("Action Taken ",a)
print("Observation ",s_)
print("Reward Gained ",r, end='\n\n')
# Add to buffers
buffer_s.append(s)
buffer_a.append(a)
buffer_r.append((r+8)/8) # normalize reward, find to be useful
s = s_
# Total reward
ep_r += r
# Update PPO
if (t+1) % BATCH == 0 or t == EP_LEN - 1:
# Get value
v_s_ = ppo.get_value(s_)
# Discounted reward
discounted_r = []
# Update rewards
for r in buffer_r[::-1]:
v_s_ = r + GAMMA * v_s_
discounted_r.append(v_s_)
discounted_r.reverse()
# Buffer states actions rewards
bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, np.newaxis]
buffer_s, buffer_a, buffer_r = [], [], []
ppo.update(bs, ba, br)
# Check if done
if done:
print("Simulation done.")
break
# Append episode rewards
if ep == 0: all_ep_r.append(ep_r)
else: all_ep_r.append(all_ep_r[-1]*0.9 + ep_r*0.1)
# Close the environment
env.close()
# Return all episode rewards
return all_ep_r
if __name__ == '__main__':
ppo = PPO()
env = gym.make('RocketLander-v0')
reward = ppo.train(env, ppo, 100)
print(reward)
我尝试过的：
- 我已尝试降低我的演员和评论家网络的学习率，但 nans 仍然存在。
- 减少了
BATCH编号，以便 PPO 更新得更快。
我已经被这个问题困扰了好几个小时了，我在网上找不到任何解决方案。我也是新手，如有错误请见谅。
更新：追溯
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
return fn(*args)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Found Inf or NaN global norm. : Tensor had NaN values
[[{{node atrain/VerifyFinite/CheckNumerics}} = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 209, in <module>
reward = ppo.train(env, ppo, 100)
File "main.py", line 191, in train
ppo.update(bs, ba, br)
File "main.py", line 118, in update
[self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
File "main.py", line 118, in <listcomp>
[self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
run_metadata_ptr)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Found Inf or NaN global norm. : Tensor had NaN values
[[node atrain/VerifyFinite/CheckNumerics (defined at main.py:90) = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
Caused by op 'atrain/VerifyFinite/CheckNumerics', defined at:
File "main.py", line 207, in <module>
ppo = PPO()
File "main.py", line 90, in __init__
gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/clip_ops.py", line 265, in clip_by_global_norm
"Found Inf or NaN global norm.")
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/numerics.py", line 47, in verify_tensor_all_finite
verify_input = array_ops.check_numerics(t, message=msg)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 817, in check_numerics
"CheckNumerics", tensor=tensor, message=message, name=name)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in __init__
self._traceback = tf_stack.extract_stack()
InvalidArgumentError (see above for traceback): Found Inf or NaN global norm. : Tensor had NaN values
[[node atrain/VerifyFinite/CheckNumerics (defined at main.py:90) = CheckNumerics[T=DT_FLOAT, message="Found Inf or NaN global norm.", _device="/job:localhost/replica:0/task:0/device:CPU:0"](atrain/global_norm/global_norm)]]
【问题讨论】：
-
在更简单的环境中尝试过您的网络吗？像连续动作的山地车。
-
@SridharThiagarajan 不，但它在技术上应该可以工作。此处提供的代码实际上是此视频中代码的略微修改版本：youtu.be/09OMoGqHexQ 和 github.com/MorvanZhou/Reinforcement-learning-with-tensorflow/…
-
我鼓励您先尝试更简单的域，以验证代码是否正常，或者它是域相关问题。
-
@SridharThiagarajan 我刚刚使用 MountainCarContinuous-v0 进行了尝试，它适用于几集，然后我遇到了同样的问题（模型返回了 NaN）。
-
在应用之前尝试剪切渐变。检查渐变是否爆炸。
标签： python numpy tensorflow reinforcement-learning openai-gym