值在 for 循环内自动更新

Question

我试图实现值迭代算法。 我有一个网格

grid = [[0, 0, 0, +1],
    [0, "W", 0, -1],
    [0, 0, 0, 0]]

行动清单

actlist = {UP:1, DOWN:2, LEFT:3, RIGHT:4}

还有一个奖励函数

reward = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0,  0]] 

我写了一个函数 T，它返回 3 个元组的元组。

def T(i,j,actions):
if(i == 0 and j == 0):
    if(actions == UP):
        return (i,i,0.8),(i,i,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,i,0.1),(i+1,j,0.1)
elif (i == 0 and j == 1):
    if(actions == UP):
        return (i,i,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 0 and j == 2):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return(i+1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i+1,j,0.1)
elif(i == 0 and j == 3):
    if(actions == UP):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == DOWN):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == LEFT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == RIGHT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
# 2nd row
elif (i == 1 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j ==1):
    if(actions == UP):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
elif (i == 1 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j == 3):
    if(actions == UP):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == DOWN):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == LEFT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == RIGHT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)      
# 3rd row
elif(i == 2 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j+1,1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif (i == 2 and j == 1):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 2 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif(i == 2 and j == 3):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1) 

这个函数在值迭代函数中被调用：

def value_iteration():
U1 = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0, 0]] 
while True:
    U=U1.copy()
    delta = 0
    for i in range(len(grid)):
        for j in range(len(grid[i])):
            U1[i][j] = max(sum(p*(R(k,l)+gamma*U[k][l]) for (k,l,p) in T(i,j,a)) for a in actlist)
            print(i,j,U1[i][j],U[i][j])
            delta = max(delta, abs(U1[i][j] - U[i][j]))
    if delta <= epsilon*(1 - gamma)/gamma:
        return U

我在更新

U=U1.copy()

在while循环中。

问题是，输出是这样的：

0 0 0.0 0.0
0 1 0.0 0.0
0 2 0.0 0.0
0 3 1.0 1.0
1 0 0.0 0.0
1 2 0.0 0.0
1 3 -1.0 -1.0
2 0 0.0 0.0
2 1 0.0 0.0
2 2 0.7000000000000001 0.7000000000000001
2 3 0.9630000000000001 0.9630000000000001

但我没有在 for 循环中更新 U。 U 应该保持不变（即全为零），而 U1 应该只改变。为什么 U 在 for 循环中会自动设置为 U1 的值？

Answer 1

U1（和U）是一个列表列表，实际上是一个列表引用列表。

您正在（浅）复制外部列表，但副本的内容仍然是对相同内部列表的引用。

试试：

import copy
U = copy.deepcopy(U1)

然后看看会发生什么。 deepcopy 将正确递归地复制列表。

Answer 2

nimish 的回答可能是最 Pythonic 的，但是如果你需要创建一个列表的副本，你也可以直接将它解包到一个新数组中：

U = U1[:]

这会创建一个不应引用旧对象的新对象

mylist = [[1,1,1],[2,2,2],[3,3,3]]
otherlist= mylist[:]
otherlist[0] = [5,5,5]
mylist
# [[1, 1, 1], [2, 2, 2], [3, 3, 3]]