python - 值在 for 循环内自动更新

Question

我试图实现值迭代算法。 我有一个网格

grid = [[0, 0, 0, +1],
    [0, "W", 0, -1],
    [0, 0, 0, 0]]

Action 列表

actlist = {UP:1, DOWN:2, LEFT:3, RIGHT:4}

还有奖励函数

reward = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0,  0]] 

我编写了一个函数 T，它返回 3 个元组的元组。

def T(i,j,actions):
if(i == 0 and j == 0):
    if(actions == UP):
        return (i,i,0.8),(i,i,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,i,0.1),(i+1,j,0.1)
elif (i == 0 and j == 1):
    if(actions == UP):
        return (i,i,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 0 and j == 2):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return(i+1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i+1,j,0.1)
elif(i == 0 and j == 3):
    if(actions == UP):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == DOWN):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == LEFT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == RIGHT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
# 2nd row
elif (i == 1 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j ==1):
    if(actions == UP):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
elif (i == 1 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j == 3):
    if(actions == UP):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == DOWN):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == LEFT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == RIGHT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)      
# 3rd row
elif(i == 2 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j+1,1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif (i == 2 and j == 1):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 2 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif(i == 2 and j == 3):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1) 

该函数在值迭代函数中调用:

def value_iteration():
U1 = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0, 0]] 
while True:
    U=U1.copy()
    delta = 0
    for i in range(len(grid)):
        for j in range(len(grid[i])):
            U1[i][j] = max(sum(p*(R(k,l)+gamma*U[k][l]) for (k,l,p) in T(i,j,a)) for a in actlist)
            print(i,j,U1[i][j],U[i][j])
            delta = max(delta, abs(U1[i][j] - U[i][j]))
    if delta <= epsilon*(1 - gamma)/gamma:
        return U

我正在更新

U=U1.copy()

在 while 循环中。

问题是，输出如下所示:

0 0 0.0 0.0
0 1 0.0 0.0
0 2 0.0 0.0
0 3 1.0 1.0
1 0 0.0 0.0
1 2 0.0 0.0
1 3 -1.0 -1.0
2 0 0.0 0.0
2 1 0.0 0.0
2 2 0.7000000000000001 0.7000000000000001
2 3 0.9630000000000001 0.9630000000000001

但我没有在 for 循环内更新 U。 U 应该保持不变(这意味着全为零)，而 U1 应该只发生变化。为什么在 for 循环中 U 会自动设置为 U1 的值？

Answer 1

U1(和 U)是列表的列表，实际上是对列表的引用的列表。

您正在(浅)复制外部列表，但副本的内容仍然是对相同内部列表的引用。

尝试:

import copy
U = copy.deepcopy(U1)

看看会发生什么。 deepcopy 将正确地递归复制列表。