python - TensorFlow 的 LSTMCell 究竟是如何运作的？

Question

我尝试从 TensorFlow 重现 LSTMCell 生成的结果，以确保我知道它的作用。

这是我的 TensorFlow 代码:

num_units = 3
lstm = tf.nn.rnn_cell.LSTMCell(num_units = num_units)

timesteps = 7
num_input = 4
X = tf.placeholder("float", [None, timesteps, num_input])
x = tf.unstack(X, timesteps, 1)
outputs, states = tf.contrib.rnn.static_rnn(lstm, x, dtype=tf.float32)

sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)

x_val = np.random.normal(size = (1, 7, num_input))

res = sess.run(outputs, feed_dict = {X:x_val})

for e in res:
    print e

这是它的输出:

[[-0.13285545 -0.13569424 -0.23993783]]
[[-0.04818152  0.05927373  0.2558436 ]]
[[-0.13818116 -0.13837864 -0.15348436]]
[[-0.232219    0.08512601  0.05254192]]
[[-0.20371495 -0.14795329 -0.2261929 ]]
[[-0.10371902 -0.0263292  -0.0914975 ]]
[[0.00286371 0.16377522 0.059478  ]]

这是我自己的实现:

n_steps, _ = X.shape
h = np.zeros(shape = self.hid_dim)
c = np.zeros(shape = self.hid_dim)

for i in range(n_steps):
    x = X[i,:]

    vec = np.concatenate([x, h])
    #vec = np.concatenate([h, x])
    gs = np.dot(vec, self.kernel) + self.bias


    g1 = gs[0*self.hid_dim : 1*self.hid_dim]
    g2 = gs[1*self.hid_dim : 2*self.hid_dim]
    g3 = gs[2*self.hid_dim : 3*self.hid_dim]
    g4 = gs[3*self.hid_dim : 4*self.hid_dim]

    I = vsigmoid(g1)
    N = np.tanh(g2)
    F = vsigmoid(g3)
    O = vsigmoid(g4)

    c = c*F + I*N

    h = O * np.tanh(c)

    print h

这是它的输出:

[-0.13285543 -0.13569425 -0.23993781]
[-0.01461723  0.08060743  0.30876374]
[-0.13142865 -0.14921292 -0.16898363]
[-0.09892188  0.11739943  0.08772941]
[-0.15569218 -0.15165766 -0.21918869]
[-0.0480604  -0.00918626 -0.06084118]
[0.0963612  0.1876516  0.11888081]

您可能会注意到，我能够重现第一个隐藏向量，但第二个和后面的所有向量都不同。我错过了什么？

Answer 1

我检查了 this 链接，您的代码几乎是完美的，但您忘记在这一行中添加忘记偏差值(默认 1.0)F = vsigmoid(g3) 它实际上是 F = vsigmoid(g3 +self.forget_bias) 或在您的情况下为 1 F = vsigmoid(g3+1)

这是我的 numpy 小鬼:

import numpy as np
import tensorflow as tf

num_units = 3
lstm = tf.nn.rnn_cell.LSTMCell(num_units = num_units)
batch=1
timesteps = 7
num_input = 4
X = tf.placeholder("float", [batch, timesteps, num_input])
x = tf.unstack(X, timesteps, 1)
outputs, states = tf.contrib.rnn.static_rnn(lstm, x, dtype=tf.float32)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
x_val = np.reshape(range(28),[batch, timesteps, num_input])
res = sess.run(outputs, feed_dict = {X:x_val})
for e in res:
    print(e)
print("\nmy imp\n")
#my impl
def sigmoid(x):
    return 1/(1+np.exp(-x))

kernel,bias=sess.run([lstm._kernel,lstm._bias])
f_b_=lstm._forget_bias
c,h=np.zeros([batch,num_input-1]),np.zeros([batch,num_input-1])
for step in range(timesteps):
    inpt=np.split(x_val,7,1)[step][0]
    lstm_mtrx=np.matmul(np.concatenate([inpt,h],1),kernel)+bias
    i,j,f,o=np.split(lstm_mtrx,4,1)
    c=sigmoid(f+f_b_)*c+sigmoid(i)*np.tanh(j)
    h=sigmoid(o)*np.tanh(c)
    print(h)

输出:

[[ 0.06964055 -0.06541953 -0.00682676]]
[[ 0.005264   -0.03234607  0.00014838]]
[[ 1.617855e-04 -1.316892e-02  8.596722e-06]]
[[ 3.9425286e-06 -5.1347450e-03  7.5078127e-08]]
[[ 8.7508155e-08 -1.9560163e-03  6.3853928e-10]]
[[ 1.8867894e-09 -7.3784427e-04  5.8551406e-12]]
[[ 4.0385355e-11 -2.7728223e-04  5.3957669e-14]]

my imp

[[ 0.06964057 -0.06541953 -0.00682676]]
[[ 0.005264   -0.03234607  0.00014838]]
[[ 1.61785520e-04 -1.31689185e-02  8.59672610e-06]]
[[ 3.94252745e-06 -5.13474567e-03  7.50781122e-08]]
[[ 8.75080644e-08 -1.95601574e-03  6.38539112e-10]]
[[ 1.88678843e-09 -7.37844070e-04  5.85513438e-12]]
[[ 4.03853841e-11 -2.77282006e-04  5.39576024e-14]]