I am trying to get the weight value in each layer in each epoch and then save it in a file. I am trying to implement the code proposed by Eric M on this page. But while still trying to get the weight value, I get an error like this:
<ipython-input-15-81ab617ec631> in on_epoch_end(self, epoch, logs)
w = self.model.layers[layer_i].get_weights()[0]
IndexError: list index out of range
What happened? Because layer_i only gets the number of layers I use. Is it because of the attention layer that I use? I also can't save it to a file because I don't know what the code produces.
Here's the callbacks and model that I use:
class GetWeights(keras.callbacks.Callback):
def __init__(self):
super(GetWeights, self).__init__()
self.weight_dict = {}
def on_epoch_end(self, epoch, logs=None):
for layer_i in range(len(self.model.layers)):
w = self.model.layers[layer_i].get_weights()[0]
b = self.model.layers[layer_i].get_weights()[1]
heat_map = sb.heatmap(w)
pyplot.show()
print('Layer %s has weights of shape %s and biases of shape %s' %(layer_i, np.shape(w), np.shape(b)))
if epoch == 0:
# create array to hold weights and biases
self.weight_dict['w_' str(layer_i 1)] = w
self.weight_dict['b_' str(layer_i 1)] = b
else:
# append new weights to previously-created weights array
self.weight_dict['w_' str(layer_i 1)] = np.dstack(
(self.weight_dict['w_' str(layer_i 1)], w))
# append new weights to previously-created weights array
self.weight_dict['b_' str(layer_i 1)] = np.dstack(
(self.weight_dict['b_' str(layer_i 1)], b))
gw = GetWeights()
model = Sequential()
model.add(LSTM(hidden_units_masukan, input_shape=(n_timesteps,n_features), return_sequences=True))
model.add(LSTM(hidden_units_masukan, input_shape=(n_timesteps,n_features), return_sequences=True))
model.add(Dropout(dropout_masukan))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(Dense(n_outputs, activation=activation_masukan))
model.compile(loss='categorical_crossentropy', optimizer=optimizer_masukan, metrics=['accuracy'])
model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size_masukan, verbose=verbose, callbacks=[gw],)
CodePudding user response：
The problem is you are trying to extract the weights and biases from each layer in your model, but a Dropout layer does not have any weights. This is why you get this error message. You need to exclude this layer. Here is a working example:
import tensorflow as tf
import seaborn as sb
import matplotlib.pyplot as plt
import numpy as np
class attention(tf.keras.layers.Layer):
def __init__(self, return_sequences=True):
self.return_sequences = return_sequences
super(attention,self).__init__()
def build(self, input_shape):
self.W=self.add_weight(name="att_weight", shape=(input_shape[-1],1),
initializer="normal")
self.b=self.add_weight(name="att_bias", shape=(input_shape[1],1),
initializer="zeros")
super(attention,self).build(input_shape)
def call(self, x):
e = tf.keras.backend.tanh(tf.keras.backend.dot(x,self.W) self.b)
a = tf.keras.backend.softmax(e, axis=1)
output = x*a
if self.return_sequences:
return output
return tf.keras.backend.sum(output, axis=1)
class GetWeights(tf.keras.callbacks.Callback):
def __init__(self):
super(GetWeights, self).__init__()
self.weight_dict = {}
def on_epoch_end(self, epoch, logs=None):
drop_out_index = 2
for i, layer in enumerate(self.model.layers):
if drop_out_index != i:
w = layer.get_weights()[0]
b = layer.get_weights()[1]
heat_map = sb.heatmap(w)
plt.show()
print('Layer %s has weights of shape %s and biases of shape %s' %(i, np.shape(w), np.shape(b)))
if epoch == 0:
# create array to hold weights and biases
self.weight_dict['w_' str(i 1)] = w
self.weight_dict['b_' str(i 1)] = b
else:
# append new weights to previously-created weights array
self.weight_dict['w_' str(i 1)] = np.dstack(
(self.weight_dict['w_' str(i 1)], w))
# append new weights to previously-created weights array
self.weight_dict['b_' str(i 1)] = np.dstack(
(self.weight_dict['b_' str(i 1)], b))
gw = GetWeights()
model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(32, input_shape=(5,10), return_sequences=True))
model.add(tf.keras.layers.LSTM(32, return_sequences=True))
model.add(tf.keras.layers.Dropout(0.1))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(tf.keras.layers.Dense(3, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
trainx = tf.random.normal((25, 5, 10))
trainy = tf.random.uniform((25, 3), maxval=3)
model.fit(trainx, trainy, epochs=1, batch_size=4, callbacks=[gw])
Model: "sequential_11"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm_22 (LSTM) (None, 5, 32) 5504
lstm_23 (LSTM) (None, 5, 32) 8320
dropout_11 (Dropout) (None, 5, 32) 0
attention_11 (attention) (None, 32) 37
dense_11 (Dense) (None, 3) 99
=================================================================
Total params: 13,960
Trainable params: 13,960
Non-trainable params: 0
_________________________________________________________________
7/7 [==============================] - ETA: 0s - loss: 4.4367 - accuracy: 0.3200
Layer 0 has weights of shape (10, 128) and biases of shape (32, 128)
Layer 1 has weights of shape (32, 128) and biases of shape (32, 128)
Layer 3 has weights of shape (32, 1) and biases of shape (5, 1)
Layer 4 has weights of shape (32, 3) and biases of shape (3,)
7/7 [==============================] - 5s 265ms/step - loss: 4.4367 - accuracy: 0.3200
<keras.callbacks.History at 0x7f3914737b10>