How to get weight in each layer and epoch then save in file

Question

I am trying to get the weight value in each layer in each epoch and then save it in a file. I am trying to implement the code proposed by Eric M on this page. But while still trying to get the weight value, I get an error like this:

<ipython-input-15-81ab617ec631> in on_epoch_end(self, epoch, logs)
w = self.model.layers[layer_i].get_weights()[0]
IndexError: list index out of range

What happened? Because layer_i only gets the number of layers I use. Is it because of the attention layer that I use? I also can't save it to a file because I don't know what the code produces.

Here's the callbacks and model that I use:

class GetWeights(keras.callbacks.Callback):
  def __init__(self):
    super(GetWeights, self).__init__()
    self.weight_dict = {}
  def on_epoch_end(self, epoch, logs=None):
    for layer_i in range(len(self.model.layers)):
      w = self.model.layers[layer_i].get_weights()[0]
      b = self.model.layers[layer_i].get_weights()[1]
      heat_map = sb.heatmap(w)
      pyplot.show()
      print('Layer %s has weights of shape %s and biases of shape %s' %(layer_i, np.shape(w), np.shape(b)))
      if epoch == 0:
        # create array to hold weights and biases
        self.weight_dict['w_'+str(layer_i+1)] = w
        self.weight_dict['b_'+str(layer_i+1)] = b
      else:
        # append new weights to previously-created weights array
        self.weight_dict['w_'+str(layer_i+1)] = np.dstack(
            (self.weight_dict['w_'+str(layer_i+1)], w))
        # append new weights to previously-created weights array
        self.weight_dict['b_'+str(layer_i+1)] = np.dstack(
            (self.weight_dict['b_'+str(layer_i+1)], b))

gw = GetWeights()
model = Sequential() 
model.add(LSTM(hidden_units_masukan, input_shape=(n_timesteps,n_features), return_sequences=True))
model.add(LSTM(hidden_units_masukan, input_shape=(n_timesteps,n_features), return_sequences=True))
model.add(Dropout(dropout_masukan))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(Dense(n_outputs, activation=activation_masukan))
model.compile(loss='categorical_crossentropy', optimizer=optimizer_masukan, metrics=['accuracy'])
model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size_masukan, verbose=verbose, callbacks=[gw],)

Answer 1

The problem is you are trying to extract the weights and biases from each layer in your model, but a Dropout layer does not have any weights. This is why you get this error message. You need to exclude this layer. Here is a working example:

import tensorflow as tf
import seaborn as sb
import matplotlib.pyplot as plt
import numpy as np

class attention(tf.keras.layers.Layer):
  def __init__(self, return_sequences=True):
      self.return_sequences = return_sequences
      super(attention,self).__init__()
  def build(self, input_shape):
      self.W=self.add_weight(name="att_weight", shape=(input_shape[-1],1),
                            initializer="normal")
      self.b=self.add_weight(name="att_bias", shape=(input_shape[1],1),
                            initializer="zeros")
      super(attention,self).build(input_shape)
  def call(self, x):
      e = tf.keras.backend.tanh(tf.keras.backend.dot(x,self.W)+self.b)
      a = tf.keras.backend.softmax(e, axis=1)
      output = x*a
      if self.return_sequences:
          return output
      return tf.keras.backend.sum(output, axis=1)

class GetWeights(tf.keras.callbacks.Callback):
  def __init__(self):
    super(GetWeights, self).__init__()
    self.weight_dict = {}
  def on_epoch_end(self, epoch, logs=None):
    drop_out_index = 2
    for i, layer in enumerate(self.model.layers):
      if drop_out_index != i:
        w = layer.get_weights()[0]
        b = layer.get_weights()[1]
        heat_map = sb.heatmap(w)
        plt.show()
        print('Layer %s has weights of shape %s and biases of shape %s' %(i, np.shape(w), np.shape(b)))
        if epoch == 0:
          # create array to hold weights and biases
          self.weight_dict['w_'+str(i+1)] = w
          self.weight_dict['b_'+str(i+1)] = b
        else:
          # append new weights to previously-created weights array
          self.weight_dict['w_'+str(i+1)] = np.dstack(
              (self.weight_dict['w_'+str(i+1)], w))
          # append new weights to previously-created weights array
          self.weight_dict['b_'+str(i+1)] = np.dstack(
              (self.weight_dict['b_'+str(i+1)], b))

gw = GetWeights()
model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(32, input_shape=(5,10), return_sequences=True))
model.add(tf.keras.layers.LSTM(32, return_sequences=True))
model.add(tf.keras.layers.Dropout(0.1))
model.add(attention(return_sequences=False)) # receive 3D and output 2D
model.add(tf.keras.layers.Dense(3, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

trainx = tf.random.normal((25, 5, 10))
trainy = tf.random.uniform((25, 3), maxval=3)
model.fit(trainx, trainy, epochs=1, batch_size=4, callbacks=[gw])

Model: "sequential_11"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 lstm_22 (LSTM)              (None, 5, 32)             5504      
                                                                 
 lstm_23 (LSTM)              (None, 5, 32)             8320      
                                                                 
 dropout_11 (Dropout)        (None, 5, 32)             0         
                                                                 
 attention_11 (attention)    (None, 32)                37        
                                                                 
 dense_11 (Dense)            (None, 3)                 99        
                                                                 
=================================================================
Total params: 13,960
Trainable params: 13,960
Non-trainable params: 0
_________________________________________________________________
7/7 [==============================] - ETA: 0s - loss: 4.4367 - accuracy: 0.3200     

Layer 0 has weights of shape (10, 128) and biases of shape (32, 128)

Layer 1 has weights of shape (32, 128) and biases of shape (32, 128)

Layer 3 has weights of shape (32, 1) and biases of shape (5, 1)

Layer 4 has weights of shape (32, 3) and biases of shape (3,)
7/7 [==============================] - 5s 265ms/step - loss: 4.4367 - accuracy: 0.3200
<keras.callbacks.History at 0x7f3914737b10>