I have finetuned a model and I have saved all weights with name checkpoint_vgg16.h5. I want to apply gradCAM on my finetuned model (not pretrained VGG model) and if it required that using something for GradCAM while training phase, I didn't use anything.
Implementation details like that:
base_model = keras.applications.VGG16(include_top=False, input_shape=(128, 88, 3))
model1 = Sequential()
for layer in base_model.layers:
model1.add(layer)
model1.add(Flatten())
model1.add(Dense(4096, activation="relu"))
model1.add(Dense(124, activation="softmax"))
model1.load_weights("checkpoint_vgg16.h5")
model = Model(model1.input, model1.layers[-1].output)
GradCAM class:
class GradCAM:
def __init__(self, model, classIdx, layerName=None):
# store the model, the class index used to measure the class
# activation map, and the layer to be used when visualizing
# the class activation map
self.model = model
self.classIdx = classIdx
self.layerName = layerName
# if the layer name is None, attempt to automatically find
# the target output layer
if self.layerName is None:
self.layerName = self.find_target_layer()
def find_target_layer(self):
# attempt to find the final convolutional layer in the network
# by looping over the layers of the network in reverse order
for layer in reversed(self.model.layers):
# check to see if the layer has a 4D output
if len(layer.output_shape) == 4:
return layer.name
# otherwise, we could not find a 4D layer so the GradCAM
# algorithm cannot be applied
raise ValueError("Could not find 4D layer. Cannot apply GradCAM.")
def compute_heatmap(self, image, eps=1e-8):
# construct our gradient model by supplying (1) the inputs
# to our pre-trained model, (2) the output of the (presumably)
# final 4D layer in the network, and (3) the output of the
# softmax activations from the model
gradModel = Model(
inputs=[self.model.inputs],
outputs=[self.model.get_layer(self.layerName).output, self.model.output])
# record operations for automatic differentiation
with tf.GradientTape() as tape:
# cast the image tensor to a float-32 data type, pass the
# image through the gradient model, and grab the loss
# associated with the specific class index
inputs = tf.cast(image, tf.float32)
(convOutputs, predictions) = gradModel(inputs)
loss = predictions[:, tf.argmax(predictions[0])]
print(loss)
# use automatic differentiation to compute the gradients
grads = tape.gradient(loss, convOutputs)
print(grads)
# compute the guided gradients
castConvOutputs = tf.cast(convOutputs > 0, "float32")
castGrads = tf.cast(grads > 0, "float32")
guidedGrads = castConvOutputs * castGrads * grads
# the convolution and guided gradients have a batch dimension
# (which we don't need) so let's grab the volume itself and
# discard the batch
convOutputs = convOutputs[0]
guidedGrads = guidedGrads[0]
# compute the average of the gradient values, and using them
# as weights, compute the ponderation of the filters with
# respect to the weights
weights = tf.reduce_mean(guidedGrads, axis=(0, 1))
cam = tf.reduce_sum(tf.multiply(weights, convOutputs), axis=-1)
# grab the spatial dimensions of the input image and resize
# the output class activation map to match the input image
# dimensions
(w, h) = (image.shape[2], image.shape[1])
heatmap = cv2.resize(cam.numpy(), (w, h))
# normalize the heatmap such that all values lie in the range
# [0, 1], scale the resulting values to the range [0, 255],
# and then convert to an unsigned 8-bit integer
numer = heatmap - np.min(heatmap)
denom = (heatmap.max() - heatmap.min()) + eps
heatmap = numer / denom
heatmap = (heatmap * 255).astype("uint8")
# return the resulting heatmap to the calling function
return heatmap
def overlay_heatmap(self, heatmap, image, alpha=0.5,
colormap=cv2.COLORMAP_VIRIDIS):
# apply the supplied color map to the heatmap and then
# overlay the heatmap on the input image
heatmap = cv2.applyColorMap(heatmap, colormap)
output = cv2.addWeighted(image, alpha, heatmap, 1 - alpha, 0)
# return a 2-tuple of the color mapped heatmap and the output,
# overlaid image
return (heatmap, output)
For the prediction part:
image = cv2.imread('sample.jpg')
image = image.astype('float32') / 255
image = np.expand_dims(image, axis=0)
preds = model.predict(image)
i = np.argmax(preds[0])
icam = GradCAM(model, i, 'block5_conv3') #block5_conv3: the last conv block in my model
print(icam)
heatmap = icam.compute_heatmap(image)
heatmap = cv2.resize(heatmap, (128, 88))
image = cv2.imread('sample.jpg')
(heatmap, output) = icam.overlay_heatmap(heatmap, image, alpha=0.5)
fig, ax = plt.subplots(1, 3)
ax[0].imshow(heatmap)
ax[1].imshow(image)
ax[2].imshow(output)
However, I get following error:
castGrads = tf.cast(grads > 0, "float32") TypeError: '>' not supported between instances of 'NoneType' and 'int'
The grads = tape.gradient(loss, convOutputs) code line which belongs to compute_heatmap method in GradCAM returns None.
How can I fix this error? I want to apply GradCAM on my model.
Ref. Grad-CAM class activation visualization
Also, this question is similar to mine but, solution didn't work for me.
