Initial implementation inspired by jacobgil's keras-grad-cam

Initial commit

Add initial Grad-CAM implementation and other initial files

Add requirements.txt for faster environment setup.

Fixes on top of https://github.com/jacobgil/keras-grad-cam (as of
today, July 12th 2021):
- jacobgil/keras-grad-cam#17 (comment)
- jacobgil/keras-grad-cam#21 (comment)

Add modifications to add thresholding

Add yapf config for vscode, move requirements.txt, and format code

Fix links to show sample images in keras-grad-cam

Move files src folder and modify gitignore

Convert folders into packages

Add preprocessing functions

Remove guided backprop code from grad-cam

Re-format grad-cam folder and add first version of algo

Remove keras_grad_cam examples and information

.gitignore

@@ -0,0 +1,9 @@
+*.DS_Store
+*.egg-info
+__pycache__
+docs/build
+.coverage
+htmlcov
+.docker
+.idea/
+grad_cam/models/

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "python.formatting.provider": "yapf",
+  "editor.formatOnSave": true
+}

README.rst

@@ -0,0 +1,21 @@
+pl-grad-cam
+================================
+
+.. contents:: Table of Contents
+
+Abstract
+--------
+ChRIS Plugin for Explainable AI visualization using the Grad-CAM algorithm
+
+Setup
+----
+Download the relevant machine learning models whose results will be used as input
+For example, for COVID-NET, download COVIDNet-CXR4-B from https://github.com/lindawangg/COVID-Net/blob/master/docs/models
+
+Note
+----
+Grad-CAM largely depends on the provided reference model, so make sure that the model that is used to determine the result that is used as input exactly matches the provided reference model.
+
+Acknowledgement
+---------------
+Jacob Gildenblat (jacobgil) for initial Grad-CAM implementation using keras: https://github.com/jacobgil/keras-grad-cam

grad_cam/data.py

@@ -0,0 +1,21 @@
+import cv2
+
+
+def crop_top(img, percent=0.15):
+    offset = int(img.shape[0] * percent)
+    return img[offset:]
+
+
+def central_crop(img):
+    size = min(img.shape[0], img.shape[1])
+    offset_h = int((img.shape[0] - size) / 2)
+    offset_w = int((img.shape[1] - size) / 2)
+    return img[offset_h:offset_h + size, offset_w:offset_w + size]
+
+
+def process_image_file(filepath, top_percent, size):
+    img = cv2.imread(filepath)
+    img = crop_top(img, percent=top_percent)
+    img = central_crop(img)
+    img = cv2.resize(img, (size, size))
+    return img

grad_cam/grad-cam.py

@@ -0,0 +1,210 @@
+import tensorflow as tf
+import os
+
+# To remove TF Warnings
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+import warnings
+
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+from keras.applications.vgg16 import (VGG16, preprocess_input,
+                                      decode_predictions)
+from keras.preprocessing import image
+from keras.layers.core import Lambda
+from keras.models import Model
+from numpy.lib.function_base import place
+from tensorflow.python.framework import ops
+import keras.backend as K
+
+import numpy as np
+import keras
+import sys
+import cv2
+
+
+def target_category_loss(x, category_index, nb_classes):
+    return tf.multiply(x, K.one_hot([category_index], nb_classes))
+
+
+def target_category_loss_output_shape(input_shape):
+    return input_shape
+
+
+def normalize(x):
+    # utility function to normalize a tensor by its L2 norm
+    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
+
+
+# def load_image(path):
+#     img = image.load_img(path, target_size=(224, 224))
+#     x = image.img_to_array(img)
+#     x = np.expand_dims(x, axis=0)
+#     x = preprocess_input(x)
+#     return x
+
+
+def _compute_gradients(tensor, var_list):
+    grads = tf.gradients(tensor, var_list)
+    return [
+        grad if grad is not None else tf.zeros_like(var)
+        for var, grad in zip(var_list, grads)
+    ]
+
+
+def grad_cam(graph, image, category_index, layer_name):
+    labels_tensor = graph.get_tensor_by_name('norm_dense_1_target:0')
+    sample_weights = graph.get_tensor_by_name('norm_dense_1_sample_weights:0')
+    pred_tensor = graph.get_tensor_by_name('norm_dense_1/Softmax:0')
+    in_tensor = graph.get_tensor_by_name('input_1:0')
+    # loss expects unscaled logits since it performs a softmax on logits internally for efficiency
+
+    # Define loss and optimizer
+    loss_op = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits_v2(
+            logits=pred_tensor, labels=labels_tensor) * sample_weights)
+    conv_output = graph.get_tensor_by_name('conv5_block3_out')
+    grads = normalize(_compute_gradients(loss_op, [conv_output])[0])
+    gradient_function = K.function([in_tensor], [conv_output, grads])
+
+    output, grads_val = gradient_function([image])
+    output, grads_val = output[0, :], grads_val[0, :, :, :]
+
+    weights = np.mean(grads_val, axis=(0, 1))
+    cam = np.ones(output.shape[0:2], dtype=np.float32)
+
+    for i, w in enumerate(weights):
+        cam += w * output[:, :, i]
+
+    #shift values by min
+    cam -= np.min(cam)
+    # cam = np.maximum(cam, 0)
+    heatmap = cam / np.max(cam)
+
+    #Return to BGR [0..255] from the preprocessed image
+    image = image[0, :]
+    image -= np.min(image)
+    image = np.minimum(image, 255)
+
+    _, cam = cv2.threshold(np.uint8(255 * heatmap), 0, 255,
+                           cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    cam = cam / np.max(cam)
+    cam = 1 - cam
+    edges = cv2.Canny(np.uint8(255 * cam), 0, 255)
+    # cv2 channels formated as bgr
+    from copy import deepcopy
+    outlined_image = deepcopy(image)
+    outlined_image[:, :, 2] += edges
+    outlined_image[outlined_image > 255] = 255
+
+    # masked_image = image*cam.reshape((224,224, 1))
+    # cam = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET)
+    #cam = np.float32(cam) #+  np.float32(image)
+    #cam = cam / np.max(cam)
+
+    return np.uint8(outlined_image), heatmap
+
+
+# model = VGG16(weights='imagenet')
+
+# predictions = model.predict(preprocessed_input)
+# print(predictions)
+# top_1 = decode_predictions(predictions)[0][0]
+# print(decode_predictions(predictions))
+# print(f'Predicted class: {predicted_class}')
+# print('%s (%s) with probability %.2f' % (top_1[1], top_1[0], top_1[2]))
+
+# predicted_class = np.argmax(predictions)
+import os
+import json
+from data import process_image_file
+import utils
+
+
+def readInput():
+    res = []
+    res.append(process_image_file(sys.argv[1], 0.08, 480))
+
+    with open(sys.argv[2], 'r') as f:
+        input = json.load(f)
+        res.append(
+            np.array([
+                float(x) for x in
+                [input['Normal'], input['Pneumonia'], input['COVID-19']]
+            ]))
+    return res[0], res[1]
+
+
+preprocessed_input, prediction_matrix = readInput()
+
+print(np.shape(prediction_matrix))
+print(prediction_matrix)
+
+mapping = {'Normal': 0, 'Pneumonia': 1, 'COVID-19': 2}
+
+tf.reset_default_graph()
+with tf.Session() as sess:
+    init = tf.global_variables_initializer()
+    args = {
+        'weightspath': os.path.abspath('models/COVIDNet-CXR4-B'),
+        'ckptname': 'model-1545',
+        'modelused': 'modelB'
+    }
+    tf.get_default_graph()
+    saver = tf.train.import_meta_graph(
+        os.path.join(args['weightspath'], 'model.meta'))
+    saver.restore(sess, os.path.join(args['weightspath'], args['ckptname']))
+
+    graph = tf.get_default_graph()
+    placeholders = [
+        op for op in graph.get_operations() if op.type == "Placeholder"
+    ]
+    print(placeholders)
+    image_tensor = graph.get_tensor_by_name('input_1:0')
+    pred_tensor = graph.get_tensor_by_name('norm_dense_1/Softmax:0')
+
+    x = preprocessed_input
+    x = x.astype('float32') / 255.0
+    pred = sess.run(pred_tensor,
+                    feed_dict={image_tensor: np.expand_dims(x, axis=0)})
+    print(pred)
+    labels_tensor = graph.get_tensor_by_name('norm_dense_1_target:0')
+    sample_weights = graph.get_tensor_by_name('norm_dense_1_sample_weights:0')
+    pred_logit_tensor = graph.get_tensor_by_name('norm_dense_1/MatMul:0')
+    pred_softmax_tensor = graph.get_tensor_by_name('norm_dense_1/Softmax:0')
+    # image_tensor = graph.get_tensor_by_name('input_1:0')
+    # loss expects unscaled logits since it performs a softmax on logits internally for efficiency
+    # Define loss and optimizer
+    loss_op = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits_v2(
+            logits=pred_logit_tensor, labels=labels_tensor) * sample_weights)
+    conv_output = graph.get_tensor_by_name('conv5_block3_out/add:0')
+    target_conv_layer_grad = tf.gradients(loss_op, conv_output)[0]
+    one_hot = np.zeros_like(prediction_matrix)
+    one_hot[np.argmax(prediction_matrix)] = 1
+
+    target_conv_layer_value, target_conv_layer_grad_value = sess.run(
+        [conv_output, target_conv_layer_grad],
+        feed_dict={
+            image_tensor:
+            np.expand_dims(preprocessed_input.astype('float32') / 255.0,
+                           axis=0),
+            sample_weights:
+            np.array([1]),
+            labels_tensor:
+            np.reshape(one_hot, (-1, 3))
+        })
+    print(np.reshape(prediction_matrix, (-1, 3)))
+    print(preprocessed_input.shape)
+    print(target_conv_layer_value.shape)
+    print(target_conv_layer_grad_value.shape)
+    print(np.amax(target_conv_layer_value))
+    print(np.amin(target_conv_layer_value))
+    print(np.amax(target_conv_layer_grad_value))
+    print(np.amin(target_conv_layer_grad_value))
+utils.visualize(preprocessed_input, target_conv_layer_value[0],
+                target_conv_layer_grad_value[0])
+# cam, heatmap = grad_cam(graph, preprocessed_input, predicted_,
+#                         "block5_conv3")
+# cv2.imwrite("gradcam.jpg", cam)

grad_cam/utils.py

@@ -0,0 +1,78 @@
+import skimage
+import skimage.io
+import skimage.transform
+import numpy as np
+
+from matplotlib import pyplot as plt
+from matplotlib.pyplot import imshow
+import matplotlib.image as mpimg
+
+import tensorflow as tf
+
+from skimage import io
+from skimage.transform import resize
+
+import cv2
+
+
+def visualize(image, conv_output, conv_grad):
+    output = conv_output  # [7,7,512]
+    grads_val = conv_grad  # [7,7,512]
+    print("grads_val shape:", grads_val.shape)
+
+    weights = np.mean(grads_val, axis=(0, 1))  # alpha_k, [512]
+    cam = np.zeros(output.shape[0:2], dtype=np.float32)  # [7,7]
+
+    # Taking a weighted average
+    for i, w in enumerate(weights):
+        cam += w * output[:, :, i]
+
+    # Passing through ReLU
+    cam = np.maximum(cam, 0)
+    cam = cam / np.max(cam)  # scale 0 to 1.0
+    cam = resize(cam, (224, 224), preserve_range=True)
+
+    img = image.astype(float)
+    img -= np.min(img)
+    img /= img.max()
+    # print(img)
+    cam_heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)
+    cam_heatmap = cv2.cvtColor(cam_heatmap, cv2.COLOR_BGR2RGB)
+    img = resize(img, (224, 224))
+    cam_heatmap = cam_heatmap / np.max(cam_heatmap)
+    cam_heatmap = np.float32(cam_heatmap) + np.float32(img)
+    cam_heatmap = 255 * cam_heatmap / np.max(cam_heatmap)
+    cam_heatmap = np.uint8(cam_heatmap)
+
+    fig = plt.figure()
+    ax = fig.add_subplot(111)
+    imgplot = plt.imshow(img)
+    ax.set_title('Input Image')
+
+    fig = plt.figure(figsize=(12, 16))
+    ax = fig.add_subplot(131)
+    imgplot = plt.imshow(cam_heatmap)
+    ax.set_title('Grad-CAM')
+
+    # gb_viz = np.dstack((
+    #     gb_viz[:, :, 0],
+    #     gb_viz[:, :, 1],
+    #     gb_viz[:, :, 2],
+    # ))
+    # gb_viz -= np.min(gb_viz)
+    # gb_viz /= gb_viz.max()
+
+    # ax = fig.add_subplot(132)
+    # imgplot = plt.imshow(gb_viz)
+    # ax.set_title('guided backpropagation')
+
+    # gd_gb = np.dstack((
+    #     gb_viz[:, :, 0] * cam,
+    #     gb_viz[:, :, 1] * cam,
+    #     gb_viz[:, :, 2] * cam,
+    # ))
+    # ax = fig.add_subplot(133)
+    # imgplot = plt.imshow(gd_gb)
+    # ax.set_title('guided Grad-CAM')
+
+    plt.show()