normlime_base.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. #copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. #Licensed under the Apache License, Version 2.0 (the "License");
  4. #you may not use this file except in compliance with the License.
  5. #You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. #Unless required by applicable law or agreed to in writing, software
  10. #distributed under the License is distributed on an "AS IS" BASIS,
  11. #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. #See the License for the specific language governing permissions and
  13. #limitations under the License.
  14. import os
  15. import os.path as osp
  16. import numpy as np
  17. import glob
  18. from paddlex.interpret.as_data_reader.readers import read_image
  19. import paddlex.utils.logging as logging
  20. from . import lime_base
  21. from ._session_preparation import compute_features_for_kmeans, gen_user_home
  22. def load_kmeans_model(fname):
  23. import pickle
  24. with open(fname, 'rb') as f:
  25. kmeans_model = pickle.load(f)
  26. return kmeans_model
  27. def combine_normlime_and_lime(lime_weights, g_weights):
  28. pred_labels = lime_weights.keys()
  29. combined_weights = {y: [] for y in pred_labels}
  30. for y in pred_labels:
  31. normlized_lime_weights_y = lime_weights[y]
  32. lime_weights_dict = {tuple_w[0]: tuple_w[1] for tuple_w in normlized_lime_weights_y}
  33. normlized_g_weight_y = g_weights[y]
  34. normlime_weights_dict = {tuple_w[0]: tuple_w[1] for tuple_w in normlized_g_weight_y}
  35. combined_weights[y] = [
  36. (seg_k, lime_weights_dict[seg_k] * normlime_weights_dict[seg_k])
  37. for seg_k in lime_weights_dict.keys()
  38. ]
  39. combined_weights[y] = sorted(combined_weights[y],
  40. key=lambda x: np.abs(x[1]), reverse=True)
  41. return combined_weights
  42. def avg_using_superpixels(features, segments):
  43. one_list = np.zeros((len(np.unique(segments)), features.shape[2]))
  44. for x in np.unique(segments):
  45. one_list[x] = np.mean(features[segments == x], axis=0)
  46. return one_list
  47. def centroid_using_superpixels(features, segments):
  48. from skimage.measure import regionprops
  49. regions = regionprops(segments + 1)
  50. one_list = np.zeros((len(np.unique(segments)), features.shape[2]))
  51. for i, r in enumerate(regions):
  52. one_list[i] = features[int(r.centroid[0] + 0.5), int(r.centroid[1] + 0.5), :]
  53. return one_list
  54. def get_feature_for_kmeans(feature_map, segments):
  55. from sklearn.preprocessing import normalize
  56. centroid_feature = centroid_using_superpixels(feature_map, segments)
  57. avg_feature = avg_using_superpixels(feature_map, segments)
  58. x = np.concatenate((centroid_feature, avg_feature), axis=-1)
  59. x = normalize(x)
  60. return x
  61. def precompute_normlime_weights(list_data_, predict_fn, num_samples=3000, batch_size=50, save_dir='./tmp'):
  62. # save lime weights and kmeans cluster labels
  63. precompute_lime_weights(list_data_, predict_fn, num_samples, batch_size, save_dir)
  64. # load precomputed results, compute normlime weights and save.
  65. fname_list = glob.glob(os.path.join(save_dir, 'lime_weights_s{}.npy'.format(num_samples)))
  66. return compute_normlime_weights(fname_list, save_dir, num_samples)
  67. def save_one_lime_predict_and_kmean_labels(lime_all_weights, image_pred_labels, cluster_labels, save_path):
  68. lime_weights = {}
  69. for label in image_pred_labels:
  70. lime_weights[label] = lime_all_weights[label]
  71. for_normlime_weights = {
  72. 'lime_weights': lime_weights, # a dict: class_label: (seg_label, weight)
  73. 'cluster': cluster_labels # a list with segments as indices.
  74. }
  75. np.save(save_path, for_normlime_weights)
  76. def precompute_lime_weights(list_data_, predict_fn, num_samples, batch_size, save_dir):
  77. root_path = gen_user_home()
  78. root_path = osp.join(root_path, '.paddlex')
  79. h_pre_models = osp.join(root_path, "pre_models")
  80. if not osp.exists(h_pre_models):
  81. if not osp.exists(root_path):
  82. os.makedirs(root_path)
  83. url = "https://bj.bcebos.com/paddlex/interpret/pre_models.tar.gz"
  84. pdx.utils.download_and_decompress(url, path=root_path)
  85. h_pre_models_kmeans = osp.join(h_pre_models, "kmeans_model.pkl")
  86. kmeans_model = load_kmeans_model(h_pre_models_kmeans)
  87. for data_index, each_data_ in enumerate(list_data_):
  88. if isinstance(each_data_, str):
  89. save_path = "lime_weights_s{}_{}.npy".format(num_samples, each_data_.split('/')[-1].split('.')[0])
  90. save_path = os.path.join(save_dir, save_path)
  91. else:
  92. save_path = "lime_weights_s{}_{}.npy".format(num_samples, data_index)
  93. save_path = os.path.join(save_dir, save_path)
  94. if os.path.exists(save_path):
  95. logging.info(save_path + ' exists, not computing this one.', use_color=True)
  96. continue
  97. img_file_name = each_data_ if isinstance(each_data_, str) else data_index
  98. logging.info('processing '+ img_file_name + ' [{}/{}]'.format(data_index, len(list_data_)), use_color=True)
  99. image_show = read_image(each_data_)
  100. result = predict_fn(image_show)
  101. result = result[0] # only one image here.
  102. if abs(np.sum(result) - 1.0) > 1e-4:
  103. # softmax
  104. exp_result = np.exp(result)
  105. probability = exp_result / np.sum(exp_result)
  106. else:
  107. probability = result
  108. pred_label = np.argsort(probability)[::-1]
  109. # top_k = argmin(top_n) > threshold
  110. threshold = 0.05
  111. top_k = 0
  112. for l in pred_label:
  113. if probability[l] < threshold or top_k == 5:
  114. break
  115. top_k += 1
  116. if top_k == 0:
  117. top_k = 1
  118. pred_label = pred_label[:top_k]
  119. algo = lime_base.LimeImageInterpreter()
  120. interpreter = algo.interpret_instance(image_show[0], predict_fn, pred_label, 0,
  121. num_samples=num_samples, batch_size=batch_size)
  122. X = get_feature_for_kmeans(compute_features_for_kmeans(image_show).transpose((1, 2, 0)), interpreter.segments)
  123. try:
  124. cluster_labels = kmeans_model.predict(X)
  125. except AttributeError:
  126. from sklearn.metrics import pairwise_distances_argmin_min
  127. cluster_labels, _ = pairwise_distances_argmin_min(X, kmeans_model.cluster_centers_)
  128. save_one_lime_predict_and_kmean_labels(
  129. interpreter.local_weights, pred_label,
  130. cluster_labels,
  131. save_path
  132. )
  133. def compute_normlime_weights(a_list_lime_fnames, save_dir, lime_num_samples):
  134. normlime_weights_all_labels = {}
  135. for f in a_list_lime_fnames:
  136. try:
  137. lime_weights_and_cluster = np.load(f, allow_pickle=True).item()
  138. lime_weights = lime_weights_and_cluster['lime_weights']
  139. cluster = lime_weights_and_cluster['cluster']
  140. except:
  141. logging.info('When loading precomputed LIME result, skipping' + str(f))
  142. continue
  143. logging.info('Loading precomputed LIME result,' + str(f))
  144. pred_labels = lime_weights.keys()
  145. for y in pred_labels:
  146. normlime_weights = normlime_weights_all_labels.get(y, {})
  147. w_f_y = [abs(w[1]) for w in lime_weights[y]]
  148. w_f_y_l1norm = sum(w_f_y)
  149. for w in lime_weights[y]:
  150. seg_label = w[0]
  151. weight = w[1] * w[1] / w_f_y_l1norm
  152. a = normlime_weights.get(cluster[seg_label], [])
  153. a.append(weight)
  154. normlime_weights[cluster[seg_label]] = a
  155. normlime_weights_all_labels[y] = normlime_weights
  156. # compute normlime
  157. for y in normlime_weights_all_labels:
  158. normlime_weights = normlime_weights_all_labels.get(y, {})
  159. for k in normlime_weights:
  160. normlime_weights[k] = sum(normlime_weights[k]) / len(normlime_weights[k])
  161. # check normlime
  162. if len(normlime_weights_all_labels.keys()) < max(normlime_weights_all_labels.keys()) + 1:
  163. logging.info(
  164. "\n" + \
  165. "Warning: !!! \n" + \
  166. "There are at least {} classes, ".format(max(normlime_weights_all_labels.keys()) + 1) + \
  167. "but the NormLIME has results of only {} classes. \n".format(len(normlime_weights_all_labels.keys())) + \
  168. "It may have cause unstable results in the later computation" + \
  169. " but can be improved by computing more test samples." + \
  170. "\n"
  171. )
  172. n = 0
  173. f_out = 'normlime_weights_s{}_samples_{}-{}.npy'.format(lime_num_samples, len(a_list_lime_fnames), n)
  174. while os.path.exists(
  175. os.path.join(save_dir, f_out)
  176. ):
  177. n += 1
  178. f_out = 'normlime_weights_s{}_samples_{}-{}.npy'.format(lime_num_samples, len(a_list_lime_fnames), n)
  179. continue
  180. np.save(
  181. os.path.join(save_dir, f_out),
  182. normlime_weights_all_labels
  183. )
  184. return os.path.join(save_dir, f_out)