54 KiB
54 KiB
In [1]:
## USE for Multi GPU Systems #import os #os.environ["CUDA_VISIBLE_DEVICES"]="0" import numpy as np import matplotlib.pyplot as plt import pandas as pd import math import tensorflow as tf %matplotlib inline # Importing SK-learn to calculate precision and recall import sklearn from sklearn import metrics from sklearn.utils.multiclass import unique_labels target_names = ["tap", "twotap", "swipeleft", "swiperight", "swipeup", "swipedown", "twoswipeup", "twoswipedown", "circle", "arrowheadleft", "arrowheadright", "checkmark", "flashlight", "l", "lmirrored", "screenshot", "rotate"] target_names = ["Tap", "Two tap", "Swipe left", "Swipe right", "Swipe up", "Swipe down", "Two swipe up", "Two swipe down", "Circle", "Arrowhead left", "Arrowhead right", "$\checkmark$", "$\Gamma$", "L", "L mirrored", "S", "Rotate"] tf.__version__
/usr/local/lib/python3.6/dist-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version! RequestsDependencyWarning)
Out[1]:
'1.13.1'
In [2]:
df = pd.read_pickle("DataStudyEvaluation/df_lstm_norm50.pkl") df.TaskID = df.TaskID % 17 x = np.concatenate(df.Blobs.values).reshape(-1,50,27,15,1) x = x / 255.0 # convert class vectors to binary class matrices (one-hot notation) num_classes = len(df.TaskID.unique()) y = tf.keras.utils.to_categorical(df.TaskID, num_classes) labels = sorted(df.TaskID.unique())
In [3]:
# If GPU is not available: # GPU_USE = '/cpu:0' #config = tf.ConfigProto(device_count = {"GPU": 1}) # If GPU is available: config = tf.ConfigProto() config.log_device_placement = True config.allow_soft_placement = True config.gpu_options.allow_growth=True config.gpu_options.allocator_type = 'BFC' # Limit the maximum memory used config.gpu_options.per_process_gpu_memory_fraction = 0.3 # set session config sess = tf.Session(config=config) tf.keras.backend.set_session(sess)
In [4]:
model = tf.keras.models.load_model('./ModelSnapshots/LSTM-v2-00398.h5')
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead.
In [5]:
%%time lst = [] batch = 100 for i in range(0, len(x), batch): _x = x[i : i+batch] lst.extend(model.predict(_x))
CPU times: user 30.4 s, sys: 3.28 s, total: 33.7 s Wall time: 19.6 s
In [6]:
df["TaskIDPred"] = lst df.TaskIDPred = df.TaskIDPred.apply(lambda x: np.argmax(x))
In [7]:
df_eval = df
In [8]:
print(sklearn.metrics.confusion_matrix(df_eval.TaskID.values, df_eval.TaskIDPred.values, labels=labels)) cm = sklearn.metrics.confusion_matrix(df_eval.TaskID.values, df_eval.TaskIDPred.values, labels=labels) cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print(np.round(cm,1)) print("Accuray: %.3f" % sklearn.metrics.accuracy_score(df_eval.TaskID.values, df_eval.TaskIDPred.values)) print("Recall: %.3f" % metrics.recall_score(df_eval.TaskID.values, df_eval.TaskIDPred.values, average="macro")) #print("Precision: %.3f" % metrics.average_precision_score(df_eval.TaskID.values, df_eval.TaskIDPred.values)) print("F1-Score: %.3f" % metrics.f1_score(df_eval.TaskID.values, df_eval.TaskIDPred.values, average="macro")) print(sklearn.metrics.classification_report(df_eval.TaskID.values, df_eval.TaskIDPred.values, target_names=target_names))
[[221 2 0 0 2 1 0 0 1 0 0 0 2 0 0 0 43] [ 3 297 0 0 0 1 1 0 0 0 0 0 0 0 0 0 13] [ 3 1 239 3 0 0 0 0 0 1 0 0 0 0 24 0 1] [ 2 1 1 244 0 0 0 5 1 0 0 3 5 5 3 0 4] [ 2 0 0 2 222 0 0 0 0 0 0 1 27 0 0 0 6] [ 5 0 0 0 0 246 0 1 1 0 0 2 4 21 0 2 3] [ 0 3 0 0 4 0 306 1 0 0 0 0 1 0 1 0 0] [ 0 6 0 0 1 12 2 306 0 0 0 0 6 1 0 0 0] [ 0 0 2 0 0 0 0 0 273 9 0 10 2 1 1 0 0] [ 1 0 4 1 0 0 0 0 11 249 2 4 0 9 0 8 1] [ 0 0 0 6 0 0 0 0 0 2 267 1 0 0 2 14 0] [ 1 0 0 1 4 0 0 1 19 1 0 247 4 0 1 0 0] [ 1 0 2 3 18 1 0 0 4 7 0 0 239 7 0 8 8] [ 0 0 0 3 1 6 0 0 5 5 0 7 2 272 0 0 0] [ 1 0 1 0 0 6 0 0 0 0 5 3 0 3 278 3 4] [ 0 0 8 0 0 1 0 0 6 10 5 0 3 1 21 250 1] [ 15 0 0 0 0 0 1 0 0 0 0 0 18 0 1 0 312]] [[0.8 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.2] [0. 0.9 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.1 0. 0. ] [0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0.1 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0.1 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0.1 0. 0. 0.9 0. 0. 0. 0. 0. ] [0. 0. 0. 0. 0.1 0. 0. 0. 0. 0. 0. 0. 0.8 0. 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.9 0. 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.1 0.8 0. ] [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.1 0. 0. 0. 0.9]] Accuray: 0.886 Recall: 0.885 F1-Score: 0.886 precision recall f1-score support Tap 0.87 0.81 0.84 272 Two tap 0.96 0.94 0.95 315 Swipe left 0.93 0.88 0.90 272 Swipe right 0.93 0.89 0.91 274 Swipe up 0.88 0.85 0.87 260 Swipe down 0.90 0.86 0.88 285 Two swipe up 0.99 0.97 0.98 316 Two swipe down 0.97 0.92 0.94 334 Circle 0.85 0.92 0.88 298 Arrowhead left 0.88 0.86 0.87 290 Arrowhead right 0.96 0.91 0.94 292 $\checkmark$ 0.89 0.89 0.89 279 $\Gamma$ 0.76 0.80 0.78 298 L 0.85 0.90 0.88 301 L mirrored 0.84 0.91 0.87 304 S 0.88 0.82 0.85 306 Rotate 0.79 0.90 0.84 347 accuracy 0.89 5043 macro avg 0.89 0.88 0.89 5043 weighted avg 0.89 0.89 0.89 5043
In [9]:
def plot_confusion_matrix(y_true, y_pred, classes, normalize=False, title=None, cmap=plt.cm.Blues): """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ if not title: if normalize: title = 'Normalized confusion matrix' else: title = 'Confusion matrix, without normalization' # Compute confusion matrix cm = sklearn.metrics.confusion_matrix(y_true, y_pred) # Only use the labels that appear in the data #classes = classes[unique_labels(y_true, y_pred)] if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: print('Confusion matrix, without normalization') #print(cm) fig, ax = plt.subplots(figsize=(6,6)) im = ax.imshow(cm, interpolation='nearest', cmap=cmap) #ax.figure.colorbar(im, ax=ax) # We want to show all ticks... ax.set(xticks=np.arange(cm.shape[1]), yticks=np.arange(cm.shape[0]), # ... and label them with the respective list entries xticklabels=classes, yticklabels=classes) #title=title, #ylabel='True label', #xlabel='Predicted label') # Rotate the tick labels and set their alignment. plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="anchor") # Loop over data dimensions and create text annotations. fmt = '.0f' if normalize else 'd' thresh = cm.max() / 2. for i in range(cm.shape[0]): for j in range(cm.shape[1]): ax.text(j, i, format(cm[i, j]*100, fmt), ha="center", va="center", color="white" if cm[i, j] > thresh else "black") fig.tight_layout() fig.savefig("./out/conf_matrix.pdf", bbox_inches='tight', transparent=False, pad_inches=0) return ax plot_confusion_matrix(df_eval.TaskID.values, df_eval.TaskIDPred.values, classes=target_names, normalize=True, title='Normalized confusion matrix')
Normalized confusion matrix
Out[9]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f64c079e0b8>
In [ ]: