{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'1.13.1'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## USE for Multi GPU Systems\n",
    "#import os\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "# Importing SK-learn to calculate precision and recall\n",
    "import sklearn\n",
    "from sklearn import metrics\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "target_names = [\"Knuckle\", \"Finger\"]\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle(\"DataStudyEvaluation/df_statistics.pkl\")\n",
    "\n",
    "lst = df.userID.unique()\n",
    "df[\"GestureId\"] = df.TaskID % 17\n",
    "\n",
    "#df_train = dfAll[dfAll.userID.isin(train_ids)]\n",
    "#df_test = dfAll[dfAll.userID.isin(test_ids)]\n",
    "\n",
    "x = np.concatenate(df.Blobs.values).reshape(-1,27,15,1)\n",
    "x = x / 255.0\n",
    "\n",
    "# convert class vectors to binary class matrices (one-hot notation)\n",
    "num_classes = len(df.TaskID.unique())\n",
    "y = tf.keras.utils.to_categorical(df.TaskID, num_classes)\n",
    "\n",
    "labels = sorted(df.TaskID.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.stack(df.Blobs)\n",
    "x = x.reshape(-1, 27, 15, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices (one-hot notation)\n",
    "num_classes = 2\n",
    "y = tf.keras.utils.to_categorical(df.InputMethod, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Label for image 1 is: [1. 0.]')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5428 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "i = 1\n",
    "plt.imshow(x[i].reshape(27, 15)) #np.sqrt(784) = 28\n",
    "plt.title(\"Label for image %i is: %s\" % (i, y[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If GPU is not available: \n",
    "# GPU_USE = '/cpu:0'\n",
    "#config = tf.ConfigProto(device_count = {\"GPU\": 1})\n",
    "\n",
    "\n",
    "# If GPU is available: \n",
    "config = tf.ConfigProto()\n",
    "config.log_device_placement = True\n",
    "config.allow_soft_placement = True\n",
    "config.gpu_options.allow_growth=True\n",
    "config.gpu_options.allocator_type = 'BFC'\n",
    "\n",
    "# Limit the maximum memory used\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.4\n",
    "\n",
    "# set session config\n",
    "tf.keras.backend.set_session(tf.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "loadpath = \"./ModelSnapshots/CNN-33767.h5\"\n",
    "model = tf.keras.models.load_model(loadpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16 s, sys: 1.39 s, total: 17.4 s\n",
      "Wall time: 12.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lst = []\n",
    "batch = 100\n",
    "for i in range(0, len(x), batch):\n",
    "    _x = x[i: i+batch]\n",
    "    lst.extend(model.predict(_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"InputMethodPred\"] = lst\n",
    "df.InputMethodPred = df.InputMethodPred.apply(lambda x: np.argmax(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.groupby([\"userID\", \"TaskID\", \"VersionID\"])[[\"InputMethodPred\", \"InputMethod\"]].agg(lambda x: x.tolist()).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.InputMethod = df.InputMethod.apply(lambda x: Counter(x).most_common()[0][0])\n",
    "df.InputMethodPred = df.InputMethodPred.apply(lambda x: Counter(x).most_common()[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>TaskID</th>\n",
       "      <th>VersionID</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InputMethod</th>\n",
       "      <th>InputMethodPred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>1882</td>\n",
       "      <td>1882</td>\n",
       "      <td>1882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2499</td>\n",
       "      <td>2499</td>\n",
       "      <td>2499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             userID  TaskID  VersionID\n",
       "InputMethod InputMethodPred                           \n",
       "0           0                  1882    1882       1882\n",
       "            1                   102     102        102\n",
       "1           0                    28      28         28\n",
       "            1                  2499    2499       2499"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby([\"InputMethod\", \"InputMethodPred\"]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1882  102]\n",
      " [  28 2499]]\n",
      "[[0.94858871 0.05141129]\n",
      " [0.01108033 0.98891967]]\n",
      "Accuray: 0.971\n",
      "Recall: 0.969\n",
      "Precision: 0.956\n",
      "F1-Score: 0.971\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Knuckle       0.99      0.95      0.97      1984\n",
      "      Finger       0.96      0.99      0.97      2527\n",
      "\n",
      "    accuracy                           0.97      4511\n",
      "   macro avg       0.97      0.97      0.97      4511\n",
      "weighted avg       0.97      0.97      0.97      4511\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(sklearn.metrics.confusion_matrix(df.InputMethod.values, df.InputMethodPred.values, labels=[0, 1]))\n",
    "cm = sklearn.metrics.confusion_matrix(df.InputMethod.values, df.InputMethodPred.values, labels=[0, 1], )\n",
    "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "print(cm)\n",
    "print(\"Accuray: %.3f\" % sklearn.metrics.accuracy_score(df.InputMethod.values, df.InputMethodPred.values))\n",
    "print(\"Recall: %.3f\" % metrics.recall_score(df.InputMethod.values, df.InputMethodPred.values, average=\"macro\"))\n",
    "print(\"Precision: %.3f\" % metrics.average_precision_score(df.InputMethod.values, df.InputMethodPred.values, average=\"macro\"))\n",
    "print(\"F1-Score: %.3f\" % metrics.f1_score(df.InputMethod.values, df.InputMethodPred.values, average=\"macro\"))\n",
    "print(sklearn.metrics.classification_report(df.InputMethod.values, df.InputMethodPred.values, target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
