284 lines
8 KiB
Text
284 lines
8 KiB
Text
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## Filtering the data for the LSTM: removes all the rows, where we used the revert button, when the participant performed a wrong gesture\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"%matplotlib inline\n",
|
||
|
|
"\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import pandas as pd\n",
|
||
|
|
"from multiprocessing import Pool, cpu_count"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/html": [
|
||
|
|
"<div>\n",
|
||
|
|
"<style scoped>\n",
|
||
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
||
|
|
" vertical-align: middle;\n",
|
||
|
|
" }\n",
|
||
|
|
"\n",
|
||
|
|
" .dataframe tbody tr th {\n",
|
||
|
|
" vertical-align: top;\n",
|
||
|
|
" }\n",
|
||
|
|
"\n",
|
||
|
|
" .dataframe thead th {\n",
|
||
|
|
" text-align: right;\n",
|
||
|
|
" }\n",
|
||
|
|
"</style>\n",
|
||
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
||
|
|
" <thead>\n",
|
||
|
|
" <tr style=\"text-align: right;\">\n",
|
||
|
|
" <th></th>\n",
|
||
|
|
" <th>userID</th>\n",
|
||
|
|
" <th>Timestamp</th>\n",
|
||
|
|
" <th>Current_Task</th>\n",
|
||
|
|
" <th>Task_amount</th>\n",
|
||
|
|
" <th>TaskID</th>\n",
|
||
|
|
" <th>VersionID</th>\n",
|
||
|
|
" <th>RepetitionID</th>\n",
|
||
|
|
" <th>Actual_Data</th>\n",
|
||
|
|
" <th>Is_Pause</th>\n",
|
||
|
|
" <th>Image</th>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" </thead>\n",
|
||
|
|
" <tbody>\n",
|
||
|
|
" <tr>\n",
|
||
|
|
" <th>8351</th>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>1553594010364</td>\n",
|
||
|
|
" <td>1</td>\n",
|
||
|
|
" <td>510</td>\n",
|
||
|
|
" <td>28</td>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>0</td>\n",
|
||
|
|
" <td>True</td>\n",
|
||
|
|
" <td>False</td>\n",
|
||
|
|
" <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" <tr>\n",
|
||
|
|
" <th>8352</th>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>1553594010414</td>\n",
|
||
|
|
" <td>1</td>\n",
|
||
|
|
" <td>510</td>\n",
|
||
|
|
" <td>28</td>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>0</td>\n",
|
||
|
|
" <td>True</td>\n",
|
||
|
|
" <td>False</td>\n",
|
||
|
|
" <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" <tr>\n",
|
||
|
|
" <th>8353</th>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>1553594010445</td>\n",
|
||
|
|
" <td>1</td>\n",
|
||
|
|
" <td>510</td>\n",
|
||
|
|
" <td>28</td>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>0</td>\n",
|
||
|
|
" <td>True</td>\n",
|
||
|
|
" <td>False</td>\n",
|
||
|
|
" <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" <tr>\n",
|
||
|
|
" <th>8354</th>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>1553594010485</td>\n",
|
||
|
|
" <td>1</td>\n",
|
||
|
|
" <td>510</td>\n",
|
||
|
|
" <td>28</td>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>0</td>\n",
|
||
|
|
" <td>True</td>\n",
|
||
|
|
" <td>False</td>\n",
|
||
|
|
" <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" <tr>\n",
|
||
|
|
" <th>8355</th>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>1553594010525</td>\n",
|
||
|
|
" <td>1</td>\n",
|
||
|
|
" <td>510</td>\n",
|
||
|
|
" <td>28</td>\n",
|
||
|
|
" <td>2</td>\n",
|
||
|
|
" <td>0</td>\n",
|
||
|
|
" <td>True</td>\n",
|
||
|
|
" <td>False</td>\n",
|
||
|
|
" <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" </tbody>\n",
|
||
|
|
"</table>\n",
|
||
|
|
"</div>"
|
||
|
|
],
|
||
|
|
"text/plain": [
|
||
|
|
" userID Timestamp Current_Task Task_amount TaskID VersionID \\\n",
|
||
|
|
"8351 2 1553594010364 1 510 28 2 \n",
|
||
|
|
"8352 2 1553594010414 1 510 28 2 \n",
|
||
|
|
"8353 2 1553594010445 1 510 28 2 \n",
|
||
|
|
"8354 2 1553594010485 1 510 28 2 \n",
|
||
|
|
"8355 2 1553594010525 1 510 28 2 \n",
|
||
|
|
"\n",
|
||
|
|
" RepetitionID Actual_Data Is_Pause \\\n",
|
||
|
|
"8351 0 True False \n",
|
||
|
|
"8352 0 True False \n",
|
||
|
|
"8353 0 True False \n",
|
||
|
|
"8354 0 True False \n",
|
||
|
|
"8355 0 True False \n",
|
||
|
|
"\n",
|
||
|
|
" Image \n",
|
||
|
|
"8351 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ... \n",
|
||
|
|
"8352 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ... \n",
|
||
|
|
"8353 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ... \n",
|
||
|
|
"8354 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ... \n",
|
||
|
|
"8355 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ... "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"dfAll = pd.read_pickle(\"DataStudyEvaluation/AllData.pkl\")\n",
|
||
|
|
"df_actual = dfAll[(dfAll.Actual_Data == True) & (dfAll.Is_Pause == False)]\n",
|
||
|
|
"df_actual.head()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"12"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 3,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"len(df_actual.userID.unique())"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"all: 608084, actual data: 495142\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"print(\"all: %s, actual data: %s\" % (len(dfAll), len(df_actual)))"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"CPU times: user 23.3 s, sys: 3.08 s, total: 26.3 s\n",
|
||
|
|
"Wall time: 26 s\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"%%time\n",
|
||
|
|
"# filter out all gestures, where the revert button was pressed during the study and the gestrue was repeated\n",
|
||
|
|
"def is_max(df):\n",
|
||
|
|
" df_temp = df.copy(deep=True)\n",
|
||
|
|
" max_version = df_temp.RepetitionID.max()\n",
|
||
|
|
" df_temp[\"IsMax\"] = np.where(df_temp.RepetitionID == max_version, True, False)\n",
|
||
|
|
" df_temp[\"MaxRepetition\"] = [max_version] * len(df_temp)\n",
|
||
|
|
" return df_temp\n",
|
||
|
|
"\n",
|
||
|
|
"df_filtered = df_actual.copy(deep=True)\n",
|
||
|
|
"df_grp = df_filtered.groupby([df_filtered.userID, df_filtered.TaskID, df_filtered.VersionID])\n",
|
||
|
|
"pool = Pool(cpu_count() - 1)\n",
|
||
|
|
"result_lst = pool.map(is_max, [grp for name, grp in df_grp])\n",
|
||
|
|
"df_filtered = pd.concat(result_lst)\n",
|
||
|
|
"df_filtered = df_filtered[df_filtered.IsMax == True]\n",
|
||
|
|
"pool.close()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 6,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"df_filtered.to_pickle(\"DataStudyEvaluation/df_lstm.pkl\")"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 7,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"actual: 495142, filtered data: 457271\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"print(\"actual: %s, filtered data: %s\" % (len(df_actual), len(df_filtered)))"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.6.7"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 2
|
||
|
|
}
|