{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## This notebook creates one dataframe from all participants data\n", "## It also removes 1% of the data as this is corrupted" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "\n", "from scipy.odr import *\n", "from scipy.stats import *\n", "import numpy as np\n", "import pandas as pd\n", "import os\n", "import time\n", "import matplotlib.pyplot as plt\n", "import ast\n", "from multiprocessing import Pool, cpu_count\n", "\n", "import scipy\n", "\n", "from IPython import display\n", "from matplotlib.patches import Rectangle\n", "\n", "from sklearn.metrics import mean_squared_error\n", "import json\n", "\n", "import scipy.stats as st\n", "from sklearn.metrics import r2_score\n", "\n", "\n", "from matplotlib import cm\n", "from mpl_toolkits.mplot3d import axes3d\n", "import matplotlib.pyplot as plt\n", "\n", "import copy\n", "\n", "from sklearn.model_selection import LeaveOneOut, LeavePOut\n", "\n", "from multiprocessing import Pool" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def cast_to_int(row):\n", " try:\n", " return np.array([a if float(a) >= 0 else 0 for a in row[2:-1]], dtype=np.uint8)\n", " except Exception as e:\n", " return None\n", " \n", "def load_csv(file):\n", " temp_df = pd.read_csv(file, delimiter=\";\")\n", " temp_df.Image = temp_df.Image.str.split(',')\n", " temp_df.Image = temp_df.Image.apply(cast_to_int)\n", " return temp_df" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['DataStudyCollection/17_studyData.csv', 'DataStudyCollection/2_studyData.csv', 'DataStudyCollection/12_studyData.csv', 'DataStudyCollection/15_studyData.csv', 'DataStudyCollection/5_studyData.csv', 'DataStudyCollection/1_studyData.csv', 'DataStudyCollection/14_studyData.csv', 'DataStudyCollection/10_studyData.csv', 'DataStudyCollection/13_studyData.csv', 'DataStudyCollection/18_studyData.csv', 'DataStudyCollection/6_studyData.csv', 'DataStudyCollection/16_studyData.csv', 'DataStudyCollection/3_studyData.csv', 'DataStudyCollection/7_studyData.csv', 'DataStudyCollection/8_studyData.csv', 'DataStudyCollection/9_studyData.csv', 'DataStudyCollection/11_studyData.csv', 'DataStudyCollection/4_studyData.csv']\n", "CPU times: user 1.86 s, sys: 1.03 s, total: 2.89 s\n", "Wall time: 17.3 s\n" ] } ], "source": [ "%%time\n", "pool = Pool(cpu_count() - 2)\n", "data_files = [\"DataStudyCollection/%s\" % file for file in os.listdir(\"DataStudyCollection\") if file.endswith(\".csv\") and \"studyData\" in file]\n", "print(data_files)\n", "df_lst = pool.map(load_csv, data_files)\n", "dfAll = pd.concat(df_lst)\n", "pool.close()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1010014" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = dfAll[dfAll.Image.notnull()]\n", "len(df)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loaded 1013841 values\n", "removed 3827 values (thats 0.377%)\n", "new df has size 1010014\n" ] } ], "source": [ "print(\"loaded %s values\" % len(dfAll))\n", "print(\"removed %s values (thats %s%%)\" % (len(dfAll) - len(df), round((len(dfAll) - len(df)) / len(dfAll) * 100, 3)))\n", "print(\"new df has size %s\" % len(df))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "df = df.reset_index(drop=True)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | userID | \n", "Timestamp | \n", "Current_Task | \n", "Task_amount | \n", "TaskID | \n", "VersionID | \n", "RepetitionID | \n", "Actual_Data | \n", "Is_Pause | \n", "Image | \n", "
---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "17 | \n", "1547138602677 | \n", "0 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "False | \n", "False | \n", "[1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, ... | \n", "
1 | \n", "17 | \n", "1547138602697 | \n", "0 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "False | \n", "False | \n", "[1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, ... | \n", "
2 | \n", "17 | \n", "1547138602796 | \n", "0 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "False | \n", "False | \n", "[1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, ... | \n", "
3 | \n", "17 | \n", "1547138602817 | \n", "0 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "False | \n", "False | \n", "[1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, ... | \n", "
4 | \n", "17 | \n", "1547138602863 | \n", "0 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "False | \n", "False | \n", "[1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, ... | \n", "