knuckletouch/python/Step_10_LSTM_Preprocessing.ipynb

Preprocessing for LSTM: Blobdetection and Cutting¶

In [2]:

%matplotlib inline

from scipy.odr import *
from scipy.stats import *
import numpy as np
import pandas as pd
import os
import time
import matplotlib.pyplot as plt
import ast
from multiprocessing import Pool, cpu_count

import scipy

from IPython import display
from matplotlib.patches import Rectangle

from sklearn.metrics import mean_squared_error
import json

import scipy.stats as st
from sklearn.metrics import r2_score


from matplotlib import cm
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt

import copy

from sklearn.model_selection import LeaveOneOut, LeavePOut

from multiprocessing import Pool
import cv2

In [3]:

df_filtered = pd.read_pickle("DataStudyCollection/df_lstm.pkl")

In [4]:

df_filtered.head()

Out[4]:

	userID	Timestamp	Current_Task	Task_amount	TaskID	VersionID	RepetitionID	Actual_Data	Is_Pause	Image	IsMax	MaxRepetition
291980	1	1,54515E+12	33	680	0	2	0	True	False	[0, 2, 0, 0, 0, 0, 1, 2, 2, 3, 2, 1, 1, 1, 0, ...	True	0
291981	1	1,54515E+12	33	680	0	2	0	True	False	[0, 2, 0, 0, 0, 0, 1, 2, 2, 3, 2, 1, 1, 1, 0, ...	True	0
291982	1	1,54515E+12	33	680	0	2	0	True	False	[0, 2, 0, 0, 0, 0, 1, 2, 2, 3, 2, 1, 1, 1, 0, ...	True	0
291983	1	1,54515E+12	33	680	0	2	0	True	False	[0, 2, 0, 0, 0, 0, 1, 2, 2, 3, 2, 1, 1, 1, 0, ...	True	0
291984	1	1,54515E+12	33	680	0	2	0	True	False	[0, 2, 0, 0, 0, 0, 1, 2, 2, 3, 2, 1, 1, 1, 0, ...	True	0

In [5]:

df_filtered.Image = df_filtered.Image.apply(lambda x: x.reshape(27, 15))
df_filtered.Image = df_filtered.Image.apply(lambda x: x.clip(min=0, max=255))
df_filtered.Image = df_filtered.Image.apply(lambda x: x.astype(np.uint8))
df_filtered["ImageSum"] = df_filtered.Image.apply(lambda x: np.sum(x))

In [6]:

#LSTMs new Blob detection (only detect, if there are blobs)
def detect_blobs(image):
    #image = image.reshape(27, 15)
    large = np.ones((29,17), dtype=np.uint8)
    large[1:28,1:16] = image
    temp, thresh = cv2.threshold(cv2.bitwise_not(large), 200, 255, cv2.THRESH_BINARY)
    contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    contours = [a for a in contours if cv2.contourArea(a) > 8 and cv2.contourArea(a) < 255]
    lstBlob  = []
    lstMin = []
    lstMax = []
    count = 0
    return len(contours) > 0

In [7]:

%%time
pool = Pool(cpu_count() - 1)
temp_blobs = pool.map(detect_blobs, df_filtered.Image)
pool.close()

CPU times: user 3.42 s, sys: 1.14 s, total: 4.57 s
Wall time: 4.94 s

In [8]:

df_filtered["ContainsBlobs"] = temp_blobs

In [9]:

#Label if knuckle or finger
def f(row):
    if row['TaskID'] < 17:
        #val = "Knuckle"
        val = 0
    elif row['TaskID'] >= 17:
        #val = "Finger"
        val = 1
    return val
df_filtered['InputMethod'] = df_filtered.apply(f, axis=1)

In [10]:

df_filtered.index = range(len(df_filtered))

In [11]:

%%time
# trim image sequences down to only between first and last detected blob
UserIDs = []
TaskIDs = []
VersionIDs = []
Blobs = []
for userID in df_filtered.userID.unique():
    print(userID)
    for TaskID in df_filtered[df_filtered.userID == userID].TaskID.unique():
        for VersionID in df_filtered[(df_filtered.userID == userID) & (df_filtered.TaskID == TaskID)].VersionID.unique():
            first_blob = -1
            last_blob = -1
            for index, row in df_filtered[(df_filtered.userID == userID) & (df_filtered.TaskID == TaskID) & (df_filtered.VersionID == VersionID)].iterrows():
                if row.ContainsBlobs:
                    last_blob = index
                    if first_blob == -1:
                        first_blob = index
            if first_blob >= 0 and last_blob >= 0:
                UserIDs.append(userID)
                TaskIDs.append(TaskID)
                VersionIDs.append(VersionID)
                Blobs.append(df_filtered[(df_filtered.userID == userID) & (df_filtered.TaskID == TaskID) & (df_filtered.VersionID == VersionID) & (df_filtered.index >= first_blob) & (df_filtered.index <= last_blob)].Image.tolist())

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
CPU times: user 4min 7s, sys: 424 ms, total: 4min 8s
Wall time: 4min 8s

In [12]:

UserIDs = np.array(UserIDs, dtype=np.int64)
TaskIDs = np.array(TaskIDs, dtype=np.int64)
VersionIDs = np.array(VersionIDs, dtype=np.int64)

In [13]:

df_lstm_all = pd.DataFrame()
df_lstm_all["userID"] = UserIDs
df_lstm_all["TaskID"] = TaskIDs
df_lstm_all["VersionID"] = VersionIDs
df_lstm_all["Blobs"] = Blobs
df_lstm_all.Blobs = df_lstm_all.Blobs.map(np.array)
df_lstm_all.head()

Out[13]:

	userID	TaskID	VersionID	Blobs
0	1	0	3	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...
1	1	0	5	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...
2	1	0	6	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 239,...
3	1	0	7	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...
4	1	0	8	[[[0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0...

In [33]:

df_lstm_all.head()

Out[33]:

	userID	TaskID	VersionID	Blobs	BlobCount	GestureOnly
0	1	0	3	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...	38	0
1	1	0	5	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...	57	0
2	1	0	6	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 239,...	41	0
3	1	0	7	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...	20	0
4	1	0	8	[[[0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0...	41	0

In [34]:

df_lstm_all["Length"] = df_lstm_all.Blobs.apply(lambda x: x.shape[0])

In [43]:

df_lstm_all.Length.hist(range=(0,300), bins=30)

Out[43]:

<matplotlib.axes._subplots.AxesSubplot at 0x7f1a61641278>

In [46]:

len(df_lstm_all[df_lstm_all.Length > 50]) / len(df_lstm_all)

Out[46]:

0.02870949403069926

In [52]:

df_lstm_all.Length.describe().round(1)

Out[52]:

count    10554.0
mean        15.9
std         13.6
min          1.0
25%          8.0
50%         13.0
75%         19.0
max        301.0
Name: Length, dtype: float64

In [14]:

lengths = []
for index, row in df_lstm_all.iterrows():
    lengths.append(row.Blobs.shape[0])
df_lstm_all["BlobCount"] = lengths
# add a column for pure gesture recognition without finger/knuckle
df_lstm_all["GestureOnly"] = df_lstm_all.TaskID % 17

In [15]:

df_lstm_all.BlobCount.describe()

Out[15]:

count    10554.000000
mean        15.906576
std         13.605214
min          1.000000
25%          8.000000
50%         13.000000
75%         19.000000
max        301.000000
Name: BlobCount, dtype: float64

In [16]:

df_lstm_all.groupby(df_lstm_all.GestureOnly)["BlobCount"].agg(["mean", "std"])

Out[16]:

	mean	std
GestureOnly
0	11.421429	8.940925
1	13.618683	13.864708
2	8.852596	6.315931
3	8.672913	5.580500
4	9.828767	6.793559
5	9.211221	6.861675
6	14.622496	8.338379
7	13.684524	13.263753
8	20.397129	12.916920
9	14.468599	10.042060
10	14.921440	8.909217
11	13.695578	7.661549
12	17.070853	11.755087
13	15.712219	10.545010
14	16.468354	9.826818
15	19.840836	11.239255
16	42.931624	21.024635

In [17]:

# filter on gesture lengths
print("before: %s" % len(df_lstm_all))
df_lstm = df_lstm_all[(df_lstm_all.BlobCount <= 100) & (df_lstm_all.BlobCount >= 5)]
print("after: %s" % len(df_lstm))
print("ratio: %s" % ((len(df_lstm_all) - len(df_lstm)) / len(df_lstm_all) * 100))

before: 10554
after: 9193
ratio: 12.895584612469206

In [18]:

df_lstm.BlobCount.describe()

Out[18]:

count    9193.000000
mean       17.678995
std        12.059369
min         5.000000
25%        10.000000
50%        15.000000
75%        20.000000
max        97.000000
Name: BlobCount, dtype: float64

In [19]:

lengths = []
for index, row in df_lstm.iterrows():
    lengths.append(row.Blobs.shape[0])

In [20]:

df_lstm["BlobCount"] = lengths

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.

In [21]:

In [22]:

df_lstm.BlobCount.describe()

Out[22]:

count    9193.000000
mean       17.678995
std        12.059369
min         5.000000
25%        10.000000
50%        15.000000
75%        20.000000
max        97.000000
Name: BlobCount, dtype: float64

In [23]:

def lerp(a, b, c=0.5):
    return c * b + (1.0 - c) * a

#Svens new Blob detection
def detect_blobs_return_old(image, task):
    #image = e.Image
    large = np.ones((29,17), dtype=np.uint8)
    large[1:28,1:16] = np.copy(image)
    temp, thresh = cv2.threshold(cv2.bitwise_not(large), 205, 255, cv2.THRESH_BINARY)
    contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    contours = [a for a in contours if cv2.contourArea(a) > 8 and cv2.contourArea(a) < 255]
    lstBlob  = []
    lstCenter = []
    lstMin = []
    lstMax = []
    count = 0
    contours.sort(key=lambda a: cv2.contourArea(a))
    if len(contours) > 0:
        # if two finger or knuckle
        cont_count = 2 if task in [1, 6, 7, 18, 23, 24] and len(contours) > 1 else 1
        for i in range(1, cont_count + 1):
            max_contour = contours[-1 * i]
            xmax, ymax = np.max(max_contour.reshape(len(max_contour),2), axis=0)
            xmin, ymin = np.min(max_contour.reshape(len(max_contour),2), axis=0)
            M = cv2.moments(max_contour)
            cX = int(M["m10"] / M["m00"]) - 1
            cY = int(M["m01"] / M["m00"]) - 1
            #croped_im = np.zeros((27,15))
            blob = large[max(ymin - 1, 0):min(ymax + 1, large.shape[0]),max(xmin - 1, 0):min(xmax + 1, large.shape[1])]
            #croped_im[0:blob.shape[0],0:blob.shape[1]] = blob
            #return (1, [croped_im])
            lstBlob.append(blob)
            lstCenter.append((cY, cX))
            lstMin.append(xmax-xmin)
            lstMax.append(ymax-ymin)
            count = count + 1
        return (count, lstBlob, lstCenter)
    else:
        return (0, [np.zeros((29, 19))], 0, 0)

In [24]:

# descides whether or not a normalization is neccessary
# and cuts or adds zeros
def normalize_blobs(blobs, new_len=50):
    new_count = new_len - blobs.shape[0]
    if new_count == 0:
        return blobs
    elif new_count > 0:
        temp = np.array([np.zeros((27, 15))] * new_count)
        return np.append(blobs, temp, axis=0)
    else:
        return blobs[0:new_len]

In [25]:

%%time
# normalizes all image sequences
df_lstm_norm = df_lstm.copy(deep=True)
new_blobs = []
for index, row in df_lstm.iterrows():
    new_blobs.append(normalize_blobs(row.Blobs, 50))

df_lstm_norm.Blobs = new_blobs

lengths = []
for index, row in df_lstm_norm.iterrows():
    lengths.append(row.Blobs.shape[0])
df_lstm_norm["BlobCount"] = lengths

CPU times: user 3.24 s, sys: 556 ms, total: 3.8 s
Wall time: 3.8 s

In [26]:

df_lstm_norm.BlobCount.describe()

Out[26]:

count    9193.0
mean       50.0
std         0.0
min        50.0
25%        50.0
50%        50.0
75%        50.0
max        50.0
Name: BlobCount, dtype: float64

In [27]:

df_lstm_norm.to_pickle("DataStudyCollection/df_lstm_norm50.pkl")

In [28]:

df_lstm_norm.head()

Out[28]:

	userID	TaskID	VersionID	Blobs	BlobCount	GestureOnly
0	1	0	3	[[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...	50	0
1	1	0	5	[[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...	50	0
2	1	0	6	[[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...	50	0
3	1	0	7	[[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...	50	0
4	1	0	8	[[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0...	50	0

In [29]:

print("finished")

finished

In [ ]: