441 lines
15 KiB
Python
441 lines
15 KiB
Python
from __future__ import division
|
|
import numpy as np
|
|
from numpy import linalg as LA
|
|
|
|
from time import time
|
|
|
|
from sklearn import linear_model
|
|
|
|
from sim import GazeSimulation
|
|
from svis import Ray
|
|
from minimize import findInitialW, _q, g, minimizeEnergy, minimizeEnergyGivenE, minimizeUsingScaleVector
|
|
|
|
from geom import getSphericalCoords, getAngularDiff
|
|
|
|
import visual as vs
|
|
# from visual import vector as v # for vector operations
|
|
from vector import Vector as v # for vector operations
|
|
|
|
class Experiment:
|
|
|
|
def performExperiment(self):
|
|
print 'Running Experiment...'
|
|
start = time()
|
|
self.__run__()
|
|
self.runningTime = time() - start
|
|
print 'Running Time:', self.runningTime
|
|
if self.minimizationTime:
|
|
print 'Minimization Time:', self.minimizationTime
|
|
|
|
def __run__(self):
|
|
pass
|
|
|
|
|
|
class Experiment1(Experiment):
|
|
|
|
def __run__(self):
|
|
'''
|
|
Simply evaluates 2D gaze estimation (using pupil calibration method)
|
|
'''
|
|
sim = GazeSimulation(log = False)
|
|
## Uncomment for shifting eyeball to scene camera coordinates
|
|
sim.scene_eye_distance_factor = 0.6 # controls how much to move eyeball towards scene camera [0-1]
|
|
sim.place_eyeball_on_scene_camera = False
|
|
|
|
# calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth
|
|
experiments = [
|
|
[True, False, False, True, False],
|
|
[True, False, False, False, False],
|
|
[True, False, False, False, True],
|
|
[False, True, False, True, False],
|
|
[False, True, False, False, False],
|
|
[False, True, False, False, True],
|
|
[False, False, False, True, False],
|
|
[False, False, False, False, False],
|
|
[False, False, False, False, True],
|
|
# [True, False, True, False, False],
|
|
]
|
|
|
|
nsim = 100
|
|
for experiment in experiments:
|
|
res = [sim.runCustomSimulation(experiment) for n in xrange(nsim)]
|
|
res = filter(lambda aae: aae>=0, res)
|
|
M = max(res)
|
|
m = min(res)
|
|
aae = sum(res) / len(res)
|
|
print 'AAE: %2.5f - Min AAE: %2.5f | Max AAE: %2.5f' % (aae, m, M)
|
|
|
|
eye_scene_diff = v(sim.sclera_pos) - v(sim.scene_camera.t)
|
|
print "Eyeball-SceneCamera distance was", eye_scene_diff.mag
|
|
|
|
self.sim = sim
|
|
self.minimizationTime = -1
|
|
|
|
|
|
class Experiment1p5(Experiment):
|
|
|
|
def __run__(self):
|
|
'''
|
|
Read 3D gaze estimation experiment
|
|
'''
|
|
sim = GazeSimulation(log = False)
|
|
## Uncomment for shifting eyeball to scene camera coordinates
|
|
sim.place_eyeball_on_scene_camera = True
|
|
sim.scene_eye_distance_factor = 0.6
|
|
# sim.scene_eye_distance_factor = 1.0
|
|
|
|
# calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth
|
|
# -> grid setting
|
|
# con = [True, False, True, False, True]
|
|
# -> general setting with grid learning
|
|
# con = [True, False, False, True, False]
|
|
con = [True, False, False, False, False]
|
|
# -> general setting with random learning
|
|
# con = [False, True, False, True, False]
|
|
sim.num_calibration = 36
|
|
sim.num_test = 25
|
|
aae_2d = sim.runCustomSimulation(con)
|
|
|
|
## Fetching the calibration points
|
|
t, p = sim.calibration_points, sim.tr_pupil_locations
|
|
# target positions are computed relative to the scene CCS
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # verified
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
|
|
start = time()
|
|
|
|
eye_scene_diff = v(sim.sclera_pos) - v(sim.scene_camera.t)
|
|
e = np.array(eye_scene_diff) # eyeball coords in scene CCS
|
|
print 'Real e', e
|
|
e_org = e[:]
|
|
print 'Eye-Scene distance', eye_scene_diff.mag
|
|
# w, w0 = minimizeEnergyGivenE(p, t, e)
|
|
|
|
## Finding the optimal transformation matrix by minimizing the nonlinear energy
|
|
# w0 is the initial w by solving the leastsq with e=(0,0,0)
|
|
# w is by solving the leastsq again optimizing for both e and w
|
|
w, e, w0 = minimizeEnergy(p, t)
|
|
|
|
# here were are initializing minimization with an e close to the ground
|
|
# truth e by adding random perturbations to it
|
|
# w, e, w0 = minimizeEnergy(p, t, e + np.random.rand(1, 3)[0])
|
|
|
|
print 'Estimated e', e, '( Distance:', LA.norm(e - e_org) , ')'
|
|
self.minimizationTime = time() - start
|
|
print
|
|
|
|
sim.w = w
|
|
|
|
## Fetching test points
|
|
t, p = sim.test_points, sim.te_pupil_locations
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # target coords in scene CCS
|
|
|
|
# closest point distance to scene camera
|
|
cDist = min(v(pt).mag for pt in t)
|
|
# farthest point distance to scene camera
|
|
fDist = max(v(pt).mag for pt in t)
|
|
# average point distance to scene camera
|
|
avgDist = sum(v(pt).mag for pt in t)/len(t)
|
|
|
|
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
# print
|
|
# print p
|
|
# print
|
|
|
|
|
|
qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D
|
|
# computing unit gaze vectors corresponding to pupil positions
|
|
# here we use the computed mapping matrix w
|
|
gis = map(lambda q: g(q, w), qi)
|
|
gis0 = map(lambda q: g(q, w0), qi)
|
|
|
|
# now we can compare unit gaze vectors with their corresponding gaze rays t
|
|
# normalizing gaze rays first
|
|
t = map(lambda vec: v(vec).norm(), t)
|
|
# TODO: compare spherical coordinates instead
|
|
|
|
|
|
AE = list(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t))
|
|
N = len(t)
|
|
AAE = sum(AE)/N
|
|
VAR = sum((ae - AAE)**2 for ae in AE)/N
|
|
print 'AAE:', AAE, '\nVariance:', VAR, 'STD:', np.sqrt(VAR), '\nMin:', min(AE), 'Max:', max(AE), '(N=' + str(N) + ')'
|
|
print 'Target Distances: m=%s M=%s Avg=%s' % (cDist, fDist, avgDist)
|
|
|
|
AE0 = list(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis0, t))
|
|
AAE0 = sum(AE0)/N
|
|
print 'AAE (only optimizing W for e=(0,0,0)):', AAE0
|
|
|
|
print 'AAE (2D-to-2D mapping):', aae_2d
|
|
print ('Improvement (2D-2D vs 2D-3D): %s' % round((aae_2d - AAE)*100/aae_2d, 2)) + '%'
|
|
print ('Improvement (2D-2D vs 2D-2D): %s' % round((aae_2d - AAE0)*100/aae_2d, 2)) + '%'
|
|
|
|
|
|
sim.display_test_points = True
|
|
sim.display_calibration_points = False
|
|
|
|
## Visualizing gaze vector
|
|
ray_scale_factor = 120
|
|
ground_truth_gaze_rays = []
|
|
for pt in t:
|
|
ground_truth_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.green))
|
|
sim.rays = ground_truth_gaze_rays
|
|
|
|
estimated_gaze_rays = []
|
|
for pt in gis:
|
|
estimated_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.red))
|
|
sim.rays.extend(estimated_gaze_rays)
|
|
|
|
AE = []
|
|
for pair in zip(t, gis):
|
|
gt_r, e_r = pair
|
|
base = v(sim.scene_camera.t)
|
|
gt_ray_endpoint = base + (v(gt_r) * ray_scale_factor)
|
|
e_ray_endpoint = base + (v(e_r) * ray_scale_factor)
|
|
AE.append(getAngularDiff(gt_ray_endpoint, e_ray_endpoint, base))
|
|
|
|
diff = gt_ray_endpoint - e_ray_endpoint
|
|
sim.rays.append(Ray(e_ray_endpoint, diff.mag, diff.norm(), vs.color.orange))
|
|
|
|
# Recomputing AAE (using law of cosines) sum(AE)/N
|
|
assert(round(sum(AE)/N - AAE) == 0)
|
|
|
|
self.sim = sim
|
|
print
|
|
|
|
def p3dEx():
|
|
ex = Experiment1p5()
|
|
ex.performExperiment()
|
|
return ex
|
|
|
|
def experiment2(s = 1):
|
|
sim = GazeSimulation(log = False)
|
|
sim.place_eyeball_on_scene_camera = True
|
|
|
|
# 2/3 for calibration, 1/3 for test
|
|
# sim.num_calibration = 66
|
|
# sim.num_test = 33
|
|
# sim.reset()
|
|
|
|
# calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth
|
|
# con = [True, False, False, False, True]
|
|
# con = [False, True, False, True, False]
|
|
con = [True, False, True, False, True]
|
|
# sim.runSimulation()
|
|
# sim.num_calibration = 36
|
|
sim.num_calibration = 25
|
|
sim.num_test = 25
|
|
sim.runCustomSimulation(con)
|
|
|
|
# retrieving calibration points (t for 3D target positions and p for 2D pupil locations)
|
|
t, p = sim.calibration_points, sim.tr_pupil_locations
|
|
# t = sim.tr_target_projections # projections of target points
|
|
# target positions are computed relative to the scene CCS
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # verified
|
|
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
|
|
# computing the initial mapping matrix w by solving the energy minimization
|
|
# for e=(0, 0, 0)
|
|
# w = findInitialW(p, t, True)
|
|
# print w
|
|
# w, e = minimizeEnergy(p, t)
|
|
# print w
|
|
w = np.array([[s, 0],
|
|
[0, s]])
|
|
# w = minimizeUsingScaleVector(p, t)
|
|
|
|
# e = np.array(v(sim.sclera_pos) - v(sim.scene_camera.t)) # eyeball coords in scene CCS
|
|
# w = minimizeEnergyGivenE(p, t, e)
|
|
|
|
t, p = sim.test_points, sim.te_pupil_locations
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # target coords in scene CCS
|
|
|
|
# TODO, is this necessary? (due to eye camera rotation)
|
|
p = map(lambda pt: np.array([-pt[0], pt[1]]), p)
|
|
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
|
|
# this w can be used for 2D-2D mapping (apparently!) testing it
|
|
qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D
|
|
# computing unit gaze vectors corresponding to pupil positions
|
|
# here we use the computed mapping matrix w
|
|
gis = map(lambda q: g(q, w), qi)
|
|
|
|
# now we can compare unit gaze vectors with their corresponding gaze rays t
|
|
# normalizing gaze rays
|
|
t = map(lambda vec: v(vec).norm(), t)
|
|
SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t))
|
|
# return SAE / len(t)
|
|
|
|
sim.display_test_points = True
|
|
sim.display_calibration_points = False
|
|
## Visualizing gaze vector
|
|
ray_scale_factor = 120
|
|
ground_truth_gaze_rays = []
|
|
for pt in t:
|
|
ground_truth_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.green))
|
|
sim.rays = ground_truth_gaze_rays
|
|
|
|
estimated_gaze_rays = []
|
|
for pt in gis:
|
|
estimated_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.red))
|
|
sim.rays.extend(estimated_gaze_rays)
|
|
|
|
AE = []
|
|
for pair in zip(t, gis):
|
|
gt_r, e_r = pair
|
|
base = v(sim.scene_camera.t)
|
|
gt_ray_endpoint = base + (v(gt_r) * ray_scale_factor)
|
|
e_ray_endpoint = base + (v(e_r) * ray_scale_factor)
|
|
AE.append(getAngularDiff(gt_ray_endpoint, e_ray_endpoint, base))
|
|
|
|
diff = gt_ray_endpoint - e_ray_endpoint
|
|
sim.rays.append(Ray(gt_ray_endpoint, diff.mag, -diff.norm(), vs.color.orange))
|
|
|
|
return sim
|
|
# print e
|
|
|
|
def experiment3():
|
|
sim = GazeSimulation(log = False)
|
|
# 2/3 for calibration, 1/3 for test
|
|
# sim.num_calibration = 66
|
|
# sim.num_test = 33
|
|
# sim.reset()
|
|
|
|
# calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth
|
|
con = [True, False, False, True, False]
|
|
# con = [False, True, False, True, False]
|
|
# sim.runSimulation()
|
|
sim.runCustomSimulation(con)
|
|
|
|
# retrieving calibration points (t for 3D target positions and p for 2D pupil locations)
|
|
t, p = sim.calibration_points, sim.tr_pupil_locations
|
|
# t = sim.tr_target_projections # projections of target points
|
|
# target positions are computed relative to the scene CCS
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t)
|
|
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
|
|
# Applying regression using LARS (Least Angle Regression)
|
|
qi = map(_q, p)
|
|
clf = linear_model.Lars(n_nonzero_coefs=np.inf) # n_nonzero_coefs=1
|
|
t = map(lambda vec: v(vec).norm(), t)
|
|
clf.fit(qi, t)
|
|
|
|
t, p = sim.test_points, sim.te_pupil_locations
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t)
|
|
# p = sim.eye_camera.getNormalizedPts(p)
|
|
|
|
# this w can be used for 2D-2D mapping (apparently!) testing it
|
|
qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D
|
|
# computing unit gaze vectors corresponding to pupil positions
|
|
# here we use the computed mapping matrix w
|
|
gis = map(lambda q: clf.predict(q)[0], qi)
|
|
|
|
# now we can compare unit gaze vectors with their corresponding gaze rays t
|
|
# normalizing gaze rays
|
|
t = map(lambda vec: v(vec).norm(), t)
|
|
SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t))
|
|
print SAE / len(t)
|
|
# print e
|
|
|
|
def experimentGT():
|
|
## Here we are using the ground truth gaze vectors (from the eye) and we compare
|
|
## them with ground truth vectors initiating from the scene camera
|
|
## In case eyeball and scene camera are located at the same point, the AAE should
|
|
## be reported as zero, this is only to make sure we're doing the right thing
|
|
|
|
sim = GazeSimulation(log = False)
|
|
# Uncomment to shift eyeball to scene camera
|
|
# sim.place_eyeball_on_scene_camera = True
|
|
|
|
# 2/3 for calibration, 1/3 for test
|
|
# sim.num_calibration = 66
|
|
# sim.num_test = 33
|
|
# sim.reset()
|
|
|
|
# calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth
|
|
con = [True, False, False, True, False]
|
|
# con = [False, True, False, True, False]
|
|
# sim.runSimulation()
|
|
|
|
sim.runCustomSimulation(con)
|
|
|
|
# retrieving calibration points (t for 3D target positions and p for 2D pupil locations)
|
|
t, p, tp = sim.calibration_points, sim.tr_pupil_locations, sim.tr_target_projections
|
|
# t = sim.tr_target_projections # projections of target points
|
|
# target positions are computed relative to the scene CCS
|
|
|
|
t_org = t[:]
|
|
t = map(lambda target: v(target) - v(sim.scene_camera.t), t)
|
|
|
|
rays_from_scene = []
|
|
for pt in t:
|
|
rays_from_scene.append(Ray(sim.scene_camera.t, v(pt).mag, v(pt).norm(), vs.color.red))
|
|
sim.rays = rays_from_scene
|
|
|
|
|
|
rays_from_scene_to_target_projections = []
|
|
base = v(sim.scene_camera.t) + v(sim.scene_camera.direction) * sim.scene_camera.f
|
|
for pp in tp:
|
|
# print pp
|
|
# since we wanna draw rays starting from scene camera,
|
|
# ray is relative to scene CCS
|
|
ray = base + v(pp[0], pp[1], 0) - v(sim.scene_camera.t)
|
|
rays_from_scene_to_target_projections.append(Ray(v(sim.scene_camera.t), ray.mag * 4, ray.norm(), vs.color.orange))
|
|
sim.rays.extend(rays_from_scene_to_target_projections)
|
|
|
|
t = map(lambda vec: v(vec).norm(), t)
|
|
|
|
alpha_GT = []
|
|
_deg = lambda triple: (np.degrees(triple[0]), np.degrees(triple[1]), triple[2])
|
|
print
|
|
for pt in t:
|
|
alpha_GT.append(_deg(getSphericalCoords(pt[0], pt[1], pt[2])))
|
|
# print _deg(getSphericalCoords(pt[0], pt[1], pt[2])), pt[2]
|
|
print
|
|
|
|
# starting from eyeball
|
|
base = v(sim.sclera_pos)
|
|
# direction vector from eyeball towards eye camera
|
|
d = -v(sim.eye_camera.direction).norm()
|
|
# calculating distance between eyeball and image plane of the eye camera
|
|
dist = (v(sim.eye_camera.t) - v(sim.sclera_pos)).mag - sim.eye_camera.f
|
|
|
|
p = map(lambda p: np.array((-p[0], p[1])), p) # -x due to orientation
|
|
sf = 4
|
|
p = map(lambda p: p.dot(np.array([[sf, 0],
|
|
[0 , sf]])), p)
|
|
gis = map(lambda p: (d*dist + v(p[0], p[1], 0)).norm(), p)
|
|
|
|
alpha_test = []
|
|
|
|
rays_from_eye = []
|
|
for pt in gis:
|
|
alpha_test.append(_deg(getSphericalCoords(pt[0], pt[1], pt[2])))
|
|
rays_from_eye.append(Ray(sim.sclera_pos, 200, pt, vs.color.yellow))
|
|
sim.rays.extend(rays_from_eye)
|
|
|
|
# gis = map(lambda t: (v(t) - v(sim.sclera_pos)).norm(), t_org)
|
|
|
|
# now we can compare unit gaze vectors with their corresponding gaze rays t
|
|
# normalizing gaze rays
|
|
|
|
# for _t, _gis in zip(t, gis): print _t, _gis
|
|
SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t))
|
|
print SAE / len(t)
|
|
return sim
|
|
|
|
|
|
|
|
# experiment1()
|
|
|
|
# for s in np.arange(1, 20, 0.25):
|
|
# print experiment2(s), s
|
|
|
|
# experiment3()
|
|
|
|
# experimentGT()
|
|
# experiment1p5() |