from __future__ import division import numpy as np from numpy import linalg as LA from time import time from sklearn import linear_model from sim import GazeSimulation from svis import Ray from minimize import findInitialW, _q, g, minimizeEnergy, minimizeEnergyGivenE, minimizeUsingScaleVector from geom import getSphericalCoords, getAngularDiff import visual as vs # from visual import vector as v # for vector operations from vector import Vector as v # for vector operations class Experiment: def performExperiment(self): print 'Running Experiment...' start = time() self.__run__() self.runningTime = time() - start print 'Running Time:', self.runningTime if self.minimizationTime: print 'Minimization Time:', self.minimizationTime def __run__(self): pass class Experiment1(Experiment): def __run__(self): ''' Simply evaluates 2D gaze estimation (using pupil calibration method) ''' sim = GazeSimulation(log = False) ## Uncomment for shifting eyeball to scene camera coordinates sim.scene_eye_distance_factor = 0.6 # controls how much to move eyeball towards scene camera [0-1] sim.place_eyeball_on_scene_camera = False # calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth experiments = [ [True, False, False, True, False], [True, False, False, False, False], [True, False, False, False, True], [False, True, False, True, False], [False, True, False, False, False], [False, True, False, False, True], [False, False, False, True, False], [False, False, False, False, False], [False, False, False, False, True], # [True, False, True, False, False], ] nsim = 100 for experiment in experiments: res = [sim.runCustomSimulation(experiment) for n in xrange(nsim)] res = filter(lambda aae: aae>=0, res) M = max(res) m = min(res) aae = sum(res) / len(res) print 'AAE: %2.5f - Min AAE: %2.5f | Max AAE: %2.5f' % (aae, m, M) eye_scene_diff = v(sim.sclera_pos) - v(sim.scene_camera.t) print "Eyeball-SceneCamera distance was", eye_scene_diff.mag self.sim = sim self.minimizationTime = -1 class Experiment1p5(Experiment): def __run__(self): ''' Read 3D gaze estimation experiment ''' sim = GazeSimulation(log = False) ## Uncomment for shifting eyeball to scene camera coordinates sim.place_eyeball_on_scene_camera = True sim.scene_eye_distance_factor = 0.6 # sim.scene_eye_distance_factor = 1.0 # calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth # -> grid setting # con = [True, False, True, False, True] # -> general setting with grid learning # con = [True, False, False, True, False] con = [True, False, False, False, False] # -> general setting with random learning # con = [False, True, False, True, False] sim.num_calibration = 36 sim.num_test = 25 aae_2d = sim.runCustomSimulation(con) ## Fetching the calibration points t, p = sim.calibration_points, sim.tr_pupil_locations # target positions are computed relative to the scene CCS t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # verified # p = sim.eye_camera.getNormalizedPts(p) start = time() eye_scene_diff = v(sim.sclera_pos) - v(sim.scene_camera.t) e = np.array(eye_scene_diff) # eyeball coords in scene CCS print 'Real e', e e_org = e[:] print 'Eye-Scene distance', eye_scene_diff.mag # w, w0 = minimizeEnergyGivenE(p, t, e) ## Finding the optimal transformation matrix by minimizing the nonlinear energy # w0 is the initial w by solving the leastsq with e=(0,0,0) # w is by solving the leastsq again optimizing for both e and w w, e, w0 = minimizeEnergy(p, t) # here were are initializing minimization with an e close to the ground # truth e by adding random perturbations to it # w, e, w0 = minimizeEnergy(p, t, e + np.random.rand(1, 3)[0]) print 'Estimated e', e, '( Distance:', LA.norm(e - e_org) , ')' self.minimizationTime = time() - start print sim.w = w ## Fetching test points t, p = sim.test_points, sim.te_pupil_locations t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # target coords in scene CCS # closest point distance to scene camera cDist = min(v(pt).mag for pt in t) # farthest point distance to scene camera fDist = max(v(pt).mag for pt in t) # average point distance to scene camera avgDist = sum(v(pt).mag for pt in t)/len(t) # p = sim.eye_camera.getNormalizedPts(p) # print # print p # print qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D # computing unit gaze vectors corresponding to pupil positions # here we use the computed mapping matrix w gis = map(lambda q: g(q, w), qi) gis0 = map(lambda q: g(q, w0), qi) # now we can compare unit gaze vectors with their corresponding gaze rays t # normalizing gaze rays first t = map(lambda vec: v(vec).norm(), t) # TODO: compare spherical coordinates instead AE = list(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t)) N = len(t) AAE = sum(AE)/N VAR = sum((ae - AAE)**2 for ae in AE)/N print 'AAE:', AAE, '\nVariance:', VAR, 'STD:', np.sqrt(VAR), '\nMin:', min(AE), 'Max:', max(AE), '(N=' + str(N) + ')' print 'Target Distances: m=%s M=%s Avg=%s' % (cDist, fDist, avgDist) AE0 = list(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis0, t)) AAE0 = sum(AE0)/N print 'AAE (only optimizing W for e=(0,0,0)):', AAE0 print 'AAE (2D-to-2D mapping):', aae_2d print ('Improvement (2D-2D vs 2D-3D): %s' % round((aae_2d - AAE)*100/aae_2d, 2)) + '%' print ('Improvement (2D-2D vs 2D-2D): %s' % round((aae_2d - AAE0)*100/aae_2d, 2)) + '%' sim.display_test_points = True sim.display_calibration_points = False ## Visualizing gaze vector ray_scale_factor = 120 ground_truth_gaze_rays = [] for pt in t: ground_truth_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.green)) sim.rays = ground_truth_gaze_rays estimated_gaze_rays = [] for pt in gis: estimated_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.red)) sim.rays.extend(estimated_gaze_rays) AE = [] for pair in zip(t, gis): gt_r, e_r = pair base = v(sim.scene_camera.t) gt_ray_endpoint = base + (v(gt_r) * ray_scale_factor) e_ray_endpoint = base + (v(e_r) * ray_scale_factor) AE.append(getAngularDiff(gt_ray_endpoint, e_ray_endpoint, base)) diff = gt_ray_endpoint - e_ray_endpoint sim.rays.append(Ray(e_ray_endpoint, diff.mag, diff.norm(), vs.color.orange)) # Recomputing AAE (using law of cosines) sum(AE)/N assert(round(sum(AE)/N - AAE) == 0) self.sim = sim print def p3dEx(): ex = Experiment1p5() ex.performExperiment() return ex def experiment2(s = 1): sim = GazeSimulation(log = False) sim.place_eyeball_on_scene_camera = True # 2/3 for calibration, 1/3 for test # sim.num_calibration = 66 # sim.num_test = 33 # sim.reset() # calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth # con = [True, False, False, False, True] # con = [False, True, False, True, False] con = [True, False, True, False, True] # sim.runSimulation() # sim.num_calibration = 36 sim.num_calibration = 25 sim.num_test = 25 sim.runCustomSimulation(con) # retrieving calibration points (t for 3D target positions and p for 2D pupil locations) t, p = sim.calibration_points, sim.tr_pupil_locations # t = sim.tr_target_projections # projections of target points # target positions are computed relative to the scene CCS t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # verified # p = sim.eye_camera.getNormalizedPts(p) # computing the initial mapping matrix w by solving the energy minimization # for e=(0, 0, 0) # w = findInitialW(p, t, True) # print w # w, e = minimizeEnergy(p, t) # print w w = np.array([[s, 0], [0, s]]) # w = minimizeUsingScaleVector(p, t) # e = np.array(v(sim.sclera_pos) - v(sim.scene_camera.t)) # eyeball coords in scene CCS # w = minimizeEnergyGivenE(p, t, e) t, p = sim.test_points, sim.te_pupil_locations t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # target coords in scene CCS # TODO, is this necessary? (due to eye camera rotation) p = map(lambda pt: np.array([-pt[0], pt[1]]), p) # p = sim.eye_camera.getNormalizedPts(p) # this w can be used for 2D-2D mapping (apparently!) testing it qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D # computing unit gaze vectors corresponding to pupil positions # here we use the computed mapping matrix w gis = map(lambda q: g(q, w), qi) # now we can compare unit gaze vectors with their corresponding gaze rays t # normalizing gaze rays t = map(lambda vec: v(vec).norm(), t) SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t)) # return SAE / len(t) sim.display_test_points = True sim.display_calibration_points = False ## Visualizing gaze vector ray_scale_factor = 120 ground_truth_gaze_rays = [] for pt in t: ground_truth_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.green)) sim.rays = ground_truth_gaze_rays estimated_gaze_rays = [] for pt in gis: estimated_gaze_rays.append(Ray(sim.scene_camera.t, ray_scale_factor, v(pt), vs.color.red)) sim.rays.extend(estimated_gaze_rays) AE = [] for pair in zip(t, gis): gt_r, e_r = pair base = v(sim.scene_camera.t) gt_ray_endpoint = base + (v(gt_r) * ray_scale_factor) e_ray_endpoint = base + (v(e_r) * ray_scale_factor) AE.append(getAngularDiff(gt_ray_endpoint, e_ray_endpoint, base)) diff = gt_ray_endpoint - e_ray_endpoint sim.rays.append(Ray(gt_ray_endpoint, diff.mag, -diff.norm(), vs.color.orange)) return sim # print e def experiment3(): sim = GazeSimulation(log = False) # 2/3 for calibration, 1/3 for test # sim.num_calibration = 66 # sim.num_test = 33 # sim.reset() # calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth con = [True, False, False, True, False] # con = [False, True, False, True, False] # sim.runSimulation() sim.runCustomSimulation(con) # retrieving calibration points (t for 3D target positions and p for 2D pupil locations) t, p = sim.calibration_points, sim.tr_pupil_locations # t = sim.tr_target_projections # projections of target points # target positions are computed relative to the scene CCS t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # p = sim.eye_camera.getNormalizedPts(p) # Applying regression using LARS (Least Angle Regression) qi = map(_q, p) clf = linear_model.Lars(n_nonzero_coefs=np.inf) # n_nonzero_coefs=1 t = map(lambda vec: v(vec).norm(), t) clf.fit(qi, t) t, p = sim.test_points, sim.te_pupil_locations t = map(lambda target: v(target) - v(sim.scene_camera.t), t) # p = sim.eye_camera.getNormalizedPts(p) # this w can be used for 2D-2D mapping (apparently!) testing it qi = map(_q, p) # computing feature vectors from raw pupil coordinates in 2D # computing unit gaze vectors corresponding to pupil positions # here we use the computed mapping matrix w gis = map(lambda q: clf.predict(q)[0], qi) # now we can compare unit gaze vectors with their corresponding gaze rays t # normalizing gaze rays t = map(lambda vec: v(vec).norm(), t) SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t)) print SAE / len(t) # print e def experimentGT(): ## Here we are using the ground truth gaze vectors (from the eye) and we compare ## them with ground truth vectors initiating from the scene camera ## In case eyeball and scene camera are located at the same point, the AAE should ## be reported as zero, this is only to make sure we're doing the right thing sim = GazeSimulation(log = False) # Uncomment to shift eyeball to scene camera # sim.place_eyeball_on_scene_camera = True # 2/3 for calibration, 1/3 for test # sim.num_calibration = 66 # sim.num_test = 33 # sim.reset() # calibration_grid, calibration_random_depth, test_grid, test_random_depth, test_random_fixed_depth con = [True, False, False, True, False] # con = [False, True, False, True, False] # sim.runSimulation() sim.runCustomSimulation(con) # retrieving calibration points (t for 3D target positions and p for 2D pupil locations) t, p, tp = sim.calibration_points, sim.tr_pupil_locations, sim.tr_target_projections # t = sim.tr_target_projections # projections of target points # target positions are computed relative to the scene CCS t_org = t[:] t = map(lambda target: v(target) - v(sim.scene_camera.t), t) rays_from_scene = [] for pt in t: rays_from_scene.append(Ray(sim.scene_camera.t, v(pt).mag, v(pt).norm(), vs.color.red)) sim.rays = rays_from_scene rays_from_scene_to_target_projections = [] base = v(sim.scene_camera.t) + v(sim.scene_camera.direction) * sim.scene_camera.f for pp in tp: # print pp # since we wanna draw rays starting from scene camera, # ray is relative to scene CCS ray = base + v(pp[0], pp[1], 0) - v(sim.scene_camera.t) rays_from_scene_to_target_projections.append(Ray(v(sim.scene_camera.t), ray.mag * 4, ray.norm(), vs.color.orange)) sim.rays.extend(rays_from_scene_to_target_projections) t = map(lambda vec: v(vec).norm(), t) alpha_GT = [] _deg = lambda triple: (np.degrees(triple[0]), np.degrees(triple[1]), triple[2]) print for pt in t: alpha_GT.append(_deg(getSphericalCoords(pt[0], pt[1], pt[2]))) # print _deg(getSphericalCoords(pt[0], pt[1], pt[2])), pt[2] print # starting from eyeball base = v(sim.sclera_pos) # direction vector from eyeball towards eye camera d = -v(sim.eye_camera.direction).norm() # calculating distance between eyeball and image plane of the eye camera dist = (v(sim.eye_camera.t) - v(sim.sclera_pos)).mag - sim.eye_camera.f p = map(lambda p: np.array((-p[0], p[1])), p) # -x due to orientation sf = 4 p = map(lambda p: p.dot(np.array([[sf, 0], [0 , sf]])), p) gis = map(lambda p: (d*dist + v(p[0], p[1], 0)).norm(), p) alpha_test = [] rays_from_eye = [] for pt in gis: alpha_test.append(_deg(getSphericalCoords(pt[0], pt[1], pt[2]))) rays_from_eye.append(Ray(sim.sclera_pos, 200, pt, vs.color.yellow)) sim.rays.extend(rays_from_eye) # gis = map(lambda t: (v(t) - v(sim.sclera_pos)).norm(), t_org) # now we can compare unit gaze vectors with their corresponding gaze rays t # normalizing gaze rays # for _t, _gis in zip(t, gis): print _t, _gis SAE = sum(np.degrees(np.arctan((v(p[0]).cross(p[1])/(v(p[0]).dot(p[1]))).mag)) for p in zip(gis, t)) print SAE / len(t) return sim # experiment1() # for s in np.arange(1, 20, 0.25): # print experiment2(s), s # experiment3() # experimentGT() # experiment1p5()