updated code with more comments
This commit is contained in:
mohsen-mansouryar
parent
460a71d2b3
commit
67123bb970
11 changed files with 262 additions and 1116 deletions
106
code/sim.py
106
code/sim.py
|
|
@ -1,29 +1,36 @@
|
|||
from __future__ import division
|
||||
|
||||
'''
|
||||
This is the core part of the simulation framework.
|
||||
I am basically decoupling logic from visualization, this module covers the logic.
|
||||
This module mainly covers the logic meaning that you can create
|
||||
a GazeSimulation object, specify the environment and and then
|
||||
simply run the experiment with no visualization to get the data.
|
||||
|
||||
However, you can as well visualize the environment after the simulation
|
||||
environment is set up and you ran the experiment. in order to do so,
|
||||
after you have a GazeSimulation object, simply call visualize() method on it.
|
||||
'''
|
||||
|
||||
# import visual as vs
|
||||
# from visual import vector as v # for vector operations
|
||||
from vector import Vector as v # for vector operations
|
||||
try:
|
||||
import visual as vs
|
||||
from visual import vector as v
|
||||
except ImportError:
|
||||
from vector import Vector as vector
|
||||
import numpy as np
|
||||
import cv2, cv
|
||||
|
||||
# from sklearn.neighbors import KNeighborsRegressor as knn
|
||||
# ## (u, v, u^2, v^2, uv, u^2*v^2, 1)
|
||||
# _q = lambda p: (p[0], p[1], p[0]*p[0], p[1]*p[1], p[0]*p[1], p[0]*p[1]*p[0]*p[1], 1)
|
||||
# _qi = lambda q: (q[0], q[1])
|
||||
|
||||
# from util import *
|
||||
from geom import *
|
||||
# from svis import Visualizable # Uncomment if you want visualization
|
||||
## Uncomment the following import if you want visualization
|
||||
# from svis import Visualizable
|
||||
|
||||
# 2D to 2D calibration method from the pupil project
|
||||
from pupil.calibrate import get_map_from_cloud
|
||||
|
||||
# a factor to convert from centimeter to millimeter later when
|
||||
# generating data (unit for our measurements is millimeter)
|
||||
point_scale_factor = 10
|
||||
|
||||
## Uncomment the following if you want visualization (and comment the line after)
|
||||
# class GazeSimulation(Visualizable):
|
||||
class GazeSimulation:
|
||||
_log = True
|
||||
|
|
@ -43,9 +50,9 @@ class GazeSimulation:
|
|||
# *unit for measurement is millimeter
|
||||
|
||||
# This direction should always be towards the target 3D point
|
||||
# this is just the default value
|
||||
# this is just the default value, adjust_eye method updates this vector
|
||||
pupil_direction = v(1, 0, 0)
|
||||
pupil_position = None # computed whenever target is changed
|
||||
pupil_position = None # 3D point which is computed whenever target is changed
|
||||
pupil_positions = [] # 3D positions of the pupil to be projected onto the eye camera's image plane
|
||||
|
||||
sclera_pos = v(-15, 20, -25) # wrt world coordinate system
|
||||
|
|
@ -53,6 +60,9 @@ class GazeSimulation:
|
|||
|
||||
target_position = v(0, 0, 0)
|
||||
def recomputeEyeInner(self):
|
||||
'''
|
||||
This follows our eye model to compute the position of cornea at every time.
|
||||
'''
|
||||
self.cornea_pos = self.sclera_pos + self.pupil_direction * self.sclera_cornea_center_offset
|
||||
|
||||
place_eyeball_on_scene_camera = False
|
||||
|
|
@ -109,18 +119,15 @@ class GazeSimulation:
|
|||
self.pupil_position = cornea_pos + self.pupil_direction * self.cornea_limbus_offset
|
||||
################################################################################
|
||||
################################################################################
|
||||
## calibration and Test Data
|
||||
## Calibration and Test Data
|
||||
num_calibration = 25 # calibration points
|
||||
num_test = 16
|
||||
# TODO: consider the least number of calibration points required, also investigate if accuracy is
|
||||
# significantly improved with more points (my guess is that it will not)
|
||||
|
||||
# TODO: considering measurement unit is mm, adjust these parameters accordingly to simulate more
|
||||
# realistic settings
|
||||
# TODO: also consider the range of eye movement (50x50 degrees) to compute realistic boundaries
|
||||
# for position of gaze targets
|
||||
min_xyz = -55 * point_scale_factor, -45 * point_scale_factor, 1000
|
||||
max_xyz = 55 * point_scale_factor, 45 * point_scale_factor, 2000
|
||||
|
||||
# These are considered if we have 25 calibration points and 16 test points,
|
||||
# Changing those parameters require to recompute these values
|
||||
grid_x_width = 22 * point_scale_factor
|
||||
grid_y_width = 18 * point_scale_factor
|
||||
calibration_depth, test_depth = None, None
|
||||
|
|
@ -131,9 +138,11 @@ class GazeSimulation:
|
|||
test_random_depth = False
|
||||
test_random_fixed_depth = False
|
||||
def generateCalibrationPts(self, ofr = 0):
|
||||
# For now we're considering 25 calibration points (5x5 grid) and 16 test points (inner 4x4 grid)
|
||||
# as per the real world experiment. these parameters could be given though, they're hard coded for
|
||||
# simplicity
|
||||
'''
|
||||
For now we're considering 25 calibration points (5x5 grid) and 16 test points (inner 4x4 grid)
|
||||
as per the real world experiment. these parameters could be given though, they're hard coded for
|
||||
simplicity
|
||||
'''
|
||||
self.calibration_points = generatePoints(25, self.min_xyz, self.max_xyz,
|
||||
grid=self.calibration_grid,
|
||||
randomZ=self.calibration_random_depth,
|
||||
|
|
@ -141,11 +150,14 @@ class GazeSimulation:
|
|||
xoffset=self.scene_camera.t[0],
|
||||
yoffset=self.scene_camera.t[1],
|
||||
zoffset=self.scene_camera.t[2])
|
||||
# Test point for checking spherical coordinates
|
||||
# self.calibration_points.append(np.array(v(self.scene_camera.t)+120*v(self.scene_camera.direction) + v(-10, 0, 0)))
|
||||
def generateTestPts(self, ofr = 0): # TODO
|
||||
|
||||
def generateTestPts(self, ofr = 0):
|
||||
'''
|
||||
Same as above, the values used here only apply to our setup, simply replace them with your
|
||||
own values or introduce new variables to compute them at runtime. (due to our tight time schedule
|
||||
this part of the code uses hardcoded material)
|
||||
'''
|
||||
min_xyz, max_xyz = self.min_xyz, self.max_xyz
|
||||
# if self.num_calibration == 25 and self.num_test == 16 and self.test_grid and self.calibration_grid:
|
||||
min_xyz = -55 * point_scale_factor+self.grid_x_width/2., -45 * point_scale_factor+self.grid_y_width/2., 1000
|
||||
max_xyz = 55 * point_scale_factor-self.grid_x_width/2., 45 * point_scale_factor-self.grid_y_width/2., 2000
|
||||
# The above two lines are also currently hard coded only to match the inner grid with our setting
|
||||
|
|
@ -158,10 +170,10 @@ class GazeSimulation:
|
|||
xoffset=self.scene_camera.t[0],
|
||||
yoffset=self.scene_camera.t[1],
|
||||
zoffset=self.scene_camera.t[2])
|
||||
# self.test_points = map(lambda p: v(p)-v(self.scene_camera.t), self.test_points)
|
||||
|
||||
def setCalibrationDepth(self, depth):
|
||||
'''
|
||||
sets calibration depth wrt to scene camera
|
||||
Sets calibration depth wrt to scene camera
|
||||
'''
|
||||
self.calibration_random_depth = False
|
||||
if isinstance(depth, list):
|
||||
|
|
@ -173,7 +185,7 @@ class GazeSimulation:
|
|||
|
||||
def setTestDepth(self, depth):
|
||||
'''
|
||||
sets test depth wrt to scene camera
|
||||
Sets test depth wrt to scene camera
|
||||
'''
|
||||
self.test_random_depth = False
|
||||
self.test_random_fixed_depth = False
|
||||
|
|
@ -181,29 +193,26 @@ class GazeSimulation:
|
|||
self.reset()
|
||||
|
||||
|
||||
# temporary variable to store active gaze point during the experiment
|
||||
active_gaze_point = None
|
||||
################################################################################
|
||||
################################################################################
|
||||
## Camera Setup
|
||||
def setupCameras(self):
|
||||
# Eye Camera
|
||||
self.eye_camera = Camera(label='Eye Camera', f = 13, fov=np.radians(50)) # TODO: f and fov must be realistic
|
||||
# self.eye_camera.setT((-15, 20, 0))
|
||||
self.eye_camera = Camera(label='Eye Camera', f = 13, fov=np.radians(50))
|
||||
self.eye_camera.setT((self.sclera_pos + v(0, 0,
|
||||
self.sclera_cornea_center_offset + \
|
||||
self.cornea_radius + \
|
||||
45)).tuple()) # 35mm in front of the surface of the eye
|
||||
45)).tuple()) # 45mm in front of the surface of the eye
|
||||
# rotating around y axis, camera now pointing towards negative Z
|
||||
# with this rotation, also x becomes inverted
|
||||
self.eye_camera.setR((0, np.pi, 0))
|
||||
# self.eye_camera.setDir((0, 0, -1)) # deprecated
|
||||
|
||||
# Scene Camera
|
||||
self.scene_camera = Camera(label='Scene Camera', f = 50, fov=np.radians(100)) # TODO: f and fov must be realistic
|
||||
# self.scene_camera.setT((0, 20, 0))
|
||||
self.scene_camera.setT((self.sclera_pos + v(-25, 25, 25)).tuple()) # sclera_pos is v(-15, 20, -25)
|
||||
self.scene_camera = Camera(label='Scene Camera', f = 50, fov=np.radians(100))
|
||||
self.scene_camera.setT((self.sclera_pos + v(-25, 25, 25)).tuple()) # sclera_pos is by default v(-15, 20, -25)
|
||||
self.scene_camera.setR((0, 0, 0)) # camera pointing towards positive Z
|
||||
# self.scene_camera.setDir((0, 0, 1)) # deprecated
|
||||
################################################################################
|
||||
################################################################################
|
||||
## Simulations Steps
|
||||
|
|
@ -229,7 +238,7 @@ class GazeSimulation:
|
|||
|
||||
def gazeAtNextPoint(self, calibration = True):
|
||||
'''
|
||||
Gazes at the next calibration/Test target point and records both pupil and
|
||||
Gazes at the next Calibration/Test target point and records both pupil and
|
||||
target positions.
|
||||
'''
|
||||
if calibration:
|
||||
|
|
@ -263,6 +272,7 @@ class GazeSimulation:
|
|||
targets = map(lambda p:v(p), self.calibration_points)
|
||||
targets = map(lambda p: self.scene_camera.project((p.x, p.y, p.z, 1)), targets)
|
||||
pupil_locations = map(lambda p: self.eye_camera.project((p.x, p.y, p.z, 1)), self.pupil_positions)
|
||||
|
||||
# Filtering out points outside of the image plane
|
||||
old = len(targets)
|
||||
invalid = []
|
||||
|
|
@ -276,8 +286,7 @@ class GazeSimulation:
|
|||
pupil_locations = pupil_locations[:i] + pupil_locations[i+1:]
|
||||
self.calibration_points = self.calibration_points[:i] + self.calibration_points[i+1:]
|
||||
self.log('Removed %s invalid calibration points...' % (old-len(targets)))
|
||||
# print 'Removed %s invalid target points...' % (old-len(targets))
|
||||
|
||||
|
||||
# to get calibration points p and t, refer to self.calibration_points and
|
||||
# self.pupil_locations (also target_projections has the corresponding projection
|
||||
# points for the targets)
|
||||
|
|
@ -309,10 +318,7 @@ class GazeSimulation:
|
|||
|
||||
self.mp = get_map_from_cloud(np.array([(pupil_locations[i][0], pupil_locations[i][1],
|
||||
targets[i][0], targets[i][1]) for i in xrange(len(targets))]))
|
||||
# ngh = knn(n_neighbors = 1)
|
||||
# ngh.fit(map(_q, pupil_locations), map(_q, targets))
|
||||
# self.mp = lambda p: _qi(ngh.predict(_q(p))[0])
|
||||
|
||||
|
||||
self.log('Successfully computed map...')
|
||||
# Converting these to numpy arrays to support nice operands
|
||||
targets = np.array(targets)
|
||||
|
|
@ -401,7 +407,6 @@ class GazeSimulation:
|
|||
APHE = np.mean(PHE)
|
||||
PHE_STD = np.std(PHE)
|
||||
PHE_m, PHE_M = min(PHE), max(PHE)
|
||||
# ret.extend([APHE, PHE_VAR, PHE_STD, PHE_m, PHE_M])
|
||||
return AAE, PHE, AE_STD, PHE_STD
|
||||
|
||||
################################################################################
|
||||
|
|
@ -417,7 +422,6 @@ class GazeSimulation:
|
|||
|
||||
def runTest(self):
|
||||
self.processData(calibration = False)
|
||||
# AAE = self.projectAndMap()
|
||||
return self.projectAndMap()
|
||||
|
||||
def runSimulation(self):
|
||||
|
|
@ -473,15 +477,11 @@ class GazeSimulation:
|
|||
VAR = sum((pe - APE)**2 for pe in PE)/N
|
||||
STD = np.sqrt(VAR)
|
||||
m, M = min(PE), max(PE)
|
||||
|
||||
# ret.append(['pixel_error: APE, VAR, STD, min, MAX, data', APE, VAR, STD, m, M, PE])
|
||||
ret.extend([APE, VAR, STD, m, M])
|
||||
|
||||
self.log('Computing 3D back projection of estimated points...')
|
||||
# mapped_3d = cv2.undistortPoints(np.array([mapped_targets]), cameraMatrix=camera_matrix, distCoeffs=np.array([0, 0, 0, 0, 0]))
|
||||
mapped_3d = cv2.undistortPoints(np.array([mapped_targets]), cameraMatrix=camera_matrix, distCoeffs=dist_coeffs)
|
||||
mapped_3d = map(lambda p: [p[0], p[1], 1], mapped_3d[0])
|
||||
# print v(mapped_3d[0]).mag, np.sqrt(mapped_3d[0][0]**2 + mapped_3d[0][1]**2)
|
||||
|
||||
AE = list(np.degrees(np.arctan((v(p[0]).cross(v(p[1]))/(v(p[0]).dot(v(p[1])))).mag)) for p in zip(mapped_3d, targets_3d))
|
||||
N = len(targets_3d)
|
||||
|
|
@ -534,6 +534,10 @@ class GazeSimulation:
|
|||
rays = []
|
||||
|
||||
def draw(self):
|
||||
'''
|
||||
This visualizes the simulation environment. be careful about the lines you have to
|
||||
uncomment at the top of this module so that this part works.
|
||||
'''
|
||||
from util import drawAxes, drawLine
|
||||
from svis import drawCameraFrame
|
||||
## World Axes
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue