initial commit
Xucong Zhang
4 years ago
commit
e505acdb29
@ -0,0 +1,32 @@
|
|||||||
|
### C++ ###
|
||||||
|
# Prerequisites
|
||||||
|
*.d
|
||||||
|
|
||||||
|
# Compiled Object files
|
||||||
|
*.slo
|
||||||
|
*.lo
|
||||||
|
*.o
|
||||||
|
*.obj
|
||||||
|
|
||||||
|
# Compiled Dynamic libraries
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
*.dll
|
||||||
|
|
||||||
|
# Compiled Static libraries
|
||||||
|
*.lai
|
||||||
|
*.la
|
||||||
|
*.a
|
||||||
|
*.lib
|
||||||
|
|
||||||
|
### CMake ###
|
||||||
|
CMakeLists.txt.user
|
||||||
|
CMakeCache.txt
|
||||||
|
CMakeFiles
|
||||||
|
CMakeScripts
|
||||||
|
Testing
|
||||||
|
Makefile
|
||||||
|
cmake_install.cmake
|
||||||
|
install_manifest.txt
|
||||||
|
compile_commands.json
|
||||||
|
CTestTestfile.cmake
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.0)
|
||||||
|
project(OpenGaze VERSION 0.1)
|
||||||
|
set(CMAKE_BUILD_TYPE Release)
|
||||||
|
|
||||||
|
# create a directory for models and configuration files
|
||||||
|
set(OPENGAZE_DIR "$ENV{HOME}/OpenGaze")
|
||||||
|
add_definitions(-DOPENGAZE_CON_DIR="${OPENGAZE_DIR}")
|
||||||
|
|
||||||
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/)
|
||||||
|
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/)
|
||||||
|
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
||||||
|
set(CMAKE_CXX_STANDARD 11)
|
||||||
|
|
||||||
|
# OpenCV
|
||||||
|
find_package( OpenCV 3.4 REQUIRED COMPONENTS core imgproc calib3d highgui objdetect)
|
||||||
|
|
||||||
|
# Boost, for reading configuration file
|
||||||
|
find_package(Boost 1.5 COMPONENTS system filesystem timer thread program_options REQUIRED)
|
||||||
|
set(Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIR} ${Boost_INCLUDE_DIR}/boost)
|
||||||
|
|
||||||
|
# Caffe
|
||||||
|
set(CAFFE_INSTALL_DIR "/home/xucong/library/caffe/build/install")
|
||||||
|
set(Caffe_INCLUDE_DIRS ${CAFFE_INSTALL_DIR}/include)
|
||||||
|
set(Caffe_LIBRARY_DIRS ${CAFFE_INSTALL_DIR}/lib)
|
||||||
|
set(Caffe_LIBS lmdb glog caffe)
|
||||||
|
|
||||||
|
# Face and facial landmark detection methods
|
||||||
|
option(USE_OPENFACE "with OpenFace" ON)
|
||||||
|
add_definitions(-DUSE_OPENFACE=1)
|
||||||
|
# OpenFace
|
||||||
|
set(OPENFACE_ROOT_DIR "/home/xucong/library/OpenFace")
|
||||||
|
add_definitions(-DOPENFACE_DIR="${OPENFACE_ROOT_DIR}")
|
||||||
|
set(CLM_INCLUDE_DIRS ${OPENFACE_ROOT_DIR}/lib/local/LandmarkDetector/include)
|
||||||
|
set(CLM_LIBRARY_DIRS ${OPENFACE_ROOT_DIR}/build/lib/local/LandmarkDetector)
|
||||||
|
set(CLM_LIBS LandmarkDetector tbb openblas dlib)
|
||||||
|
set(USE_OPENFACE ON) # we use OpenFace method here
|
||||||
|
|
||||||
|
# suppress auto_ptr deprecation warnings
|
||||||
|
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
|
||||||
|
add_compile_options("-Wno-deprecated-declarations")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
include_directories(./ ./include /usr/local/cuda/include ${OpenCV_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${CLM_INCLUDE_DIRS} ${Caffe_INCLUDE_DIRS})
|
||||||
|
link_directories(./ ./build/lib /usr/lib /usr/local/cuda/lib64 ${Boost_LIBRARY_DIRS} ${CLM_LIBRARY_DIRS} ${Caffe_LIBRARY_DIRS})
|
||||||
|
|
||||||
|
file(GLOB SOURCE "./src/*.cpp")
|
||||||
|
file(GLOB HEADERS "./include/*.hpp")
|
||||||
|
|
||||||
|
# compile opengaze library
|
||||||
|
add_library(opengaze SHARED ${SOURCE} ${HEADERS})
|
||||||
|
set_target_properties(opengaze PROPERTIES VERSION ${PROJECT_VERSION})
|
||||||
|
|
||||||
|
|
||||||
|
#if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
||||||
|
# set (CMAKE_INSTALL_PREFIX "/usr/local" CACHE PATH "default install path" FORCE )
|
||||||
|
#endif()
|
||||||
|
|
||||||
|
install (TARGETS opengaze EXPORT OpenGazeTargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib)
|
||||||
|
install (FILES ${HEADERS} DESTINATION include/opengaze)
|
||||||
|
|
||||||
|
# install caffe and OpenFace
|
||||||
|
install (DIRECTORY DESTINATION "${OPENGAZE_DIR}/3rdParty" DIRECTORY_PERMISSIONS
|
||||||
|
OWNER_WRITE OWNER_READ OWNER_EXECUTE
|
||||||
|
GROUP_WRITE GROUP_READ GROUP_EXECUTE
|
||||||
|
WORLD_WRITE WORLD_READ WORLD_EXECUTE)
|
||||||
|
install (FILES ${OPENFACE_ROOT_DIR}/build/lib/local/LandmarkDetector/libLandmarkDetector.a DESTINATION ${OPENGAZE_DIR}/3rdParty)
|
||||||
|
install (FILES ${Caffe_LIBRARY_DIRS}/libcaffe.so DESTINATION ${OPENGAZE_DIR}/3rdParty)
|
||||||
|
install (FILES ${Caffe_LIBRARY_DIRS}/libcaffe.so.1.0.0 DESTINATION ${OPENGAZE_DIR}/3rdParty)
|
||||||
|
|
||||||
|
# install configuration files
|
||||||
|
install (DIRECTORY DESTINATION "${OPENGAZE_DIR}" DIRECTORY_PERMISSIONS
|
||||||
|
OWNER_WRITE OWNER_READ OWNER_EXECUTE
|
||||||
|
GROUP_WRITE GROUP_READ GROUP_EXECUTE
|
||||||
|
WORLD_WRITE WORLD_READ WORLD_EXECUTE)
|
||||||
|
install (DIRECTORY DESTINATION "${OPENGAZE_DIR}/content" DIRECTORY_PERMISSIONS
|
||||||
|
OWNER_WRITE OWNER_READ OWNER_EXECUTE
|
||||||
|
GROUP_WRITE GROUP_READ GROUP_EXECUTE
|
||||||
|
WORLD_WRITE WORLD_READ WORLD_EXECUTE)
|
||||||
|
install (DIRECTORY DESTINATION "${OPENGAZE_DIR}/content/calib" DIRECTORY_PERMISSIONS
|
||||||
|
OWNER_WRITE OWNER_READ OWNER_EXECUTE
|
||||||
|
GROUP_WRITE GROUP_READ GROUP_EXECUTE
|
||||||
|
WORLD_WRITE WORLD_READ WORLD_EXECUTE)
|
||||||
|
install (DIRECTORY DESTINATION "${OPENGAZE_DIR}/content/model" DIRECTORY_PERMISSIONS
|
||||||
|
OWNER_WRITE OWNER_READ OWNER_EXECUTE
|
||||||
|
GROUP_WRITE GROUP_READ GROUP_EXECUTE
|
||||||
|
WORLD_WRITE WORLD_READ WORLD_EXECUTE)
|
||||||
|
install (FILES ./content/calib/calibration.yml DESTINATION ${OPENGAZE_DIR}/content/calib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_READ WORLD_WRITE WORLD_EXECUTE)
|
||||||
|
install (FILES ./content/calib/monitor_laptop.yml DESTINATION ${OPENGAZE_DIR}/content/calib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_READ WORLD_WRITE WORLD_EXECUTE)
|
||||||
|
install (FILES ./content/model/face_model.yml DESTINATION ${OPENGAZE_DIR}/content/model PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_READ WORLD_WRITE WORLD_EXECUTE)
|
||||||
|
install (FILES default.cfg DESTINATION ${OPENGAZE_DIR} PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_READ WORLD_WRITE WORLD_EXECUTE)
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
# OpenGaze: Open Source Toolkit for Camera-Based Gaze Estimation and Interaction
|
||||||
|
|
||||||
|
<!--The current demo video includes clips from Friends, which may violate the copyright. Although people think 28 seconds could be a boundary:https://productforums.google.com/forum/#!topic/youtube/rQhkI20Rm8k, there is no golden rule for it: https://www.youtube.com/yt/about/copyright/fair-use/#yt-copyright-protection-->
|
||||||
|
<!--//[](https://youtu.be/OORxOdu8USQ "OpenGaze Friends Demo")-->
|
||||||
|
|
||||||
|
Appearance-based gaze estimation methods that only require an off-the-shelf camera have significantly improved and promise a wide range of new applications in gaze-based interaction and attentive user interfaces. However, these methods are not yet widely used in the human-computer interaction (HCI) community.
|
||||||
|
|
||||||
|
To democratize their use in HCI, we present OpenGaze, the first software toolkit that is specifically developed for gaze interface designers. OpenGaze is open source and aims to implement state-of-the-art methods for camera-based gaze estimation and interaction.
|
||||||
|
|
||||||
|
<img src="https://github.molgen.mpg.de/perceptual/opengaze/blob/master/imgs/logo_mpiinf.png" height="80"/><img src="https://github.molgen.mpg.de/perceptual/opengaze/blob/master/imgs/logo_pui.png" height="80"><img src="https://github.molgen.mpg.de/perceptual/opengaze/blob/master/imgs/logo_osaka-u.png" height="80">
|
||||||
|
|
||||||
|
## Functionality
|
||||||
|
|
||||||
|
The toolkit is capable of performing the following gaze-related tasks:
|
||||||
|
|
||||||
|
* **Gaze Estimation**
|
||||||
|
Show estimated gaze on the screen given screen-camera relationship.
|
||||||
|
|
||||||
|
[](https://youtu.be/R1vb7mV3y_M "Gaze visualization demo")
|
||||||
|
<p> </p>
|
||||||
|
|
||||||
|
* **Gaze Visualization**
|
||||||
|
Show gaze direction inital from the center of faces in the input image.
|
||||||
|
|
||||||
|
[](https://youtu.be/8yMTvvr0rRU "Gaze visualization demo")
|
||||||
|
<p> </p>
|
||||||
|
|
||||||
|
* **Personal Calibration**
|
||||||
|
Perform personal calibration and remapped the gaze target on the screen.
|
||||||
|
|
||||||
|
[](https://youtu.be/ntBv1wcNGAo "Gaze visualization demo")
|
||||||
|
<p> </p>
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
[Unix Installation](https://github.molgen.mpg.de/perceptual/opengaze/wiki/Unix-Installation)
|
||||||
|
|
||||||
|
## Use
|
||||||
|
[Command line arguments](https://github.molgen.mpg.de/perceptual/opengaze/wiki/Command-line-arguments)
|
||||||
|
|
||||||
|
## Citation
|
||||||
|
If you use any of the resources provided on this page in any of your publications, please cite the following paper:
|
||||||
|
|
||||||
|
**Evaluation of Appearance-Based Methods and Implications for Gaze-Based Applications?** <br/>
|
||||||
|
Xucong Zhang, Yusuke Sugano, Andreas Bulling<br/>
|
||||||
|
Proc. ACM SIGCHI Conference on Human Factors in Computing Systems (CHI), 2019<br/>
|
||||||
|
|
||||||
|
BibTex, PDF
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
The license agreement can be found in Copyright.txt
|
||||||
|
|
||||||
|
You have to respect boost, OpenFace and OpenCV licenses.
|
||||||
|
|
||||||
|
Furthermore, you have to respect the licenses of the datasets used for [model training](:https://github.molgen.mpg.de/perceptual/opengaze/wiki/Model-training).
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# Release 0.1.0
|
||||||
|
|
||||||
|
Initial release of OpenGaze.
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
#ifndef CAFFE_DSPP_LAYER_HPP_
|
||||||
|
#define CAFFE_DSPP_LAYER_HPP_
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "caffe/blob.hpp"
|
||||||
|
#include "caffe/common.hpp"
|
||||||
|
#include "caffe/layers/data_layer.hpp"
|
||||||
|
#include "caffe/layer.hpp"
|
||||||
|
#include "caffe/layers/loss_layer.hpp"
|
||||||
|
#include "caffe/layers/neuron_layer.hpp"
|
||||||
|
#include "caffe/proto/caffe.pb.h"
|
||||||
|
|
||||||
|
namespace caffe {
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
class DSPPLayer : public Layer<Dtype> {
|
||||||
|
public:
|
||||||
|
explicit DSPPLayer(const LayerParameter& param)
|
||||||
|
: Layer<Dtype>(param) {}
|
||||||
|
virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top);
|
||||||
|
virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top);
|
||||||
|
virtual inline const char* type() const { return "DSPPLayer"; }
|
||||||
|
virtual inline int ExactNumBottomBlobs() const { return 2; };
|
||||||
|
virtual inline int MinTopBlobs() const { return 1; }
|
||||||
|
|
||||||
|
protected:
|
||||||
|
virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top);
|
||||||
|
//virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
// const vector<Blob<Dtype>*>& top);
|
||||||
|
virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
|
||||||
|
const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
|
||||||
|
//virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
|
||||||
|
// const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
|
||||||
|
|
||||||
|
int width_;
|
||||||
|
int height_;
|
||||||
|
int channel_;
|
||||||
|
int num_;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace caffe
|
||||||
|
|
||||||
|
#endif // CAFFE_DSPP_LAYER_HPP_
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
#ifndef CAFFE_POSE_DATA_LAYER_HPP_
|
||||||
|
#define CAFFE_POSE_DATA_LAYER_HPP_
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "caffe/blob.hpp"
|
||||||
|
#include "caffe/layer.hpp"
|
||||||
|
#include "caffe/proto/caffe.pb.h"
|
||||||
|
|
||||||
|
#include "caffe/layers/base_data_layer.hpp"
|
||||||
|
|
||||||
|
namespace caffe {
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
class PoseDataLayer : public BaseDataLayer<Dtype> {
|
||||||
|
public:
|
||||||
|
explicit PoseDataLayer(const LayerParameter& param)
|
||||||
|
: BaseDataLayer<Dtype>(param), has_new_data_(false) {}
|
||||||
|
virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top);
|
||||||
|
|
||||||
|
virtual inline const char* type() const { return "PoseData"; }
|
||||||
|
virtual inline int ExactNumBottomBlobs() const { return 0; }
|
||||||
|
virtual inline int ExactNumTopBlobs() const { return 2; }
|
||||||
|
|
||||||
|
virtual void AddDatumVector(const vector<Datum>& datum_vector);
|
||||||
|
virtual void AddMatVector(const vector<cv::Mat>& mat_vector,
|
||||||
|
const vector<float>& labels);
|
||||||
|
|
||||||
|
// Reset should accept const pointers, but can't, because the memory
|
||||||
|
// will be given to Blob, which is mutable
|
||||||
|
void Reset(Dtype* data, Dtype* label, int n);
|
||||||
|
void set_batch_size(int new_size);
|
||||||
|
|
||||||
|
int batch_size() { return batch_size_; }
|
||||||
|
int channels() { return channels_; }
|
||||||
|
int height() { return height_; }
|
||||||
|
int width() { return width_; }
|
||||||
|
|
||||||
|
protected:
|
||||||
|
virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top);
|
||||||
|
|
||||||
|
int batch_size_, channels_, height_, width_, size_;
|
||||||
|
Dtype* data_;
|
||||||
|
Dtype* labels_;
|
||||||
|
int n_;
|
||||||
|
size_t pos_;
|
||||||
|
Blob<Dtype> added_data_;
|
||||||
|
Blob<Dtype> added_label_;
|
||||||
|
bool has_new_data_;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace caffe
|
||||||
|
|
||||||
|
#endif
|
||||||
@ -0,0 +1,92 @@
|
|||||||
|
#include <cmath>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "caffe/layer.hpp"
|
||||||
|
#include "caffe/layers/dspp_layer.hpp"
|
||||||
|
|
||||||
|
#include <boost/spirit/include/phoenix_core.hpp>
|
||||||
|
#include <boost/spirit/include/phoenix_operator.hpp>
|
||||||
|
|
||||||
|
|
||||||
|
namespace caffe {
|
||||||
|
template <typename Dtype>
|
||||||
|
void DSPPLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top) {
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void DSPPLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
|
||||||
|
|
||||||
|
num_ = bottom[1]->shape()[0];
|
||||||
|
channel_ = bottom[1]->shape()[1]; // the input data size
|
||||||
|
height_ = bottom[1]->shape()[2];
|
||||||
|
width_ = bottom[1]->shape()[3];
|
||||||
|
|
||||||
|
// init output size
|
||||||
|
vector<int> output_shape;
|
||||||
|
output_shape.push_back(num_);
|
||||||
|
output_shape.push_back(channel_);
|
||||||
|
output_shape.push_back(height_);
|
||||||
|
output_shape.push_back(width_);
|
||||||
|
top[0]->Reshape(output_shape);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void DSPPLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top) {
|
||||||
|
Dtype* top_data = top[0]->mutable_cpu_data();
|
||||||
|
|
||||||
|
caffe_set<Dtype>(top[0]->count(), 0, top_data); // initilize to be 0
|
||||||
|
|
||||||
|
for (int n=0; n<num_; ++n) {
|
||||||
|
for (int h = 0; h < height_; ++h) { // for the input data size
|
||||||
|
for (int w = 0; w < width_; ++w) {
|
||||||
|
for (int c = 0; c < channel_; ++c) {
|
||||||
|
top_data[top[0]->offset(n, c, h, w)] = bottom[1]->data_at(n, c, h, w) * bottom[0]->data_at(n, 0, h, w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
top_data = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void DSPPLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
|
||||||
|
const vector<bool>& propagate_down,
|
||||||
|
const vector<Blob<Dtype>*>& bottom) {
|
||||||
|
if (propagate_down[0]) {
|
||||||
|
const Dtype* top_diff = top[0]->cpu_diff();
|
||||||
|
Dtype* data_diff = bottom[1]->mutable_cpu_diff();
|
||||||
|
Dtype* heat_map_diff = bottom[0]->mutable_cpu_diff();
|
||||||
|
|
||||||
|
caffe_set<Dtype>(bottom[1]->count(), 0, data_diff);
|
||||||
|
caffe_set<Dtype>(bottom[0]->count(), 0, heat_map_diff);
|
||||||
|
// Dtype activation_h, activation_w;
|
||||||
|
|
||||||
|
for (int n = 0; n < num_; ++n) {
|
||||||
|
for (int h = 0; h < height_; ++h) {
|
||||||
|
for (int w = 0; w < width_; ++w) {
|
||||||
|
for (int c = 0; c < channel_; ++c) {
|
||||||
|
|
||||||
|
Dtype buffer = top_diff[top[0]->offset(n, c, h, w)];
|
||||||
|
data_diff[bottom[1]->offset(n, c, h, w)] = buffer * (bottom[0]->data_at(n, 0, h, w));
|
||||||
|
|
||||||
|
buffer *= bottom[1]->data_at(n,c,h,w) / channel_;
|
||||||
|
|
||||||
|
heat_map_diff[bottom[0]->offset(n,0,h,w)] += buffer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
top_diff = NULL;
|
||||||
|
data_diff = NULL;
|
||||||
|
heat_map_diff = NULL;
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_CLASS(DSPPLayer);
|
||||||
|
REGISTER_LAYER_CLASS(DSPP);
|
||||||
|
|
||||||
|
} // namespace caffe
|
||||||
@ -0,0 +1,128 @@
|
|||||||
|
#include <opencv2/core/core.hpp>
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "caffe/layers/pose_data_layer.hpp"
|
||||||
|
|
||||||
|
namespace caffe {
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top) {
|
||||||
|
batch_size_ = this->layer_param_.memory_data_param().batch_size();
|
||||||
|
channels_ = this->layer_param_.memory_data_param().channels();
|
||||||
|
height_ = this->layer_param_.memory_data_param().height();
|
||||||
|
width_ = this->layer_param_.memory_data_param().width();
|
||||||
|
size_ = channels_ * height_ * width_;
|
||||||
|
CHECK_GT(batch_size_ * size_, 0) <<
|
||||||
|
"batch_size, channels, height, and width must be specified and"
|
||||||
|
" positive in memory_data_param";
|
||||||
|
int label_shape_[] = {batch_size_, 4};
|
||||||
|
vector<int> label_shape(label_shape_, label_shape_+2);
|
||||||
|
top[0]->Reshape(batch_size_, channels_, height_, width_);
|
||||||
|
top[1]->Reshape(label_shape);
|
||||||
|
added_data_.Reshape(batch_size_, channels_, height_, width_);
|
||||||
|
added_label_.Reshape(label_shape);
|
||||||
|
data_ = NULL;
|
||||||
|
labels_ = NULL;
|
||||||
|
added_data_.cpu_data();
|
||||||
|
added_label_.cpu_data();
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::AddDatumVector(const vector<Datum>& datum_vector) {
|
||||||
|
CHECK(!has_new_data_) <<
|
||||||
|
"Can't add data until current data has been consumed.";
|
||||||
|
size_t num = datum_vector.size();
|
||||||
|
CHECK_GT(num, 0) << "There is no datum to add.";
|
||||||
|
CHECK_EQ(num % batch_size_, 0) <<
|
||||||
|
"The added data must be a multiple of the batch size.";
|
||||||
|
added_data_.Reshape(num, channels_, height_, width_);
|
||||||
|
int label_shape_[] = {(int)num, 4};
|
||||||
|
vector<int> label_shape(label_shape_, label_shape_+2);
|
||||||
|
added_label_.Reshape(label_shape);
|
||||||
|
// Apply data transformations (mirror, scale, crop...)
|
||||||
|
this->data_transformer_->Transform(datum_vector, &added_data_);
|
||||||
|
// Copy Labels
|
||||||
|
Dtype* top_label = added_label_.mutable_cpu_data();
|
||||||
|
for (int item_id = 0; item_id < num; ++item_id) {
|
||||||
|
top_label[item_id] = datum_vector[item_id].label();
|
||||||
|
}
|
||||||
|
// num_images == batch_size_
|
||||||
|
Dtype* top_data = added_data_.mutable_cpu_data();
|
||||||
|
Reset(top_data, top_label, num);
|
||||||
|
has_new_data_ = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::AddMatVector(const vector<cv::Mat>& mat_vector,
|
||||||
|
const vector<float>& labels) {
|
||||||
|
size_t num = mat_vector.size();
|
||||||
|
CHECK(!has_new_data_) <<
|
||||||
|
"Can't add mat until current data has been consumed.";
|
||||||
|
CHECK_GT(num, 0) << "There is no mat to add";
|
||||||
|
CHECK_EQ(num % batch_size_, 0) <<
|
||||||
|
"The added data must be a multiple of the batch size.";
|
||||||
|
added_data_.Reshape(num, channels_, height_, width_);
|
||||||
|
int label_shape_[] = {(int)num, 4};
|
||||||
|
vector<int> label_shape(label_shape_, label_shape_+2);
|
||||||
|
added_label_.Reshape(label_shape);
|
||||||
|
// Apply data transformations (mirror, scale, crop...)
|
||||||
|
this->data_transformer_->Transform(mat_vector, &added_data_);
|
||||||
|
// Copy Labels
|
||||||
|
Dtype* top_label = added_label_.mutable_cpu_data();
|
||||||
|
for (int item_id = 0; item_id < num; ++item_id) {
|
||||||
|
top_label[item_id] = labels[item_id];
|
||||||
|
}
|
||||||
|
// num_images == batch_size_
|
||||||
|
Dtype* top_data = added_data_.mutable_cpu_data();
|
||||||
|
Reset(top_data, top_label, num);
|
||||||
|
has_new_data_ = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::Reset(Dtype* data, Dtype* labels, int n) {
|
||||||
|
CHECK(data);
|
||||||
|
CHECK(labels);
|
||||||
|
CHECK_EQ(n % batch_size_, 0) << "n must be a multiple of batch size";
|
||||||
|
// Warn with transformation parameters since a memory array is meant to
|
||||||
|
// be generic and no transformations are done with Reset().
|
||||||
|
//if (this->layer_param_.has_transform_param()) {
|
||||||
|
// LOG(WARNING) << this->type() << " does not transform array data on Reset()";
|
||||||
|
//}
|
||||||
|
data_ = data;
|
||||||
|
labels_ = labels;
|
||||||
|
n_ = n;
|
||||||
|
pos_ = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::set_batch_size(int new_size) {
|
||||||
|
CHECK(!has_new_data_) <<
|
||||||
|
"Can't change batch_size until current data has been consumed.";
|
||||||
|
batch_size_ = new_size;
|
||||||
|
added_data_.Reshape(batch_size_, channels_, height_, width_);
|
||||||
|
int label_shape_[] = {(int)batch_size_, 4};
|
||||||
|
vector<int> label_shape(label_shape_, label_shape_+2);
|
||||||
|
added_label_.Reshape(label_shape);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Dtype>
|
||||||
|
void PoseDataLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
|
||||||
|
const vector<Blob<Dtype>*>& top) {
|
||||||
|
CHECK(data_) << "PoseDataLayer needs to be initalized by calling Reset";
|
||||||
|
top[0]->Reshape(batch_size_, channels_, height_, width_);
|
||||||
|
int label_shape_[] = {(int)batch_size_, 4};
|
||||||
|
vector<int> label_shape(label_shape_, label_shape_+2);
|
||||||
|
added_label_.Reshape(label_shape);
|
||||||
|
top[0]->set_cpu_data(data_ + pos_ * size_);
|
||||||
|
top[1]->set_cpu_data(labels_ + pos_);
|
||||||
|
pos_ = (pos_ + batch_size_) % n_;
|
||||||
|
if (pos_ == 0)
|
||||||
|
has_new_data_ = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_CLASS(PoseDataLayer);
|
||||||
|
REGISTER_LAYER_CLASS(PoseData);
|
||||||
|
|
||||||
|
} // namespace caffe
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
%YAML:1.0
|
||||||
|
camera_matrix: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 3
|
||||||
|
dt: f
|
||||||
|
data: [ 1891.07, 0.0, 640, 0.0, 1891.07, 360, 0.0, 0.0, 1.0]
|
||||||
|
dist_coeffs: !!opencv-matrix
|
||||||
|
rows: 1
|
||||||
|
cols: 5
|
||||||
|
dt: f
|
||||||
|
data: [1.68091e-02, -7.14552e-02, -5.65886e-03, -5.23482e-04, -3.39946e-02]
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
%YAML:1.0
|
||||||
|
monitor_W: 516
|
||||||
|
monitor_H: 323
|
||||||
|
monitor_R: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 3
|
||||||
|
dt: f
|
||||||
|
data: [ -0.99955, -0.02891, -0.0082861, -0.028948, 0.99957, 0.0044949, 0.0081526, 0.0047327, -0.99996]
|
||||||
|
monitor_T: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 1
|
||||||
|
dt: f
|
||||||
|
data: [269.41, 48.561, 5.8344]
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
%YAML:1.0
|
||||||
|
monitor_W: 310
|
||||||
|
monitor_H: 174
|
||||||
|
monitor_R: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 3
|
||||||
|
dt: f
|
||||||
|
data: [ -0.99988, -0.009735, -0.01203, -0.0094674, 0.99971, -0.022108, 0.012242, -0.021992, -0.99968]
|
||||||
|
monitor_T: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 1
|
||||||
|
dt: f
|
||||||
|
data: [149.91, 29.575, -18.884]
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
%YAML:1.0
|
||||||
|
face_model: !!opencv-matrix
|
||||||
|
rows: 3
|
||||||
|
cols: 6
|
||||||
|
dt: f
|
||||||
|
data: [ -45.096768, -21.312858, 21.312858, 45.096768, -26.299577, 26.299577, -0.483773,0.483773, 0.483773, -0.483773, 68.595035,68.595035, 2.397030, -2.397030, -2.397030, 2.397030, -0.000000, -0.000000]
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
## input and ouput
|
||||||
|
# input_type = camera # camera, video, or directory
|
||||||
|
# input = 0 # caemra id, video file name, or directory of image files
|
||||||
|
# output = /BS/zhang-semi/work/opengaze/test/
|
||||||
|
|
||||||
|
# input = YOUR_VIDEO OR IMAGE FOLDER
|
||||||
|
# output = MUST BE A DIRECTORY
|
||||||
|
|
||||||
|
## gaze estimation method
|
||||||
|
# gaze_method = MPIIGaze # OpenFace MPIIGaze
|
||||||
|
# gpu_id = 0
|
||||||
|
|
||||||
|
## gaze estimation method/model selection
|
||||||
|
# face_model = 1 # 1 for the face model, 0 for eye image model
|
||||||
|
|
||||||
|
## CNN model for face image, trained on MPIIGaze + EYEDIAP HD
|
||||||
|
# cnn_param_path = YOUR_PATH/alexnet_face.prototxt
|
||||||
|
# cnn_model_path = YOUR_PATH/alexnet_face.caffemodel
|
||||||
|
|
||||||
|
# calibration file, calibration file
|
||||||
|
# calib_camera = YOUR_PATH/calibration.yml
|
||||||
|
# calib_screen = YOUR_PATH/monitor.yml
|
||||||
|
|
||||||
|
## parameters for personal calibration
|
||||||
|
# per_model_save_path = YOUR_PATH/user1.txt
|
||||||
|
# num_calibration = 9
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
OPENGAZE_DIR=~/OpenGaze
|
||||||
|
|
||||||
|
mkdir -p $OPENGAZE_DIR/content/caffeModel
|
||||||
|
|
||||||
|
cd $OPENGAZE_DIR/content/caffeModel
|
||||||
|
|
||||||
|
wget https://datasets.d2.mpi-inf.mpg.de/MPIIGaze/alexnet_face.prototxt
|
||||||
|
|
||||||
|
wget https://datasets.d2.mpi-inf.mpg.de/MPIIGaze/alexnet_face.caffemodel
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.0)
|
||||||
|
project(OpenGazeExe VERSION 1.0)
|
||||||
|
|
||||||
|
set(OPENGAZE_DIR "$ENV{HOME}/OpenGaze")
|
||||||
|
|
||||||
|
add_definitions(-DOPENGAZE_DIR="${CMAKE_SOURCE_DIR}")
|
||||||
|
|
||||||
|
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/)
|
||||||
|
|
||||||
|
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
||||||
|
|
||||||
|
set(CMAKE_CXX_STANDARD 11)
|
||||||
|
|
||||||
|
find_package( OpenCV 3.1 REQUIRED COMPONENTS calib3d highgui objdetect imgproc core)
|
||||||
|
|
||||||
|
# Boost, for reading configuration file
|
||||||
|
find_package(Boost 1.5 COMPONENTS system filesystem timer thread program_options REQUIRED)
|
||||||
|
set(Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIR} ${Boost_INCLUDE_DIR}/boost)
|
||||||
|
|
||||||
|
include_directories(/usr/local/include/opengaze /usr/local/cuda/include ${OpenCV_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS})
|
||||||
|
link_directories(/usr/lib /usr/local/lib /usr/local/cuda/lib64 ${Boost_LIBRARY_DIRS} ${OPENGAZE_DIR}/3rdParty)
|
||||||
|
|
||||||
|
## -lX11 is for getting screen resolution in pixel in the Linux system
|
||||||
|
set(LIBS opengaze LandmarkDetector ${OpenCV_LIBS} ${Boost_LIBRARIES} tbb openblas dlib lmdb glog caffe X11)
|
||||||
|
|
||||||
|
add_executable(GazeVisualization GazeVisualization.cpp)
|
||||||
|
target_link_libraries(GazeVisualization ${LIBS})
|
||||||
|
|
||||||
|
add_executable(Calibration Calibration.cpp)
|
||||||
|
target_link_libraries(Calibration ${LIBS})
|
||||||
|
|
||||||
|
add_executable(GazeEstimation GazeEstimation.cpp)
|
||||||
|
target_link_libraries(GazeEstimation ${LIBS})
|
||||||
|
|
||||||
|
add_executable(DataExtraction DataExtraction.cpp)
|
||||||
|
target_link_libraries(DataExtraction ${LIBS})
|
||||||
@ -0,0 +1,29 @@
|
|||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#include "opengaze/opengaze.hpp"
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
using namespace cv;
|
||||||
|
using namespace opengaze;
|
||||||
|
|
||||||
|
vector<string> get_arguments(int argc, char **argv) {
|
||||||
|
vector<string> arguments;
|
||||||
|
for (int i = 0; i < argc; ++i){
|
||||||
|
arguments.emplace_back(string(argv[i]));
|
||||||
|
}
|
||||||
|
return arguments;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
vector<string> arguments = get_arguments(argc, argv);
|
||||||
|
OpenGaze open_gaze(argc, argv);
|
||||||
|
int num_calibration_point = 20;
|
||||||
|
open_gaze.runPersonalCalibration(num_calibration_point);
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#include "opengaze/opengaze.hpp"
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
using namespace cv;
|
||||||
|
using namespace opengaze;
|
||||||
|
|
||||||
|
vector<string> get_arguments(int argc, char **argv) {
|
||||||
|
vector<string> arguments;
|
||||||
|
for (int i = 0; i < argc; ++i){
|
||||||
|
arguments.emplace_back(string(argv[i]));
|
||||||
|
}
|
||||||
|
return arguments;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
vector<string> arguments = get_arguments(argc, argv);
|
||||||
|
OpenGaze open_gaze(argc, argv);
|
||||||
|
open_gaze.runDataExtraction();
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#include "opengaze/opengaze.hpp"
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
using namespace cv;
|
||||||
|
using namespace opengaze;
|
||||||
|
|
||||||
|
vector<string> get_arguments(int argc, char **argv) {
|
||||||
|
vector<string> arguments;
|
||||||
|
for (int i = 0; i < argc; ++i){
|
||||||
|
arguments.emplace_back(string(argv[i]));
|
||||||
|
}
|
||||||
|
return arguments;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
vector<string> arguments = get_arguments(argc, argv);
|
||||||
|
OpenGaze open_gaze(argc, argv);
|
||||||
|
open_gaze.runGazeOnScreen();
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#include "opengaze/opengaze.hpp"
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
using namespace cv;
|
||||||
|
using namespace opengaze;
|
||||||
|
|
||||||
|
vector<string> get_arguments(int argc, char **argv) {
|
||||||
|
vector<string> arguments;
|
||||||
|
for (int i = 0; i < argc; ++i){
|
||||||
|
arguments.emplace_back(string(argv[i]));
|
||||||
|
}
|
||||||
|
return arguments;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv)
|
||||||
|
{
|
||||||
|
vector<string> arguments = get_arguments(argc, argv);
|
||||||
|
OpenGaze open_gaze(argc, argv);
|
||||||
|
open_gaze.runGazeVisualization();
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 177 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
@ -0,0 +1,88 @@
|
|||||||
|
#ifndef DATA_HPP
|
||||||
|
#define DATA_HPP
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
namespace opengaze{
|
||||||
|
|
||||||
|
/**
|
||||||
|
* face and facial landmark detection data
|
||||||
|
* @param face_id personal id from tracking across frames
|
||||||
|
* @param certainty detection score, 1 is the best, -1 is the worst
|
||||||
|
* @param landmarks detected six facial landmarks as four eye corners and two mouth corners
|
||||||
|
* @param face_bb detected face bounding box
|
||||||
|
*/
|
||||||
|
struct FaceData
|
||||||
|
{
|
||||||
|
unsigned long face_id;
|
||||||
|
double certainty;
|
||||||
|
cv::Point2f landmarks[6];
|
||||||
|
cv::Rect_<int> face_bb;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* eye image related data
|
||||||
|
* @param leye_pos/reye_pose 3D eyeball center position for left and right eyes in the original camera coordinate system
|
||||||
|
* @param leye_img/reye_img eye image
|
||||||
|
* @param leye_rot/reye_rot rotation matrix during the data normalization procedure
|
||||||
|
*/
|
||||||
|
struct EyeData
|
||||||
|
{
|
||||||
|
// cv::Mat head_r, head_t;
|
||||||
|
cv::Mat leye_pos, reye_pos; //
|
||||||
|
|
||||||
|
// normalized eyes
|
||||||
|
cv::Mat leye_img, reye_img;
|
||||||
|
cv::Mat leye_rot, reye_rot;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* face patch data related to data normalization
|
||||||
|
* @param head_r head pose as center of the face
|
||||||
|
* @param head_t head translation as center of the face
|
||||||
|
* @param face_rot rotation matrix during the data normalization procedure
|
||||||
|
* @param face_center 3D face center in the original camera coordinate system
|
||||||
|
* @param debug_img use for debug to show the normalized face image
|
||||||
|
* @param face_patch normalized face image
|
||||||
|
*/
|
||||||
|
struct FacePatchData
|
||||||
|
{
|
||||||
|
cv::Mat head_r, head_t;
|
||||||
|
cv::Mat face_rot;
|
||||||
|
cv::Mat face_center;
|
||||||
|
cv::Mat debug_img;
|
||||||
|
cv::Mat face_patch;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* gaze data
|
||||||
|
* @param lgaze3d/lgaze3d gaze directions of left and right eyes in the camera coordinate system
|
||||||
|
* @param gaze3d gaze direction estimated from face patch in the in the camera coordinate system
|
||||||
|
* @param lgaze2d/rgaze2d projected gaze positions on the screen coordinate from left and right eyes
|
||||||
|
* @param gaze2d projected gaze positions from face patch on the screen coordinate
|
||||||
|
*/
|
||||||
|
struct GazeData
|
||||||
|
{
|
||||||
|
cv::Vec3f lgaze3d, rgaze3d;
|
||||||
|
cv::Vec3f gaze3d;
|
||||||
|
cv::Point2f lgaze2d, rgaze2d;
|
||||||
|
cv::Point2f gaze2d;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* The general output data structure
|
||||||
|
* @param face_data store face and facial landmark detection data
|
||||||
|
* @param eye_data store data related to eye image input
|
||||||
|
* @param face_patch_data normalized face path data
|
||||||
|
* @param gaze_data gaze data in 2D and 3D spaces
|
||||||
|
*/
|
||||||
|
struct Sample
|
||||||
|
{
|
||||||
|
FaceData face_data;
|
||||||
|
EyeData eye_data;
|
||||||
|
FacePatchData face_patch_data;
|
||||||
|
GazeData gaze_data;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#endif //DATA_HPP
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
#ifndef FACE_DETECTOR_HPP
|
||||||
|
#define FACE_DETECTOR_HPP
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#if USE_DLIB
|
||||||
|
// if we use dlib
|
||||||
|
#include <dlib/opencv.h>
|
||||||
|
#include <dlib/image_processing/frontal_face_detector.h>
|
||||||
|
#include <dlib/image_processing/render_face_detections.h>
|
||||||
|
#include <dlib/image_processing.h>
|
||||||
|
#include <dlib/gui_widgets.h>
|
||||||
|
#include <dlib/image_io.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
#include "data.hpp"
|
||||||
|
|
||||||
|
namespace opengaze{
|
||||||
|
|
||||||
|
class FaceDetector {
|
||||||
|
public:
|
||||||
|
FaceDetector();
|
||||||
|
~FaceDetector();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* face and facial landmark detection selection
|
||||||
|
* The current implementation is only OpenFace. OpenFace use dlib for face detection
|
||||||
|
*/
|
||||||
|
enum Method{OpenFace, OpenCV, Dlib};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* main function to detect and track face and facial landmarks
|
||||||
|
* @param input_img input image
|
||||||
|
* @param output output data structure
|
||||||
|
*/
|
||||||
|
void track_faces(cv::Mat input_img, std::vector<opengaze::Sample> &output);
|
||||||
|
|
||||||
|
void reset();
|
||||||
|
void setMethodType(Method method_type) {method_type_ = method_type;}
|
||||||
|
Method getMethodType() {return method_type_;}
|
||||||
|
void initialize(int number_users);
|
||||||
|
|
||||||
|
private:
|
||||||
|
Method method_type_;
|
||||||
|
|
||||||
|
#if USE_DLIB
|
||||||
|
dlib::frontal_face_detector dlib_detector_;
|
||||||
|
dlib::shape_predictor dlib_sp_;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// parameters for OpenFace
|
||||||
|
std::vector<bool> active_models_;
|
||||||
|
unsigned long num_faces_max_;
|
||||||
|
int detection_skip_frames_, tracking_loss_limit_;
|
||||||
|
float detection_resize_rate_;
|
||||||
|
float nonoverlap_threshold_;
|
||||||
|
double certainty_threshold_;
|
||||||
|
int landmark_indices_[6];
|
||||||
|
int frame_counter_;
|
||||||
|
unsigned long current_face_id_;
|
||||||
|
std::vector<unsigned long> face_ids_;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#endif //FACE_DETECTOR_HPP
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
#ifndef GAZE_ESTIMATOR_HPP
|
||||||
|
#define GAZE_ESTIMATOR_HPP
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
|
||||||
|
#include "data.hpp"
|
||||||
|
#include "face_detector.hpp"
|
||||||
|
#include "normalizer.hpp"
|
||||||
|
#include "gaze_predictor.hpp"
|
||||||
|
|
||||||
|
namespace opengaze{
|
||||||
|
|
||||||
|
class GazeEstimator {
|
||||||
|
public:
|
||||||
|
GazeEstimator();
|
||||||
|
~GazeEstimator();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* On the current implementation, we only has the "MPIIGaze" method which uses the input face/eye image
|
||||||
|
* and output gaze direction directly. It is an appearance-based method. The "OpenFace" can also output
|
||||||
|
* the gaze vector according to the pupil detection results. However, "OpenFace" implementation is not
|
||||||
|
* included inside our OpenGaze toolkit yet.
|
||||||
|
*/
|
||||||
|
enum Method{MPIIGaze, OpenFace};
|
||||||
|
/**
|
||||||
|
* for the "MPIIGaze" method, the input image can be face or eye. The full-face patch model can output
|
||||||
|
* more accurate gaze prediction than the eye image model, while the eye image base model is much faster.
|
||||||
|
*/
|
||||||
|
enum InputType{face, eye};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the main function to estimate the gaze.
|
||||||
|
* It performs the face and facial landmarks detection, head pose estimation and then gaze prediction.
|
||||||
|
* @param input_image input scene image
|
||||||
|
* @param output data structure for output
|
||||||
|
*/
|
||||||
|
void estimateGaze(cv::Mat input_image, std::vector<opengaze::Sample> &output);
|
||||||
|
void getImagePatch(cv::Mat input_image, std::vector<opengaze::Sample> &outputs);
|
||||||
|
void setCameraParameters(cv::Mat camera_matrix, cv::Mat camera_dist);
|
||||||
|
void setRootPath(std::string root_path);
|
||||||
|
void setMethod(Method, std::vector<std::string> arguments);
|
||||||
|
void initialFaceDetector(int number_users);
|
||||||
|
|
||||||
|
Method method_type_;
|
||||||
|
InputType input_type_; // the input type
|
||||||
|
|
||||||
|
private:
|
||||||
|
// class instances
|
||||||
|
FaceDetector face_detector_;
|
||||||
|
Normalizer normalizer_;
|
||||||
|
GazePredictor gaze_predictor_;
|
||||||
|
// camera intrinsic matrix
|
||||||
|
cv::Mat camera_matrix_;
|
||||||
|
// camera distortion matrix
|
||||||
|
cv::Mat camera_dist_;
|
||||||
|
// the root pat is used for load configuration file and models
|
||||||
|
std::string root_path_;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#endif //GAZE_ESTIMATOR_HPP
|
||||||
@ -0,0 +1,29 @@
|
|||||||
|
#ifndef GAZE_PREDICTOR_HPP
|
||||||
|
#define GAZE_PREDICTOR_HPP
|
||||||
|
|
||||||
|
#include <opencv2/opencv.hpp>
|
||||||
|
#include "data.hpp"
|
||||||
|
#include "face_detector.hpp"
|
||||||
|
|
||||||
|
|
||||||
|
namespace opengaze{
|
||||||
|
|
||||||
|
class GazePredictor {
|
||||||
|
|
||||||
|
public:
|
||||||
|
GazePredictor();
|
||||||
|
~GazePredictor();
|
||||||
|
|
||||||
|
void initiaMPIIGaze(std::vector<std::string> arguments);
|
||||||
|
cv::Point3f predictGazeMPIIGaze(cv::Mat face_patch);
|
||||||
|
|
||||||
|
private:
|
||||||
|
int model_type_;
|
||||||
|
bool is_extract_feature;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||