#include #include // if we use OpenFace #if USE_OPENFACE #include // from "OpenFace-master/lib/local/LandmarkDetector/include/" #endif #include "face_detector.hpp" using namespace std; using namespace cv; vector det_parameters_; vector clnf_models_; namespace opengaze { FaceDetector::FaceDetector() { method_type_ = Method::OpenFace; } FaceDetector::~FaceDetector() {} void FaceDetector::initialize(int number_users=5) { string root_path = OPENFACE_DIR; root_path = root_path + "/build/bin"; //string openface_root = OpenFace_ROOT_DIR; // (currently) hard-coded setting num_faces_max_ = number_users; detection_resize_rate_ = 2.0; // resize the input image to detect face, crucial for speed detection_skip_frames_ = 1; nonoverlap_threshold_ = 0.5; certainty_threshold_ = 0.0; // the smaller the better, 1 is the best, -1 is the worst landmark_indices_[0] = 36; landmark_indices_[1] = 39; landmark_indices_[2] = 42; landmark_indices_[3] = 45; landmark_indices_[4] = 48; landmark_indices_[5] = 54; tracking_loss_limit_ = 10; // initialize the tracking models LandmarkDetector::FaceModelParameters det_parameter; det_parameter.reinit_video_every = -1; // This is so that the model would not try re-initialising itself det_parameter.curr_face_detector = LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR; det_parameter.model_location = root_path + "/model/main_clm_wild.txt"; det_parameter.haar_face_detector_location = root_path + "/classifiers/haarcascade_frontalface_alt.xml";// this line will be disable due to "curr_face_detector" det_parameter.mtcnn_face_detector_location = root_path + "/model/mtcnn_detector/MTCNN_detector.txt"; det_parameter.use_face_template = true; det_parameter.reinit_video_every = 5; // det_parameter.quiet_mode = true; not avaliable fro OpenFace v2.1 // // For in the wild fitting these parameters are suitable det_parameter.window_sizes_init = vector(4); det_parameter.window_sizes_init[0] = 15; det_parameter.window_sizes_init[1] = 13; det_parameter.window_sizes_init[2] = 11; det_parameter.window_sizes_init[3] = 9; det_parameter.sigma = 1.25; det_parameter.reg_factor = 35; det_parameter.weight_factor = 2.5; det_parameter.num_optimisation_iteration = 10; det_parameter.curr_face_detector = LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR; det_parameters_.push_back(det_parameter); LandmarkDetector::CLNF clnf_model_ = LandmarkDetector::CLNF(det_parameter.model_location); if (!clnf_model_.loaded_successfully){ cout << "ERROR: Could not load the landmark detector" << endl; exit(-1); } clnf_model_.face_detector_HAAR.load(det_parameter.haar_face_detector_location); clnf_model_.haar_face_detector_location = det_parameter.haar_face_detector_location; clnf_model_.face_detector_MTCNN.Read(det_parameter.mtcnn_face_detector_location); clnf_model_.mtcnn_face_detector_location = det_parameter.mtcnn_face_detector_location; // If can't find MTCNN face detector, default to HOG one if (det_parameter.curr_face_detector == LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR && clnf_model_.face_detector_MTCNN.empty()){ cout << "INFO: defaulting to HOG-SVM face detector" << endl; det_parameter.curr_face_detector = LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR; } clnf_models_.reserve(num_faces_max_); clnf_models_.push_back(clnf_model_); active_models_.push_back(false); for(int i=1; i& clnf_models, vector >& face_detections){ // Go over the model and eliminate detections that are not informative (there already is a tracker there) for (size_t model = 0; model < clnf_models.size(); ++model){ // See if the detections intersect cv::Rect_ model_rect = clnf_models[model].GetBoundingBox(); for (int detection=face_detections.size()-1; detection >= 0; --detection) { double intersection_area = (model_rect & face_detections[detection]).area(); double union_area = model_rect.area() + face_detections[detection].area() - 2 * intersection_area; // If the model is already tracking what we're detecting ignore the detection, this is determined by amount of overlap if (intersection_area / union_area > 0.5) { face_detections.erase(face_detections.begin() + detection); } } } } double NonOverlapingDetection(const LandmarkDetector::CLNF &ref_model, const LandmarkDetector::CLNF &tgt_model) { Rect_ ref_rect = ref_model.GetBoundingBox(); Rect_ tgt_rect = tgt_model.GetBoundingBox(); double intersection_area = (ref_rect & tgt_rect).area(); double union_area = ref_rect.area() + tgt_rect.area() - 2 * intersection_area; return intersection_area/union_area; } void FaceDetector::track_faces(cv::Mat input_img, std::vector &output) { if(input_img.channels() < 3){ cout << "The input must be a color image!" < grayscale_image; cvtColor(input_img, grayscale_image, CV_BGR2GRAY); bool all_models_active = true; for(unsigned int model = 0; model < clnf_models_.size(); ++model) { if(!active_models_[model]) { all_models_active = false; break; } } // Detect faces // Get the detections (every Xth frame and when there are free models available for tracking) std::vector > face_detections; cv::Mat small_grayscale_image_; if (frame_counter_ % detection_skip_frames_ == 0 && !all_models_active) { // resized image for faster face detection if (detection_resize_rate_ != 1) resize(grayscale_image, small_grayscale_image_, Size(), 1.0/detection_resize_rate_, 1.0/detection_resize_rate_); else small_grayscale_image_ = grayscale_image; if (det_parameters_[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR){ vector confidences; LandmarkDetector::DetectFacesHOG(face_detections, small_grayscale_image_, clnf_models_[0].face_detector_HOG, confidences); } else if (det_parameters_[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HAAR_DETECTOR){ LandmarkDetector::DetectFaces(face_detections, small_grayscale_image_, clnf_models_[0].face_detector_HAAR); } else{ vector confidences; LandmarkDetector::DetectFacesMTCNN(face_detections, small_grayscale_image_, clnf_models_[0].face_detector_MTCNN, confidences); } // resize the face deteciton back if (detection_resize_rate_ != 1) { for(auto& face_detection : face_detections) { face_detection.x *= detection_resize_rate_; face_detection.y *= detection_resize_rate_; face_detection.width *= detection_resize_rate_; face_detection.height *= detection_resize_rate_; } } // Keep only non overlapping detections (also convert to a concurrent vector NonOverlapingDetections(clnf_models_, face_detections); } vector< tbb::atomic > face_detections_used(face_detections.size()); // Go through every model and update the tracking tbb::parallel_for(0, (int)clnf_models_.size(), [&](int model) { //for (unsigned int model = 0; model < clnf_models_.size(); ++model) { bool detection_success = false; // If the current model has failed more than threshold, remove it if (clnf_models_[model].failures_in_a_row > tracking_loss_limit_) { active_models_[model] = false; clnf_models_[model].Reset(); } // If the model is inactive reactivate it with new detections if (!active_models_[model]){ for (size_t detection_ind = 0; detection_ind < face_detections.size(); ++detection_ind) { // if it was not taken by another tracker take it (if it is false swap it to true and enter detection, this makes it parallel safe) if (!face_detections_used[detection_ind].compare_and_swap(true, false)) { // Reinitialise the model clnf_models_[model].Reset(); // This ensures that a wider window is used for the initial landmark localisation clnf_models_[model].detection_success = false; LandmarkDetector::DetectLandmarksInVideo(input_img, face_detections[detection_ind], clnf_models_[model], det_parameters_[model], grayscale_image); // This activates the model active_models_[model] = true; face_ids_[model] = current_face_id_; current_face_id_++; // break out of the loop as the tracker has been reinitialised break; } } } else { // The actual facial landmark detection / tracking detection_success = LandmarkDetector::DetectLandmarksInVideo(input_img, clnf_models_[model], det_parameters_[model], grayscale_image); } //} }); // Go through every model and check the results for(size_t model=0; model nonoverlap_threshold_) overlapping = true; } if(overlapping){ active_models_[model] = false; face_ids_[model] = 0; clnf_models_[model].Reset(); continue; } if(clnf_models_[model].detection_certainty < certainty_threshold_) continue; Sample temp; temp.face_data.certainty = clnf_models_[model].detection_certainty; temp.face_data.face_id = face_ids_[model]; temp.face_data.face_bb.x = (int)clnf_models_[model].GetBoundingBox().x; temp.face_data.face_bb.y = (int)clnf_models_[model].GetBoundingBox().y; temp.face_data.face_bb.height = (int)clnf_models_[model].GetBoundingBox().height; temp.face_data.face_bb.width = (int)clnf_models_[model].GetBoundingBox().width; for(int p=0; p<6; p++){ int num_p = landmark_indices_[p]; temp.face_data.landmarks[p] = Point2d( clnf_models_[model].detected_landmarks.at(num_p,0), clnf_models_[model].detected_landmarks.at(num_p+68,0) ); } output.emplace_back(temp); } } }