From 0f2c34c1141393a1cfed4a3adcdb382040eede88 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 15:08:29 +0200 Subject: [PATCH 01/27] class OpenfaceVideo and threaded data extraction --- CMakeLists.txt | 4 +- README.md | 13 +- download_models.sh | 0 exe/FeatureExtractionPython/CMakeLists.txt | 13 + .../FeatureExtractionPython.cpp | 365 ++++++++++++++++++ .../Utilities/include/RecorderOpenFace.h | 22 ++ 6 files changed, 414 insertions(+), 3 deletions(-) mode change 100644 => 100755 download_models.sh create mode 100644 exe/FeatureExtractionPython/CMakeLists.txt create mode 100644 exe/FeatureExtractionPython/FeatureExtractionPython.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d5e96952..04afad2f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ else() MESSAGE(FATAL_ERROR "OpenCV not found in the system.") endif() -find_package( Boost 1.5.9 COMPONENTS filesystem system) +find_package( Boost 1.5.9 COMPONENTS filesystem system thread python REQUIRED) if(${Boost_FOUND}) MESSAGE("Boost information:") MESSAGE(" Boost_VERSION: ${Boost_VERSION}") @@ -50,6 +50,7 @@ else() MESSAGE("Boost not found in the system.") endif() +find_package(PythonLibs REQUIRED) # Move LandmarkDetector model file(GLOB files "lib/local/LandmarkDetector/model/*.txt") @@ -245,3 +246,4 @@ add_subdirectory(exe/FaceLandmarkImg) add_subdirectory(exe/FaceLandmarkVid) add_subdirectory(exe/FaceLandmarkVidMulti) add_subdirectory(exe/FeatureExtraction) +add_subdirectory(exe/FeatureExtractionPython) diff --git a/README.md b/README.md index 61b40e226..0fa610184 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,16 @@ # OpenFace 2.2.0: a facial behavior analysis toolkit -[![Build Status](https://travis-ci.org/TadasBaltrusaitis/OpenFace.svg?branch=master)](https://travis-ci.org/TadasBaltrusaitis/OpenFace) -[![Build status](https://ci.appveyor.com/api/projects/status/8msiklxfbhlnsmxp/branch/master?svg=true)](https://ci.appveyor.com/project/TadasBaltrusaitis/openface/branch/master) +*Modified version of [original repository](https://github.com/TadasBaltrusaitis/OpenFace) to allow usage of webcam feature extraction from python. This is NOT a stable version.* + +See 'exe/FeatureExtractionPython' for code and launch 'build/bin/OpenfaceVideo' + +requires python dev libs: + +```bash +$ sudo apt-get install python-dev python3-dev +``` + +## Description Over the past few years, there has been an increased interest in automatic facial behavior analysis and understanding. We present OpenFace – a tool intended for computer vision and machine learning diff --git a/download_models.sh b/download_models.sh old mode 100644 new mode 100755 diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt new file mode 100644 index 000000000..efe19a33e --- /dev/null +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -0,0 +1,13 @@ +# Local libraries +include_directories(${PYTHON_INCLUDE_DIRS}) +link_directories(${PYTHON_LIBRARIES}) +include_directories(${LandmarkDetector_SOURCE_DIR}/include) + +add_executable(OpenfaceVideo FeatureExtractionPython.cpp) +target_link_libraries(OpenfaceVideo LandmarkDetector) +target_link_libraries(OpenfaceVideo FaceAnalyser) +target_link_libraries(OpenfaceVideo GazeAnalyser) +target_link_libraries(OpenfaceVideo Utilities) +target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) + +install (TARGETS OpenfaceVideo DESTINATION bin) \ No newline at end of file diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp new file mode 100644 index 000000000..6030e0521 --- /dev/null +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -0,0 +1,365 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt + +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace 2.0: Facial Behavior Analysis Toolkit +// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency +// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018 +// +// Convolutional experts constrained local model for facial landmark detection. +// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency, +// in Computer Vision and Pattern Recognition Workshops, 2017. +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +/////////////////////////////////////////////////////////////////////////////// + + +// FeatureExtraction.cpp : Defines the entry point for the feature extraction console application. + +#include +#include +#include +#include +#include +#include +#include + +// Local includes +#include "LandmarkCoreIncludes.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef CONFIG_DIR +#define CONFIG_DIR "~" +#endif + +#define INFO_STREAM( stream ) \ +std::cout << stream << std::endl + +#define WARN_STREAM( stream ) \ +std::cout << "Warning: " << stream << std::endl + +#define ERROR_STREAM( stream ) \ +std::cout << "Error: " << stream << std::endl + +static void printErrorAndAbort(const std::string & error) +{ + std::cout << error << std::endl; +} + +#define FATAL_STREAM( stream ) \ +printErrorAndAbort( std::string( "Fatal error: " ) + stream ) + +int OpenfaceVideoWorker( + std::vector& args, + const bool& running, + boost::function callback, + boost::thread_group & thread_group, + boost::thread * this_thread ) { + + // Load the modules that are being used for tracking and face analysis + // Load face landmark detector + LandmarkDetector::FaceModelParameters det_parameters(args); + // Always track gaze in feature extraction + LandmarkDetector::CLNF face_model(det_parameters.model_location); + + if (!face_model.loaded_successfully) + { + std::cout << "ERROR: Could not load the landmark detector" << std::endl; + return 1; + } + + // Load facial feature extractor and AU analyser + FaceAnalysis::FaceAnalyserParameters face_analysis_params(args); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); + + if (!face_model.eye_model) + { + std::cout << "WARNING: no eye model found" << std::endl; + } + + if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0) + { + std::cout << "WARNING: no Action Unit models found" << std::endl; + } + + Utilities::SequenceCapture sequence_reader; + + // A utility for visualizing the results + Utilities::Visualizer visualizer(args); + + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + + while ( running ) // this is not a for loop as we might also be reading from a webcam + { + + // The sequence reader chooses what to open based on command line arguments provided + if (!sequence_reader.Open(args)) + break; + + INFO_STREAM("Device or file opened"); + + if (sequence_reader.IsWebcam()) + { + INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); + INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); + visualizer.vis_track = true; + } + + cv::Mat captured_image; + + Utilities::RecorderOpenFaceParameters recording_params(args, true, sequence_reader.IsWebcam(), + sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); + if (!face_model.eye_model) + { + recording_params.setOutputGaze(false); + } + + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, args); + + if (recording_params.outputGaze() && !face_model.eye_model) + std::cout << "WARNING: no eye model defined, but outputting gaze" << std::endl; + + captured_image = sequence_reader.GetNextFrame(); + + // For reporting progress + double reported_completion = 0; + + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) + { + // Converting to grayscale + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); + + + // The actual facial landmark detection / tracking + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters, grayscale_image); + + // Gaze tracking, absolute gaze direction + cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); + + if (detection_success && face_model.eye_model) + { + GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); + gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); + } + + // Do face alignment + cv::Mat sim_warped_img; + cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; + + // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus) + { + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); + face_analyser.GetLatestAlignedFace(sim_warped_img); + face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); + } + + // Work out the pose of the head from the tracked model + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + + // Keeping track of FPS + fps_tracker.AddFrame(); + + // Displaying the tracking visualizations + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetObservationFaceAlign(sim_warped_img); + visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); + visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); + visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + visualizer.SetFps(fps_tracker.GetFPS()); + + // detect key presses + char character_press = visualizer.ShowObservation(); + + // quit processing the current sequence (useful when in Webcam mode) + if (character_press == 'q') + { + break; + } + + // Setting up the recorder output + open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); + open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), + face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); + open_face_rec.SetObservationPose(pose_estimate); + open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); + open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); + open_face_rec.SetObservationFaceID(0); + open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber()); + open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.WriteObservation(); + open_face_rec.WriteObservationTracked(); + + // CALLING OPENVIDEO CALLBACK + callback( open_face_rec ); + + // Reporting progress + if (sequence_reader.GetProgress() >= reported_completion / 10.0) + { + std::cout << reported_completion * 10 << "% "; + if (reported_completion == 10) + { + std::cout << std::endl; + } + reported_completion = reported_completion + 1; + } + + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); + + } + + INFO_STREAM("Closing output recorder"); + open_face_rec.Close(); + INFO_STREAM("Closing input reader"); + sequence_reader.Close(); + INFO_STREAM("Closed successfully"); + + if (recording_params.outputAUs()) + { + INFO_STREAM("Postprocessing the Action Unit predictions"); + face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); + } + + // Reset the models for the next video + face_analyser.Reset(); + face_model.Reset(); + + thread_group.remove_thread(this_thread); + delete this_thread; + + } + +} + +class OpenfaceVideo { + +public: + + OpenfaceVideo() : worker(0) {} + + ~OpenfaceVideo() { + // clean stop + stop(); + } + + void load_arguments(int argc, char **argv) { + for (int i = 0; i < argc; ++i) { + arguments.push_back(std::string(argv[i])); + } + append_argument( "-2Dfp", "" ); + append_argument( "-3Dfp", "" ); + append_argument( "-pose", "" ); + append_argument( "-gaze", "" ); + append_argument( "-device", "0" ); + } + + void start() { + + stop(); + thread_running = true; + worker = new boost::thread(); + boost::function callback = boost::bind( &OpenfaceVideo::new_frame, this, _1 ); + *worker = boost::thread( + boost::bind( + &OpenfaceVideoWorker, + arguments, + thread_running, + callback, + boost::ref(openfacevideo_threads), + worker + ) + ); + openfacevideo_threads.add_thread(worker); + } + + void new_frame( Utilities::RecorderOpenFace& rec ) { + std::cout << "new_frame " << rec.GetCSVFile() << std::endl; + } + + void stop() { + if ( thread_running ) { + thread_running = false; + usleep( 1000 ); + worker = 0; + } + } + + inline bool is_running() const { + return worker != 0; + } + +private: + + void append_argument( std::string a, std::string value ) { + std::vector::iterator it = arguments.begin(); + std::vector::iterator ite = arguments.end(); + bool found = false; + for ( ; it != ite; ++it ) { + if ( (*it).compare( a ) == 0 ) { + std::cout << "arg " << a << " is already there" << std::endl; + found = true; break; + } + } + if ( !found ) { + arguments.push_back( a ); + if (value.length() > 0) { + arguments.push_back( value ); + } + } + } + + std::vector arguments; + bool thread_running; + boost::thread_group openfacevideo_threads; + boost::thread* worker; + +}; + +int main(int argc, char **argv) { + + OpenfaceVideo* ov = new OpenfaceVideo(); + ov->load_arguments( argc, argv ); + ov->start(); + while( ov->is_running() ) { + usleep( 100 ); + } + delete ov; + return 0; + +} diff --git a/lib/local/Utilities/include/RecorderOpenFace.h b/lib/local/Utilities/include/RecorderOpenFace.h index 07805dc63..50a2f1bff 100644 --- a/lib/local/Utilities/include/RecorderOpenFace.h +++ b/lib/local/Utilities/include/RecorderOpenFace.h @@ -110,6 +110,28 @@ namespace Utilities void WriteObservationTracked(); std::string GetCSVFile() { return csv_filename; } + + // DIRTY polymorph.cool HACKING + + inline const cv::Mat_& get_landmarks_2D() const { + return landmarks_2D; + } + + inline const cv::Mat_& get_landmarks_3D() const { + return landmarks_3D; + } + + inline const cv::Vec6f& get_head_pose() const { + return head_pose; + } + + inline const cv::Point3f& get_gaze_direction(int i) const { + if ( i == 0 ) { + return gaze_direction0; + } else { + return gaze_direction1; + } + } private: From bf3a27d10b3617cab8a81aead8f49b31f9c18730 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 15:21:39 +0200 Subject: [PATCH 02/27] retrieval of landmarks 3d out of the recorder --- exe/FeatureExtractionPython/FeatureExtractionPython.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 6030e0521..641b1d7c0 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -309,7 +309,11 @@ class OpenfaceVideo { } void new_frame( Utilities::RecorderOpenFace& rec ) { - std::cout << "new_frame " << rec.GetCSVFile() << std::endl; + + const cv::Mat_& l3d = rec.get_landmarks_3D(); + cv::Size s = l3d.size(); + std::cout << "new_frame " << rec.GetCSVFile() << ", rows: " << s.height << ", cols: " << s.width << std::endl; + } void stop() { From 0a7f2d84d90538db3073d9f493d38f633e9e6a07 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 16:45:01 +0200 Subject: [PATCH 03/27] compilation of library + python bindings for OpenfaceVideo --- exe/FeatureExtractionPython/CMakeLists.txt | 13 +- .../FeatureExtractionPython.cpp | 126 +++++++++++++++++- 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index efe19a33e..01cb1af4a 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -2,12 +2,21 @@ include_directories(${PYTHON_INCLUDE_DIRS}) link_directories(${PYTHON_LIBRARIES}) include_directories(${LandmarkDetector_SOURCE_DIR}/include) - + +# EXEC add_executable(OpenfaceVideo FeatureExtractionPython.cpp) target_link_libraries(OpenfaceVideo LandmarkDetector) target_link_libraries(OpenfaceVideo FaceAnalyser) target_link_libraries(OpenfaceVideo GazeAnalyser) target_link_libraries(OpenfaceVideo Utilities) target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) +install (TARGETS OpenfaceVideo DESTINATION bin) -install (TARGETS OpenfaceVideo DESTINATION bin) \ No newline at end of file +# LIBRARY +add_library(PyOpenfaceVideo SHARED FeatureExtractionPython.cpp) +target_link_libraries(PyOpenfaceVideo LandmarkDetector) +target_link_libraries(PyOpenfaceVideo FaceAnalyser) +target_link_libraries(PyOpenfaceVideo GazeAnalyser) +target_link_libraries(PyOpenfaceVideo Utilities) +target_link_libraries(PyOpenfaceVideo ${PYTHON_LIBRARIES}) +install (TARGETS PyOpenfaceVideo DESTINATION bin) \ No newline at end of file diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 641b1d7c0..892913601 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -76,6 +76,10 @@ static void printErrorAndAbort(const std::string & error) #define FATAL_STREAM( stream ) \ printErrorAndAbort( std::string( "Fatal error: " ) + stream ) +#define LANDMARKS_NUM 68 + +boost::shared_mutex _access; + int OpenfaceVideoWorker( std::vector& args, const bool& running, @@ -267,6 +271,30 @@ int OpenfaceVideoWorker( } +struct OpenfaceV3 { + float x; + float y; + float z; + OpenfaceV3() : x(0), y(0), z(0) {} + void operator = ( const cv::Point3f& src ) { + x = src.x; + y = src.y; + z = src.z; + } +} + +struct OpenfaceFrame { + bool updated; + OpenfaceV3 gaze_0; + OpenfaceV3 gaze_1; + OpenfaceV3 head_rot; + OpenfaceV3 head_pos; + std::vector landmarks; + OpenfaceFrame(): updated(false) { + landmarks.resize(LANDMARKS_NUM); + } +}; + class OpenfaceVideo { public: @@ -306,14 +334,26 @@ class OpenfaceVideo { ) ); openfacevideo_threads.add_thread(worker); + } void new_frame( Utilities::RecorderOpenFace& rec ) { - const cv::Mat_& l3d = rec.get_landmarks_3D(); + boost::unique_lock< boost::shared_mutex > lock(_access); + cv::Size s = l3d.size(); std::cout << "new_frame " << rec.GetCSVFile() << ", rows: " << s.height << ", cols: " << s.width << std::endl; + frame.gaze_0 = rec.get_gaze_direction(0); + frame.gaze_1 = rec.get_gaze_direction(1); + const cv::Mat_& l3d = rec.get_landmarks_3D(); + for ( int c = 0; c < s.width; ++c ) { + landmarks[i].x = l3d.at( c,0 ); + landmarks[i].y = l3d.at( c,1 ); + landmarks[i].z = l3d.at( c,2 ); + } + frame.updated = false; + } void stop() { @@ -327,6 +367,17 @@ class OpenfaceVideo { inline bool is_running() const { return worker != 0; } + + inline bool data_updated() const { + boost::unique_lock< boost::shared_mutex > lock(_access); + return frame.updated; + } + + inline const OpenfaceFrame& get_frame() { + boost::unique_lock< boost::shared_mutex > lock(_access); + frame.updated = false; + return frame + } private: @@ -350,11 +401,84 @@ class OpenfaceVideo { std::vector arguments; bool thread_running; + bool _data_updated; boost::thread_group openfacevideo_threads; boost::thread* worker; + OpenfaceFrame frame; + }; +class with_gil { +public: + with_gil() { state_ = PyGILState_Ensure(); } + ~with_gil() { PyGILState_Release(state_); } + with_gil(const with_gil&) = delete; + with_gil& operator=(const with_gil&) = delete; +private: + PyGILState_STATE state_; +}; + +class py_callable { +public: + + /// @brief Constructor that assumes the caller has the GIL locked. + py_callable(const boost::python::object& object) { + with_gil gil; + object_.reset( + // GIL locked, so it is safe to copy. + new boost::python::object{object}, + // Use a custom deleter to hold GIL when the object is deleted. + [](boost::python::object* object) { + with_gil gil; + delete object; + }); + } + + // Use default copy-constructor and assignment-operator. + py_callable(const py_callable&) = default; + py_callable& operator=(const py_callable&) = default; + + template + void operator()(Args... args) { + // Lock the GIL as the python object is going to be invoked. + with_gil gil; + (*object_)(std::forward(args)...); + } + +private: + std::shared_ptr object_; + +}; + +BOOST_PYTHON_MODULE(PyOpenfaceVideo) { + + using namespace boost::python; + + class_("OpenfaceV3") + .def_read("x", &OpenfaceV3::x) + .def_read("y", &OpenfaceV3::y) + .def_read("z", &OpenfaceV3::z); + + class_>("OpenfaceV3Vector") + .def(vector_indexing_suite>()); + + class_("OpenfaceFrame") + .def_read("gaze_0", &OpenfaceV3::gaze_0) + .def_read("gaze_1", &OpenfaceV3::gaze_1) + .def_read("head_rot", &OpenfaceV3::head_rot) + .def_read("head_pos", &OpenfaceV3::head_pos) + .def_read("landmarks", &OpenfaceV3::landmarks); + + class_("OpenfaceVideo") + .def("start", &OpenfaceVideo::start) + .def("stop", &OpenfaceVideo::stop) + .def("is_running", &OpenfaceVideo::is_running) + .def("data_updated", &OpenfaceVideo::data_updated) + .def("get_frame", &OpenfaceVideo::get_frame); + +} + int main(int argc, char **argv) { OpenfaceVideo* ov = new OpenfaceVideo(); From 4aabd11297b6bcd0e392092b2aa2cef79b98a522 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 16:54:13 +0200 Subject: [PATCH 04/27] parsing of head pose + frame dump method --- .../FeatureExtractionPython.cpp | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 892913601..d420a714d 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -32,6 +32,7 @@ // /////////////////////////////////////////////////////////////////////////////// +// tuned by frankiezafe, http://polymorph.cool // FeatureExtraction.cpp : Defines the entry point for the feature extraction console application. @@ -281,6 +282,11 @@ struct OpenfaceV3 { y = src.y; z = src.z; } + void set( const float& vx, const float& vy, const float& vz ) { + x = vx; + y = vy; + z = vz; + } } struct OpenfaceFrame { @@ -293,6 +299,20 @@ struct OpenfaceFrame { OpenfaceFrame(): updated(false) { landmarks.resize(LANDMARKS_NUM); } + void print() { + std::cout << + "OpenfaceFrame" << std::endl << + "\t" << "updated: " << updated << std::endl << + "\t" << "gaze_0: " << gaze_0.x << ", " << gaze_0.y << ", " << gaze_0.z << std::endl << + "\t" << "gaze_1: " << gaze_1.x << ", " << gaze_1.y << ", " << gaze_1.z << std::endl << + "\t" << "head_rot: " << head_rot.x << ", " << head_rot.y << ", " << head_rot.z << std::endl << + "\t" << "head_pos: " << head_pos.x << ", " << head_pos.y << ", " << head_pos.z << std::endl; + for ( int i = 0; i < LANDMARKS_NUM; ++i ) { + std::cout << "\tlandmark [" << i < "] : " << + landmarks[i].x << ", " << landmarks[i].y << ", " << landmarks[i].z << std::endl; + } + + } }; class OpenfaceVideo { @@ -346,6 +366,11 @@ class OpenfaceVideo { frame.gaze_0 = rec.get_gaze_direction(0); frame.gaze_1 = rec.get_gaze_direction(1); + + cv::Vec6f h = rec.get_head_pose(); + frame.head_rot.set( h.at(0), h.at(1), h.at(2) ) + frame.head_pos.set( h.at(3), h.at(4), h.at(5) ) + const cv::Mat_& l3d = rec.get_landmarks_3D(); for ( int c = 0; c < s.width; ++c ) { landmarks[i].x = l3d.at( c,0 ); @@ -353,6 +378,7 @@ class OpenfaceVideo { landmarks[i].z = l3d.at( c,2 ); } frame.updated = false; + frame.print(); } @@ -368,7 +394,7 @@ class OpenfaceVideo { return worker != 0; } - inline bool data_updated() const { + inline bool new_frame() const { boost::unique_lock< boost::shared_mutex > lock(_access); return frame.updated; } @@ -474,7 +500,7 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { .def("start", &OpenfaceVideo::start) .def("stop", &OpenfaceVideo::stop) .def("is_running", &OpenfaceVideo::is_running) - .def("data_updated", &OpenfaceVideo::data_updated) + .def("new_frame", &OpenfaceVideo::data_updated) .def("get_frame", &OpenfaceVideo::get_frame); } From 73d7e5405fc8590866f0997373d076819747665b Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:02:42 +0200 Subject: [PATCH 05/27] adding flag -fPIC to compile library --- exe/FeatureExtractionPython/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 01cb1af4a..53141e217 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -14,6 +14,7 @@ install (TARGETS OpenfaceVideo DESTINATION bin) # LIBRARY add_library(PyOpenfaceVideo SHARED FeatureExtractionPython.cpp) +set_property(TARGET PyOpenfaceVideo PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(PyOpenfaceVideo LandmarkDetector) target_link_libraries(PyOpenfaceVideo FaceAnalyser) target_link_libraries(PyOpenfaceVideo GazeAnalyser) From 5e7e595d8ed434a36545052001ba53bfbcf0c942 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:03:15 +0200 Subject: [PATCH 06/27] debugging of all methods to enable python binding --- .../FeatureExtractionPython.cpp | 153 ++++++++++++++---- 1 file changed, 121 insertions(+), 32 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index d420a714d..d4cd5d6d4 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include // Local includes @@ -273,32 +274,68 @@ int OpenfaceVideoWorker( } struct OpenfaceV3 { + float x; float y; float z; + OpenfaceV3() : x(0), y(0), z(0) {} + + OpenfaceV3( const OpenfaceV3& src ) { + (*this) = src; + } + + ~OpenfaceV3() {} + + bool operator == ( const OpenfaceV3& src ) const { + return x == src.x && y == src.y && z == src.z; + } + + bool operator != ( const OpenfaceV3& src ) const { + return x != src.x || y != src.y || z != src.z; + } + + void operator = ( const OpenfaceV3& src ) { + x = src.x; + y = src.y; + z = src.z; + } + void operator = ( const cv::Point3f& src ) { x = src.x; y = src.y; z = src.z; } + void set( const float& vx, const float& vy, const float& vz ) { x = vx; y = vy; z = vz; } -} + +}; + +typedef std::vector OpenfaceV3Vector; struct OpenfaceFrame { + bool updated; OpenfaceV3 gaze_0; OpenfaceV3 gaze_1; OpenfaceV3 head_rot; OpenfaceV3 head_pos; std::vector landmarks; + OpenfaceFrame(): updated(false) { landmarks.resize(LANDMARKS_NUM); } + + OpenfaceFrame( const OpenfaceFrame& src ) { + (*this) = src; + } + + ~OpenfaceFrame() {} + void print() { std::cout << "OpenfaceFrame" << std::endl << @@ -308,11 +345,56 @@ struct OpenfaceFrame { "\t" << "head_rot: " << head_rot.x << ", " << head_rot.y << ", " << head_rot.z << std::endl << "\t" << "head_pos: " << head_pos.x << ", " << head_pos.y << ", " << head_pos.z << std::endl; for ( int i = 0; i < LANDMARKS_NUM; ++i ) { - std::cout << "\tlandmark [" << i < "] : " << + std::cout << "\tlandmark [" << i << "] : " << landmarks[i].x << ", " << landmarks[i].y << ", " << landmarks[i].z << std::endl; } - } + + bool operator == ( const OpenfaceFrame& src ) { + if ( + gaze_0 != src.gaze_0 || + gaze_1 != src.gaze_1 || + head_rot != src.head_rot || + head_pos != src.head_pos + ) { + return false; + } + for ( int i = 0; i < LANDMARKS_NUM; ++i ) { + if ( landmarks[i] != src.landmarks[i] ) { return false; } + } + return true; + } + + bool operator != ( const OpenfaceFrame& src ) { + if ( + gaze_0 != src.gaze_0 || + gaze_1 != src.gaze_1 || + head_rot != src.head_rot || + head_pos != src.head_pos + ) { + return true; + } + for ( int i = 0; i < LANDMARKS_NUM; ++i ) { + if ( landmarks[i] != src.landmarks[i] ) { return true; } + } + return false; + } + + void operator = ( const OpenfaceFrame& src ) { + + updated = src.updated; + gaze_0 = src.gaze_0; + gaze_1 = src.gaze_1; + head_rot = src.head_rot; + head_pos = src.head_pos; + for ( int i = 0; i < LANDMARKS_NUM; ++i ) { + landmarks[i] = src.landmarks[i]; + } + + } + + + }; class OpenfaceVideo { @@ -321,6 +403,10 @@ class OpenfaceVideo { OpenfaceVideo() : worker(0) {} + OpenfaceVideo( const OpenfaceVideo& src ) : worker(0) { + std::cout << "OpenfaceVideo copy cstr is not implemented" << std::endl; + } + ~OpenfaceVideo() { // clean stop stop(); @@ -337,7 +423,7 @@ class OpenfaceVideo { append_argument( "-device", "0" ); } - void start() { + bool start() { stop(); thread_running = true; @@ -355,12 +441,15 @@ class OpenfaceVideo { ); openfacevideo_threads.add_thread(worker); + return true; + } void new_frame( Utilities::RecorderOpenFace& rec ) { boost::unique_lock< boost::shared_mutex > lock(_access); + const cv::Mat_& l3d = rec.get_landmarks_3D(); cv::Size s = l3d.size(); std::cout << "new_frame " << rec.GetCSVFile() << ", rows: " << s.height << ", cols: " << s.width << std::endl; @@ -368,41 +457,42 @@ class OpenfaceVideo { frame.gaze_1 = rec.get_gaze_direction(1); cv::Vec6f h = rec.get_head_pose(); - frame.head_rot.set( h.at(0), h.at(1), h.at(2) ) - frame.head_pos.set( h.at(3), h.at(4), h.at(5) ) + frame.head_rot.set( h[0], h[1], h[2] ); + frame.head_pos.set( h[3], h[4], h[5] ); - const cv::Mat_& l3d = rec.get_landmarks_3D(); for ( int c = 0; c < s.width; ++c ) { - landmarks[i].x = l3d.at( c,0 ); - landmarks[i].y = l3d.at( c,1 ); - landmarks[i].z = l3d.at( c,2 ); + frame.landmarks[c].x = l3d.at( c,0 ); + frame.landmarks[c].y = l3d.at( c,1 ); + frame.landmarks[c].z = l3d.at( c,2 ); } frame.updated = false; frame.print(); } - void stop() { + bool stop() { if ( thread_running ) { thread_running = false; usleep( 1000 ); worker = 0; + return true; } + return false; } - inline bool is_running() const { + bool is_running() const { return worker != 0; } - inline bool new_frame() const { + bool has_frame() const { boost::unique_lock< boost::shared_mutex > lock(_access); return frame.updated; } - inline const OpenfaceFrame& get_frame() { + OpenfaceFrame get_frame() { boost::unique_lock< boost::shared_mutex > lock(_access); frame.updated = false; - return frame + return frame; } private: @@ -482,27 +572,26 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { using namespace boost::python; class_("OpenfaceV3") - .def_read("x", &OpenfaceV3::x) - .def_read("y", &OpenfaceV3::y) - .def_read("z", &OpenfaceV3::z); + .def_readonly("x", &OpenfaceV3::x) + .def_readonly("y", &OpenfaceV3::y) + .def_readonly("z", &OpenfaceV3::z); - class_>("OpenfaceV3Vector") - .def(vector_indexing_suite>()); + class_("OpenfaceV3Vector") + .def(vector_indexing_suite()); - class_("OpenfaceFrame") - .def_read("gaze_0", &OpenfaceV3::gaze_0) - .def_read("gaze_1", &OpenfaceV3::gaze_1) - .def_read("head_rot", &OpenfaceV3::head_rot) - .def_read("head_pos", &OpenfaceV3::head_pos) - .def_read("landmarks", &OpenfaceV3::landmarks); + class_("OpenfaceFrame") + .def_readonly("gaze_0", &OpenfaceFrame::gaze_0) + .def_readonly("gaze_1", &OpenfaceFrame::gaze_1) + .def_readonly("head_rot", &OpenfaceFrame::head_rot) + .def_readonly("head_pos", &OpenfaceFrame::head_pos) + .def_readonly("landmarks", &OpenfaceFrame::landmarks); class_("OpenfaceVideo") - .def("start", &OpenfaceVideo::start) - .def("stop", &OpenfaceVideo::stop) - .def("is_running", &OpenfaceVideo::is_running) - .def("new_frame", &OpenfaceVideo::data_updated) - .def("get_frame", &OpenfaceVideo::get_frame); - + .add_property("start", &OpenfaceVideo::start) + .add_property("stop", &OpenfaceVideo::stop) + .add_property("is_running", &OpenfaceVideo::is_running) + .add_property("has_frame", &OpenfaceVideo::has_frame) + .add_property("get_frame", &OpenfaceVideo::get_frame); } int main(int argc, char **argv) { From f0f6e03d667902fd6efb4a87a5e6d0988a1125bf Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:08:26 +0200 Subject: [PATCH 07/27] link to ReVA repo in readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fa610184..18198b50a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # OpenFace 2.2.0: a facial behavior analysis toolkit -*Modified version of [original repository](https://github.com/TadasBaltrusaitis/OpenFace) to allow usage of webcam feature extraction from python. This is NOT a stable version.* +*Modified version of [original repository](https://github.com/TadasBaltrusaitis/OpenFace) to allow usage of webcam feature extraction from python. This is NOT a stable version.* + +part of the [ReVA-toolkit](https://github.com/numediart/ReVA-toolkit) See 'exe/FeatureExtractionPython' for code and launch 'build/bin/OpenfaceVideo' From c06a425fe9db0f7b0540285f1c3da046e4d0f738 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:08:39 +0200 Subject: [PATCH 08/27] minor --- exe/FeatureExtractionPython/FeatureExtractionPython.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index d4cd5d6d4..22b630d39 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -392,10 +392,8 @@ struct OpenfaceFrame { } } - - - -}; + +}; class OpenfaceVideo { From f68eb87f874f3b3bc10315ee13566365e6c1ebc8 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:33:02 +0200 Subject: [PATCH 09/27] adding set_device method in OpenfaceVideo class --- .../FeatureExtractionPython.cpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 22b630d39..617d5cd78 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -381,7 +381,6 @@ struct OpenfaceFrame { } void operator = ( const OpenfaceFrame& src ) { - updated = src.updated; gaze_0 = src.gaze_0; gaze_1 = src.gaze_1; @@ -390,7 +389,6 @@ struct OpenfaceFrame { for ( int i = 0; i < LANDMARKS_NUM; ++i ) { landmarks[i] = src.landmarks[i]; } - } }; @@ -414,15 +412,36 @@ class OpenfaceVideo { for (int i = 0; i < argc; ++i) { arguments.push_back(std::string(argv[i])); } - append_argument( "-2Dfp", "" ); + //append_argument( "-2Dfp", "" ); append_argument( "-3Dfp", "" ); append_argument( "-pose", "" ); append_argument( "-gaze", "" ); append_argument( "-device", "0" ); } + + void set_device( int d ) { + + stop(); + + if ( arguments.empty() ) { + load_arguments(0,0); + } + for ( int i = 0, imax = arguments.size(); i < imax; ++i ) { + if ( arguments[i].compare( "-device" ) == 0 ) { + arguments[i+1] = std::to_string(d); + return; + } + } + append_argument( "-device", std::to_string(d) ); + + } bool start() { + if ( arguments.empty() ) { + load_arguments(0,0); + } + stop(); thread_running = true; worker = new boost::thread(); @@ -585,6 +604,7 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { .def_readonly("landmarks", &OpenfaceFrame::landmarks); class_("OpenfaceVideo") + .add_property("set_device", &OpenfaceVideo::set_device) .add_property("start", &OpenfaceVideo::start) .add_property("stop", &OpenfaceVideo::stop) .add_property("is_running", &OpenfaceVideo::is_running) From d99fa9e13568778bcd5f4b3d72a6eeaef510ccba Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 18:40:52 +0200 Subject: [PATCH 10/27] cmake flags for python 3 --- exe/FeatureExtractionPython/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 53141e217..489bbc572 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -1,4 +1,7 @@ # Local libraries +#set(Python_ADDITIONAL_VERSIONS 3.5) +#find_package(PythonLibs 3 REQUIRED) + include_directories(${PYTHON_INCLUDE_DIRS}) link_directories(${PYTHON_LIBRARIES}) include_directories(${LandmarkDetector_SOURCE_DIR}/include) @@ -13,8 +16,8 @@ target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) install (TARGETS OpenfaceVideo DESTINATION bin) # LIBRARY -add_library(PyOpenfaceVideo SHARED FeatureExtractionPython.cpp) -set_property(TARGET PyOpenfaceVideo PROPERTY POSITION_INDEPENDENT_CODE ON) +add_library(PyOpenfaceVideo STATIC FeatureExtractionPython.cpp) +# set_property(TARGET PyOpenfaceVideo PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(PyOpenfaceVideo LandmarkDetector) target_link_libraries(PyOpenfaceVideo FaceAnalyser) target_link_libraries(PyOpenfaceVideo GazeAnalyser) From 1720f6f74df714ad61aee34ddfdb2eb5f3e58b4b Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 19:56:21 +0200 Subject: [PATCH 11/27] preparation of shared library compilation --- .gitignore | 5 ++++ CMakeLists.txt | 1 - exe/FeatureExtractionPython/CMakeLists.txt | 27 ++++++++++++++-------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 14d939274..0966141ca 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,8 @@ lib/local/CppInerop/Debug/ # IDE-generated folders .idea + +4.1.0.zip +dlib-19.13/* +opencv-3.4.0/* +opencv-4.1.0/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 04afad2f1..7b31cf3a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -150,7 +150,6 @@ find_package(dlib 19.13) if(${dlib_FOUND}) message("dlib information:") message(" dlib version: ${dlib_VERSION}") - if (NOT TARGET dlib) add_library(dlib INTERFACE IMPORTED GLOBAL) endif() diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 489bbc572..75d470e66 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -1,26 +1,35 @@ # Local libraries #set(Python_ADDITIONAL_VERSIONS 3.5) -#find_package(PythonLibs 3 REQUIRED) +#find_package(PythonLibs 3 REQUIRED) +#find_package(Boost REQUIRED COMPONENTS python-py35) +#set(PYTHON_DOT_VERSION 3.5) +#set(PYTHON_INCLUDE /usr/include/python3.5) +#set(PYTHON_LIBRARY /usr/lib/python3.5/config-x86_64-linux-gnu) + +set(Python_ADDITIONAL_VERSIONS 3.5) +find_package(PythonLibs 3 REQUIRED) include_directories(${PYTHON_INCLUDE_DIRS}) link_directories(${PYTHON_LIBRARIES}) include_directories(${LandmarkDetector_SOURCE_DIR}/include) # EXEC -add_executable(OpenfaceVideo FeatureExtractionPython.cpp) -target_link_libraries(OpenfaceVideo LandmarkDetector) -target_link_libraries(OpenfaceVideo FaceAnalyser) -target_link_libraries(OpenfaceVideo GazeAnalyser) -target_link_libraries(OpenfaceVideo Utilities) -target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) -install (TARGETS OpenfaceVideo DESTINATION bin) +#add_executable(OpenfaceVideo FeatureExtractionPython.cpp) +#target_link_libraries(OpenfaceVideo LandmarkDetector) +#target_link_libraries(OpenfaceVideo FaceAnalyser) +#target_link_libraries(OpenfaceVideo GazeAnalyser) +#target_link_libraries(OpenfaceVideo Utilities) +#target_link_libraries(OpenfaceVideo ${Boost_LIBRARIES}) +#target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) +#install (TARGETS OpenfaceVideo DESTINATION bin) # LIBRARY -add_library(PyOpenfaceVideo STATIC FeatureExtractionPython.cpp) +add_library(PyOpenfaceVideo SHARED FeatureExtractionPython.cpp) # set_property(TARGET PyOpenfaceVideo PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(PyOpenfaceVideo LandmarkDetector) target_link_libraries(PyOpenfaceVideo FaceAnalyser) target_link_libraries(PyOpenfaceVideo GazeAnalyser) target_link_libraries(PyOpenfaceVideo Utilities) +target_link_libraries(PyOpenfaceVideo ${Boost_LIBRARIES}) target_link_libraries(PyOpenfaceVideo ${PYTHON_LIBRARIES}) install (TARGETS PyOpenfaceVideo DESTINATION bin) \ No newline at end of file From 20641a14ad423781503fdcc82e9b2d312b63e0ff Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 20:28:49 +0200 Subject: [PATCH 12/27] modification to cmake to compile a shared library (requires manual edition of dlib cmake) --- README.md | 10 ++- install.sh | 3 +- lib/local/FaceAnalyser/CMakeLists.txt | 2 +- lib/local/GazeAnalyser/CMakeLists.txt | 2 +- lib/local/LandmarkDetector/CMakeLists.txt | 2 +- lib/local/Utilities/CMakeLists.txt | 2 +- recompile.sh | 97 +++++++++++++++++++++++ 7 files changed, 111 insertions(+), 7 deletions(-) create mode 100755 recompile.sh diff --git a/README.md b/README.md index 18198b50a..04d74536d 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,20 @@ part of the [ReVA-toolkit](https://github.com/numediart/ReVA-toolkit) -See 'exe/FeatureExtractionPython' for code and launch 'build/bin/OpenfaceVideo' - requires python dev libs: ```bash $ sudo apt-get install python-dev python3-dev ``` +run `install.sh` (it will crash) + +open **dlib-19.13/dlib/CMakeLists.txt**, locate **add_library(dlib_all_source_cpp STATIC all/source.cpp)** and change it to **add_library(dlib_all_source_cpp SHARED all/source.cpp)** + +run `recompile.sh` + +[soource](https://github.com/TadasBaltrusaitis/OpenFace/issues/23#issuecomment-229495315) + ## Description Over the past few years, there has been an increased interest in automatic facial behavior analysis diff --git a/install.sh b/install.sh index cb01115bb..422f95195 100755 --- a/install.sh +++ b/install.sh @@ -58,6 +58,7 @@ sudo apt-get -y install zip sudo apt-get -y install libopenblas-dev liblapack-dev sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev +sudo apt-get -y python-dev python3-dev echo "Essential dependencies installed." # OpenCV Dependency @@ -72,7 +73,7 @@ cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB make -j4 sudo make install cd ../.. -rm 4.1.0.zip +# rm 4.1.0.zip sudo rm -r opencv-4.1.0 echo "OpenCV installed." diff --git a/lib/local/FaceAnalyser/CMakeLists.txt b/lib/local/FaceAnalyser/CMakeLists.txt index c5c2bc8de..38414fff7 100644 --- a/lib/local/FaceAnalyser/CMakeLists.txt +++ b/lib/local/FaceAnalyser/CMakeLists.txt @@ -27,7 +27,7 @@ SET(HEADERS ) -add_library( FaceAnalyser ${SOURCE} ${HEADERS}) +add_library( FaceAnalyser SHARED ${SOURCE} ${HEADERS}) add_library( OpenFace::FaceAnalyser ALIAS FaceAnalyser) target_include_directories(FaceAnalyser PUBLIC diff --git a/lib/local/GazeAnalyser/CMakeLists.txt b/lib/local/GazeAnalyser/CMakeLists.txt index 9c567c90e..d2e55bd31 100644 --- a/lib/local/GazeAnalyser/CMakeLists.txt +++ b/lib/local/GazeAnalyser/CMakeLists.txt @@ -12,7 +12,7 @@ SET(HEADERS include/GazeEstimation.h ) -add_library( GazeAnalyser ${SOURCE} ${HEADERS}) +add_library( GazeAnalyser SHARED ${SOURCE} ${HEADERS}) add_library( OpenFace::GazeAnalyser ALIAS GazeAnalyser) target_include_directories(GazeAnalyser PUBLIC diff --git a/lib/local/LandmarkDetector/CMakeLists.txt b/lib/local/LandmarkDetector/CMakeLists.txt index a561361e1..0d3ca3ec4 100644 --- a/lib/local/LandmarkDetector/CMakeLists.txt +++ b/lib/local/LandmarkDetector/CMakeLists.txt @@ -37,7 +37,7 @@ SET(HEADERS include/stdafx.h ) -add_library( LandmarkDetector ${SOURCE} ${HEADERS} ) +add_library( LandmarkDetector SHARED ${SOURCE} ${HEADERS} ) add_library( OpenFace::LandmarkDetector ALIAS LandmarkDetector) target_include_directories(LandmarkDetector PUBLIC diff --git a/lib/local/Utilities/CMakeLists.txt b/lib/local/Utilities/CMakeLists.txt index 8fcbc9790..c706b387c 100644 --- a/lib/local/Utilities/CMakeLists.txt +++ b/lib/local/Utilities/CMakeLists.txt @@ -23,7 +23,7 @@ SET(HEADERS include/ConcurrentQueue.h ) -add_library( Utilities ${SOURCE} ${HEADERS}) +add_library( Utilities SHARED ${SOURCE} ${HEADERS}) add_library( OpenFace::Utilities ALIAS Utilities) target_include_directories(Utilities PUBLIC diff --git a/recompile.sh b/recompile.sh new file mode 100755 index 000000000..689d3d32e --- /dev/null +++ b/recompile.sh @@ -0,0 +1,97 @@ +#!/bin/bash +#============================================================================== +# Title: install.sh +# Description: Install everything necessary for OpenFace to compile. +# Will install all required dependencies, only use if you do not have the dependencies +# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed +# Author: Daniyal Shahrokhian , Tadas Baltrusaitis +# Date: 20190630 +# Version : 1.03 +# Usage: bash install.sh +#============================================================================== + +# Exit script if any command fails +set -e +set -o pipefail + +if [ $# -ne 0 ] + then + echo "Usage: recompile.sh" + exit 1 +fi + +# Essential Dependencies +#echo "Installing Essential dependencies..." +#sudo apt-get -y update +#if [[ `lsb_release -rs` != "18.04" ]] +# then +# echo "Adding ppa:ubuntu-toolchain-r/test apt-repository " +# sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y +# sudo apt-get -y update +#fi +#sudo apt-get -y install build-essential +#sudo apt-get -y install gcc-8 g++-8 +# Ubuntu 16.04 does not have newest CMake so need to build it manually +#if [[ `lsb_release -rs` != "18.04" ]]; then +# sudo apt-get --purge remove cmake-qt-gui -y +# sudo apt-get --purge remove cmake -y +# mkdir -p cmake_tmp +# cd cmake_tmp +# wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz +# tar -xzvf cmake-3.10.1.tar.gz +# cd cmake-3.10.1/ +# ./bootstrap +# make -j4 +# sudo make install +# cd ../.. +# sudo rm -r cmake_tmp +#else +# sudo apt-get -y install cmake +#fi + +#sudo apt-get -y install zip +#sudo apt-get -y install libopenblas-dev liblapack-dev +#sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev +#sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev +#echo "Essential dependencies installed." + +# OpenCV Dependency +#echo "Downloading OpenCV..." +#wget https://github.com/opencv/opencv/archive/4.1.0.zip +#unzip 4.1.0.zip +cd opencv-4.1.0 +mkdir -p build +cd build +echo "Installing OpenCV..." +cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. +make -j7 +sudo make install +cd ../.. +#rm 4.1.0.zip +#sudo rm -r opencv-4.1.0 +echo "OpenCV installed." + +# dlib dependecy +echo "Downloading dlib" +#wget http://dlib.net/files/dlib-19.13.tar.bz2; +#tar xf dlib-19.13.tar.bz2; +cd dlib-19.13; +mkdir -p build; +cd build; +echo "Installing dlib" +cmake ..; +cmake --build . --config Release; +sudo make install; +sudo ldconfig; +cd ../..; +#rm -r dlib-19.13.tar.bz2 +echo "dlib installed" + +# OpenFace installation +echo "Installing OpenFace..." +mkdir -p build +cd build +cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. +make +cd .. +echo "OpenFace successfully installed." \ No newline at end of file From 9325a04ff6ba2a71ed7a978a345d3fc3c1b49a4c Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 21:41:56 +0200 Subject: [PATCH 13/27] bug fixes in installation scripts --- .gitignore | 2 + README.md | 9 ++--- compile.sh | 55 +++++++++++++++++++++++++++ install.sh | 104 ++------------------------------------------------- prepare.sh | 76 +++++++++++++++++++++++++++++++++++++ recompile.sh | 97 ----------------------------------------------- 6 files changed, 139 insertions(+), 204 deletions(-) create mode 100755 compile.sh create mode 100755 prepare.sh delete mode 100755 recompile.sh diff --git a/.gitignore b/.gitignore index 0966141ca..2e7f998db 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,5 @@ lib/local/CppInerop/Debug/ dlib-19.13/* opencv-3.4.0/* opencv-4.1.0/* +dlib-19.13.tar.bz2 +cmake_tmp/* diff --git a/README.md b/README.md index 04d74536d..d60451c5e 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,10 @@ requires python dev libs: $ sudo apt-get install python-dev python3-dev ``` -run `install.sh` (it will crash) +- run `chmod +x prepare.sh && ./prepare.sh` : this will download all dependencies and install system libs +- run `chmod +x compile.sh && ./compile.sh` : this will compile everything -open **dlib-19.13/dlib/CMakeLists.txt**, locate **add_library(dlib_all_source_cpp STATIC all/source.cpp)** and change it to **add_library(dlib_all_source_cpp SHARED all/source.cpp)** - -run `recompile.sh` - -[soource](https://github.com/TadasBaltrusaitis/OpenFace/issues/23#issuecomment-229495315) +[source](https://github.com/TadasBaltrusaitis/OpenFace/issues/23#issuecomment-229495315) ## Description diff --git a/compile.sh b/compile.sh new file mode 100755 index 000000000..77af50e65 --- /dev/null +++ b/compile.sh @@ -0,0 +1,55 @@ +#!/bin/bash +#============================================================================== +# Title: install.sh +# Description: Install everything necessary for OpenFace to compile. +# Will install all required dependencies, only use if you do not have the dependencies +# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed +# Author: Daniyal Shahrokhian , Tadas Baltrusaitis +# Date: 20190630 +# Version : 1.03 +# Usage: bash install.sh +#============================================================================== + +# Exit script if any command fails +set -e +set -o pipefail + +if [ $# -ne 0 ] + then + echo "Usage: recompile.sh" + exit 1 +fi + +# OpenCV Dependency +cd opencv-4.1.0 +mkdir -p build +cd build +echo "Installing OpenCV..." +cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. +make -j7 +sudo make install +cd ../.. +echo "OpenCV installed." + +# dlib dependecy +echo "Downloading dlib" +cd dlib-19.13; +rm -rf build +mkdir -p build; +cd build; +echo "Installing dlib" +cmake -D CMAKE_BUILD_TYPE=RELEASE -D DLIB_IN_PROJECT_BUILD=ON -D DLIB_USE_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. +make -j7 +sudo make install; +sudo ldconfig; +cd ../..; +echo "dlib installed" + +# OpenFace installation +echo "Installing OpenFace..." +mkdir -p build +cd build +cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. +make +cd .. +echo "OpenFace successfully installed." diff --git a/install.sh b/install.sh index 422f95195..1b1136870 100755 --- a/install.sh +++ b/install.sh @@ -1,103 +1,5 @@ #!/bin/bash -#============================================================================== -# Title: install.sh -# Description: Install everything necessary for OpenFace to compile. -# Will install all required dependencies, only use if you do not have the dependencies -# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed -# Author: Daniyal Shahrokhian , Tadas Baltrusaitis -# Date: 20190630 -# Version : 1.03 -# Usage: bash install.sh -#============================================================================== -# Exit script if any command fails -set -e -set -o pipefail - -if [ $# -ne 0 ] - then - echo "Usage: install.sh" - exit 1 -fi - -# Essential Dependencies -echo "Installing Essential dependencies..." - -# If we're not on 18.04 -sudo apt-get -y update - -if [[ `lsb_release -rs` != "18.04" ]] - then - echo "Adding ppa:ubuntu-toolchain-r/test apt-repository " - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get -y update -fi - -sudo apt-get -y install build-essential -sudo apt-get -y install gcc-8 g++-8 - -# Ubuntu 16.04 does not have newest CMake so need to build it manually -if [[ `lsb_release -rs` != "18.04" ]]; then - sudo apt-get --purge remove cmake-qt-gui -y - sudo apt-get --purge remove cmake -y - mkdir -p cmake_tmp - cd cmake_tmp - wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz - tar -xzvf cmake-3.10.1.tar.gz - cd cmake-3.10.1/ - ./bootstrap - make -j4 - sudo make install - cd ../.. - sudo rm -r cmake_tmp -else - sudo apt-get -y install cmake -fi - -sudo apt-get -y install zip -sudo apt-get -y install libopenblas-dev liblapack-dev -sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev -sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev -sudo apt-get -y python-dev python3-dev -echo "Essential dependencies installed." - -# OpenCV Dependency -echo "Downloading OpenCV..." -wget https://github.com/opencv/opencv/archive/4.1.0.zip -unzip 4.1.0.zip -cd opencv-4.1.0 -mkdir -p build -cd build -echo "Installing OpenCV..." -cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_CUDA=OFF -D BUILD_SHARED_LIBS=OFF .. -make -j4 -sudo make install -cd ../.. -# rm 4.1.0.zip -sudo rm -r opencv-4.1.0 -echo "OpenCV installed." - -# dlib dependecy -echo "Downloading dlib" -wget http://dlib.net/files/dlib-19.13.tar.bz2; -tar xf dlib-19.13.tar.bz2; -cd dlib-19.13; -mkdir -p build; -cd build; -echo "Installing dlib" -cmake ..; -cmake --build . --config Release; -sudo make install; -sudo ldconfig; -cd ../..; -rm -r dlib-19.13.tar.bz2 -echo "dlib installed" - -# OpenFace installation -echo "Installing OpenFace..." -mkdir -p build -cd build -cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. -make -cd .. -echo "OpenFace successfully installed." \ No newline at end of file +bash download_models.sh +bash prepare.sh +bash compile.sh \ No newline at end of file diff --git a/prepare.sh b/prepare.sh new file mode 100755 index 000000000..59563053d --- /dev/null +++ b/prepare.sh @@ -0,0 +1,76 @@ +#!/bin/bash +#============================================================================== +# Title: install.sh +# Description: Install everything necessary for OpenFace to compile. +# Will install all required dependencies, only use if you do not have the dependencies +# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed +# Author: Daniyal Shahrokhian , Tadas Baltrusaitis +# Date: 20190630 +# Version : 1.03 +# Usage: bash install.sh +#============================================================================== + +# Exit script if any command fails +set -e +set -o pipefail + +if [ $# -ne 0 ] + then + echo "Usage: prepare.sh" + exit 1 +fi + +# Essential Dependencies +echo "Installing Essential dependencies..." + +# If we're not on 18.04 +sudo apt-get -y update + +if [[ `lsb_release -rs` != "18.04" ]] + then + echo "Adding ppa:ubuntu-toolchain-r/test apt-repository " + sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y + sudo apt-get -y update +fi + +sudo apt-get -y install build-essential +sudo apt-get -y install gcc-8 g++-8 + +# Ubuntu 16.04 does not have newest CMake so need to build it manually +if [[ `lsb_release -rs` != "18.04" ]]; then + sudo apt-get --purge remove cmake-qt-gui -y + sudo apt-get --purge remove cmake -y + mkdir -p cmake_tmp + cd cmake_tmp + wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz + tar -xzvf cmake-3.10.1.tar.gz + cd cmake-3.10.1/ + ./bootstrap + make -j4 + sudo make install + cd ../.. + sudo rm -r cmake_tmp +else + sudo apt-get -y install cmake +fi + +sudo apt-get -y install zip +sudo apt-get -y install libopenblas-dev liblapack-dev +sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev +sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev +sudo apt-get -y install python-dev python3-dev +echo "Essential dependencies installed." + +# OpenCV Dependency +echo "Downloading OpenCV..." +wget https://github.com/opencv/opencv/archive/4.1.0.zip +unzip 4.1.0.zip +sudo rm -r opencv-4.1.0 +echo "OpenCV ready." + +# dlib dependecy +echo "Downloading dlib" +wget http://dlib.net/files/dlib-19.13.tar.bz2; +tar xf dlib-19.13.tar.bz2; +rm -r dlib-19.13.tar.bz2 +echo "dlib ready" \ No newline at end of file diff --git a/recompile.sh b/recompile.sh deleted file mode 100755 index 689d3d32e..000000000 --- a/recompile.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -#============================================================================== -# Title: install.sh -# Description: Install everything necessary for OpenFace to compile. -# Will install all required dependencies, only use if you do not have the dependencies -# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed -# Author: Daniyal Shahrokhian , Tadas Baltrusaitis -# Date: 20190630 -# Version : 1.03 -# Usage: bash install.sh -#============================================================================== - -# Exit script if any command fails -set -e -set -o pipefail - -if [ $# -ne 0 ] - then - echo "Usage: recompile.sh" - exit 1 -fi - -# Essential Dependencies -#echo "Installing Essential dependencies..." -#sudo apt-get -y update -#if [[ `lsb_release -rs` != "18.04" ]] -# then -# echo "Adding ppa:ubuntu-toolchain-r/test apt-repository " -# sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y -# sudo apt-get -y update -#fi -#sudo apt-get -y install build-essential -#sudo apt-get -y install gcc-8 g++-8 -# Ubuntu 16.04 does not have newest CMake so need to build it manually -#if [[ `lsb_release -rs` != "18.04" ]]; then -# sudo apt-get --purge remove cmake-qt-gui -y -# sudo apt-get --purge remove cmake -y -# mkdir -p cmake_tmp -# cd cmake_tmp -# wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz -# tar -xzvf cmake-3.10.1.tar.gz -# cd cmake-3.10.1/ -# ./bootstrap -# make -j4 -# sudo make install -# cd ../.. -# sudo rm -r cmake_tmp -#else -# sudo apt-get -y install cmake -#fi - -#sudo apt-get -y install zip -#sudo apt-get -y install libopenblas-dev liblapack-dev -#sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev -#sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev -#echo "Essential dependencies installed." - -# OpenCV Dependency -#echo "Downloading OpenCV..." -#wget https://github.com/opencv/opencv/archive/4.1.0.zip -#unzip 4.1.0.zip -cd opencv-4.1.0 -mkdir -p build -cd build -echo "Installing OpenCV..." -cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. -make -j7 -sudo make install -cd ../.. -#rm 4.1.0.zip -#sudo rm -r opencv-4.1.0 -echo "OpenCV installed." - -# dlib dependecy -echo "Downloading dlib" -#wget http://dlib.net/files/dlib-19.13.tar.bz2; -#tar xf dlib-19.13.tar.bz2; -cd dlib-19.13; -mkdir -p build; -cd build; -echo "Installing dlib" -cmake ..; -cmake --build . --config Release; -sudo make install; -sudo ldconfig; -cd ../..; -#rm -r dlib-19.13.tar.bz2 -echo "dlib installed" - -# OpenFace installation -echo "Installing OpenFace..." -mkdir -p build -cd build -cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. -make -cd .. -echo "OpenFace successfully installed." \ No newline at end of file From 2f944469017033fd47087a1540ccd222d1cae606 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 21:53:26 +0200 Subject: [PATCH 14/27] error fix on files removal --- README.md | 12 ++++-------- compile.sh | 2 +- prepare.sh | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d60451c5e..d38f72879 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,12 @@ part of the [ReVA-toolkit](https://github.com/numediart/ReVA-toolkit) -requires python dev libs: +If it is your first install, just run `install.sh`. -```bash -$ sudo apt-get install python-dev python3-dev -``` +If you need to debug or change configuration, use the scripts below: -- run `chmod +x prepare.sh && ./prepare.sh` : this will download all dependencies and install system libs -- run `chmod +x compile.sh && ./compile.sh` : this will compile everything - -[source](https://github.com/TadasBaltrusaitis/OpenFace/issues/23#issuecomment-229495315) +- run `chmod +x prepare.sh && ./prepare.sh` : download all dependencies and install system libs +- run `chmod +x compile.sh && ./compile.sh` : compile everything ## Description diff --git a/compile.sh b/compile.sh index 77af50e65..304d2fa1f 100755 --- a/compile.sh +++ b/compile.sh @@ -39,7 +39,7 @@ mkdir -p build; cd build; echo "Installing dlib" cmake -D CMAKE_BUILD_TYPE=RELEASE -D DLIB_IN_PROJECT_BUILD=ON -D DLIB_USE_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. -make -j7 +make -j4 sudo make install; sudo ldconfig; cd ../..; diff --git a/prepare.sh b/prepare.sh index 59563053d..03c8d8292 100755 --- a/prepare.sh +++ b/prepare.sh @@ -65,7 +65,7 @@ echo "Essential dependencies installed." echo "Downloading OpenCV..." wget https://github.com/opencv/opencv/archive/4.1.0.zip unzip 4.1.0.zip -sudo rm -r opencv-4.1.0 +rm -r 4.1.0.zip echo "OpenCV ready." # dlib dependecy From 072ea67a283492fd735d10dca9878f1f2233c384 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 22:13:08 +0200 Subject: [PATCH 15/27] fix cmake to ensure compatibility with python 3 --- CMakeLists.txt | 2 -- exe/FeatureExtractionPython/CMakeLists.txt | 6 ------ 2 files changed, 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b31cf3a9..af134324d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,8 +50,6 @@ else() MESSAGE("Boost not found in the system.") endif() -find_package(PythonLibs REQUIRED) - # Move LandmarkDetector model file(GLOB files "lib/local/LandmarkDetector/model/*.txt") foreach(file ${files}) diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 75d470e66..7a892c177 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -1,10 +1,4 @@ # Local libraries -#set(Python_ADDITIONAL_VERSIONS 3.5) -#find_package(PythonLibs 3 REQUIRED) -#find_package(Boost REQUIRED COMPONENTS python-py35) -#set(PYTHON_DOT_VERSION 3.5) -#set(PYTHON_INCLUDE /usr/include/python3.5) -#set(PYTHON_LIBRARY /usr/lib/python3.5/config-x86_64-linux-gnu) set(Python_ADDITIONAL_VERSIONS 3.5) find_package(PythonLibs 3 REQUIRED) From 56e48be29bed434cb3e7c3dda7d3277f18595d5c Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 22:46:53 +0200 Subject: [PATCH 16/27] upgrading cmake to version 3.15 for native python3 support --- CMakeLists.txt | 2 +- exe/FeatureExtractionPython/CMakeLists.txt | 7 +++++-- prepare.sh | 10 +++++++--- python_scripts/libPyOpenfaceVideo_tester.py | 21 +++++++++++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 python_scripts/libPyOpenfaceVideo_tester.py diff --git a/CMakeLists.txt b/CMakeLists.txt index af134324d..1566e1e46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.8) +cmake_minimum_required (VERSION 3.12) set(CMAKE_CXX_STANDARD 17) project(OpenFace VERSION 2.0.2) diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 7a892c177..945d0d305 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -1,7 +1,10 @@ # Local libraries -set(Python_ADDITIONAL_VERSIONS 3.5) -find_package(PythonLibs 3 REQUIRED) +find_package(Python3 REQUIRED) + +#set(Python_ADDITIONAL_VERSIONS 3.5) +#find_package(PythonLibs 3 REQUIRED) +#find_package(PythonInterp 3 REQUIRED) include_directories(${PYTHON_INCLUDE_DIRS}) link_directories(${PYTHON_LIBRARIES}) diff --git a/prepare.sh b/prepare.sh index 03c8d8292..33594476a 100755 --- a/prepare.sh +++ b/prepare.sh @@ -42,9 +42,13 @@ if [[ `lsb_release -rs` != "18.04" ]]; then sudo apt-get --purge remove cmake -y mkdir -p cmake_tmp cd cmake_tmp - wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz - tar -xzvf cmake-3.10.1.tar.gz - cd cmake-3.10.1/ + #wget https://cmake.org/files/v3.10/cmake-3.10.1.tar.gz + #tar -xzvf cmake-3.10.1.tar.gz + #cd cmake-3.10.1/ + #--> switching to cmake 3.15 for python3 + wget https://cmake.org/files/v3.15/cmake-3.15.0.tar.gz + tar -xzvf cmake-3.15.0.tar.gz + cd cmake-3.15.0/ ./bootstrap make -j4 sudo make install diff --git a/python_scripts/libPyOpenfaceVideo_tester.py b/python_scripts/libPyOpenfaceVideo_tester.py new file mode 100644 index 000000000..97238d982 --- /dev/null +++ b/python_scripts/libPyOpenfaceVideo_tester.py @@ -0,0 +1,21 @@ +import os, sys +CURRENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/build/exe/FeatureExtractionPython' +print(CURRENT_DIR) +sys.path.append(CURRENT_DIR) + +import time +#import keyboard +from libPyOpenfaceVideo import * + +ov = libPyOpenfaceVideo.OpenfaceVideo() +ov.start() + +while( ov.is_running() ): + + #if keyboard.is_pressed('q'): + # ov.stop() + + if ov.has_frame(): + print( ov.get_frame() ) + + time.sleep( 0.01 ) \ No newline at end of file From 6499462d51e9a4e26b75bf2bd0d0e9ba0ed7d4bd Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Wed, 28 Aug 2019 23:25:33 +0200 Subject: [PATCH 17/27] missing python dev libs + move find python3 to main cmake --- CMakeLists.txt | 11 +++++++++++ exe/FeatureExtractionPython/CMakeLists.txt | 15 ++++----------- prepare.sh | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1566e1e46..440ff7246 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,17 @@ else() MESSAGE(FATAL_ERROR "OpenCV not found in the system.") endif() +find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +if(${Python3_FOUND}) + MESSAGE("Python3 information:") + MESSAGE(" Python3_VERSION: ${Python3_VERSION}") + MESSAGE(" Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") + MESSAGE(" Python3_LIBRARIES: ${Python3_LIBRARIES}") + MESSAGE(" Python3_LIBRARY_DIRS: ${Python3_LIBRARY_DIRS}") +else() + MESSAGE("Python3 not found in the system.") +endif() + find_package( Boost 1.5.9 COMPONENTS filesystem system thread python REQUIRED) if(${Boost_FOUND}) MESSAGE("Boost information:") diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 945d0d305..83f146b8d 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -1,13 +1,6 @@ # Local libraries - -find_package(Python3 REQUIRED) - -#set(Python_ADDITIONAL_VERSIONS 3.5) -#find_package(PythonLibs 3 REQUIRED) -#find_package(PythonInterp 3 REQUIRED) - -include_directories(${PYTHON_INCLUDE_DIRS}) -link_directories(${PYTHON_LIBRARIES}) +include_directories(${Python3_INCLUDE_DIRS}) +link_directories(${Python3_LIBRARIES}) include_directories(${LandmarkDetector_SOURCE_DIR}/include) # EXEC @@ -17,7 +10,7 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include) #target_link_libraries(OpenfaceVideo GazeAnalyser) #target_link_libraries(OpenfaceVideo Utilities) #target_link_libraries(OpenfaceVideo ${Boost_LIBRARIES}) -#target_link_libraries(OpenfaceVideo ${PYTHON_LIBRARIES}) +#target_link_libraries(OpenfaceVideo ${Python3_LIBRARIES}) #install (TARGETS OpenfaceVideo DESTINATION bin) # LIBRARY @@ -28,5 +21,5 @@ target_link_libraries(PyOpenfaceVideo FaceAnalyser) target_link_libraries(PyOpenfaceVideo GazeAnalyser) target_link_libraries(PyOpenfaceVideo Utilities) target_link_libraries(PyOpenfaceVideo ${Boost_LIBRARIES}) -target_link_libraries(PyOpenfaceVideo ${PYTHON_LIBRARIES}) +target_link_libraries(PyOpenfaceVideo ${Python3_LIBRARIES}) install (TARGETS PyOpenfaceVideo DESTINATION bin) \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index 33594476a..71b408556 100755 --- a/prepare.sh +++ b/prepare.sh @@ -62,7 +62,7 @@ sudo apt-get -y install zip sudo apt-get -y install libopenblas-dev liblapack-dev sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev -sudo apt-get -y install python-dev python3-dev +sudo apt-get -y install python3-dev python3-dev python3-pip python3-tk python3-lxml python3-six echo "Essential dependencies installed." # OpenCV Dependency From 63e50c7c1a24228b66c9b00b6a45cfc6c3a5bf17 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 00:01:49 +0200 Subject: [PATCH 18/27] switching to boost 1.17.0 for python3 support --- .gitignore | 1 + CMakeLists.txt | 2 +- clear.sh | 20 ++++++++++++++++++++ compile.sh | 31 +++++++++++++++---------------- prepare.sh | 27 ++++++++++++--------------- 5 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 clear.sh diff --git a/.gitignore b/.gitignore index 2e7f998db..cd20cdf77 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,4 @@ opencv-3.4.0/* opencv-4.1.0/* dlib-19.13.tar.bz2 cmake_tmp/* +boost_1_71_0/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 440ff7246..0f7be3e1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,7 +50,7 @@ else() MESSAGE("Python3 not found in the system.") endif() -find_package( Boost 1.5.9 COMPONENTS filesystem system thread python REQUIRED) +find_package( Boost 1.71.0 COMPONENTS filesystem system thread python REQUIRED) if(${Boost_FOUND}) MESSAGE("Boost information:") MESSAGE(" Boost_VERSION: ${Boost_VERSION}") diff --git a/clear.sh b/clear.sh new file mode 100644 index 000000000..0b52293f5 --- /dev/null +++ b/clear.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Exit script if any command fails +set -e +set -o pipefail + +if [ $# -ne 0 ] + then + echo "Usage: recompile.sh" + exit 1 +fi + +rm -r boost_1_71_0.tar.bz2* +rm -r 4.1.0.zip* +rm -r dlib-19.13.tar.bz2* + +rm -r cmake_tmp +rm -r boost_1_71_0 +rm -r dlib-19.13 +rm -r opencv-4.1.0 \ No newline at end of file diff --git a/compile.sh b/compile.sh index 304d2fa1f..430032591 100755 --- a/compile.sh +++ b/compile.sh @@ -1,14 +1,4 @@ #!/bin/bash -#============================================================================== -# Title: install.sh -# Description: Install everything necessary for OpenFace to compile. -# Will install all required dependencies, only use if you do not have the dependencies -# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed -# Author: Daniyal Shahrokhian , Tadas Baltrusaitis -# Date: 20190630 -# Version : 1.03 -# Usage: bash install.sh -#============================================================================== # Exit script if any command fails set -e @@ -20,6 +10,15 @@ if [ $# -ne 0 ] exit 1 fi +# boost 1.71.0 Dependency (https://github.com/zpoint/Boost-Python-Examples) +echo "Installing boost 1.71.0..." +cd boost_1_71_0 +./bootstrap.sh --with-python=/usr/bin/python3 --with-python-version=3.5 --with-python-root=/usr/local/lib/python3.5 --prefix=/usr/local +sudo ./b2 install -a --with=all +sudo ldconfig +cd .. +echo "boost 1.71.0 installed." + # OpenCV Dependency cd opencv-4.1.0 mkdir -p build @@ -33,16 +32,16 @@ echo "OpenCV installed." # dlib dependecy echo "Downloading dlib" -cd dlib-19.13; +cd dlib-19.13 rm -rf build -mkdir -p build; -cd build; +mkdir -p build +cd build echo "Installing dlib" cmake -D CMAKE_BUILD_TYPE=RELEASE -D DLIB_IN_PROJECT_BUILD=ON -D DLIB_USE_CUDA=OFF -D BUILD_SHARED_LIBS=ON .. make -j4 -sudo make install; -sudo ldconfig; -cd ../..; +sudo make install +sudo ldconfig +cd ../.. echo "dlib installed" # OpenFace installation diff --git a/prepare.sh b/prepare.sh index 71b408556..dcc9ad9a2 100755 --- a/prepare.sh +++ b/prepare.sh @@ -1,14 +1,4 @@ #!/bin/bash -#============================================================================== -# Title: install.sh -# Description: Install everything necessary for OpenFace to compile. -# Will install all required dependencies, only use if you do not have the dependencies -# already installed or if you don't mind specific versions of gcc,g++,cmake,opencv etc. installed -# Author: Daniyal Shahrokhian , Tadas Baltrusaitis -# Date: 20190630 -# Version : 1.03 -# Usage: bash install.sh -#============================================================================== # Exit script if any command fails set -e @@ -63,18 +53,25 @@ sudo apt-get -y install libopenblas-dev liblapack-dev sudo apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev sudo apt-get -y install libtbb2 libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev sudo apt-get -y install python3-dev python3-dev python3-pip python3-tk python3-lxml python3-six -echo "Essential dependencies installed." +echo "######## Essential dependencies installed." + +# Install boost 1.71.0 +echo "######## Downloading Boost..." +wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.bz2 +tar xf boost_1_71_0.tar.bz2 +rm -r boost_1_71_0.tar.bz2 +echo "######## Boost ready." # OpenCV Dependency -echo "Downloading OpenCV..." +echo "######## Downloading OpenCV..." wget https://github.com/opencv/opencv/archive/4.1.0.zip unzip 4.1.0.zip rm -r 4.1.0.zip -echo "OpenCV ready." +echo "######## OpenCV ready." # dlib dependecy -echo "Downloading dlib" +echo "######## Downloading dlib" wget http://dlib.net/files/dlib-19.13.tar.bz2; tar xf dlib-19.13.tar.bz2; rm -r dlib-19.13.tar.bz2 -echo "dlib ready" \ No newline at end of file +echo "######## dlib ready" \ No newline at end of file From 419ecc2eea5e8327eddb1c3df604091e5995862f Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 00:02:48 +0200 Subject: [PATCH 19/27] switching to boost 1.17.0 for python3 support --- clear.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 clear.sh diff --git a/clear.sh b/clear.sh old mode 100644 new mode 100755 index 0b52293f5..b003c77b2 --- a/clear.sh +++ b/clear.sh @@ -10,11 +10,11 @@ if [ $# -ne 0 ] exit 1 fi -rm -r boost_1_71_0.tar.bz2* -rm -r 4.1.0.zip* -rm -r dlib-19.13.tar.bz2* +rm -rf boost_1_71_0.tar.bz2* +rm -rf 4.1.0.zip* +rm -rf dlib-19.13.tar.bz2* -rm -r cmake_tmp -rm -r boost_1_71_0 -rm -r dlib-19.13 -rm -r opencv-4.1.0 \ No newline at end of file +rm -rf cmake_tmp +rm -rf boost_1_71_0 +rm -rf dlib-19.13 +rm -rf opencv-4.1.0 \ No newline at end of file From dbebf12802fa294283b489eb1e6c9215272913c5 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 00:09:10 +0200 Subject: [PATCH 20/27] clear script --- README.md | 7 +++++-- clear.sh | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d38f72879..1e5c3130b 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,11 @@ If it is your first install, just run `install.sh`. If you need to debug or change configuration, use the scripts below: -- run `chmod +x prepare.sh && ./prepare.sh` : download all dependencies and install system libs -- run `chmod +x compile.sh && ./compile.sh` : compile everything +- `bash download_models.sh` : download and install models +- `bash prepare.sh` : download all dependencies and install system libs +- `bash compile.sh` : compile everything + +For a hard reset of the folder, use `bash clear.sh` to remove boost, opencv, dlib and build folders. ## Description diff --git a/clear.sh b/clear.sh index b003c77b2..4a138e4ea 100755 --- a/clear.sh +++ b/clear.sh @@ -14,7 +14,8 @@ rm -rf boost_1_71_0.tar.bz2* rm -rf 4.1.0.zip* rm -rf dlib-19.13.tar.bz2* -rm -rf cmake_tmp -rm -rf boost_1_71_0 -rm -rf dlib-19.13 -rm -rf opencv-4.1.0 \ No newline at end of file +sudo rm -rf cmake_tmp +sudo rm -rf boost_1_71_0 +sudo rm -rf dlib-19.13 +sudo rm -rf opencv-4.1.0 +sudo rm -rf build From 0c298c46ca6748077ede66e04ec95e1719a79e60 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 01:07:57 +0200 Subject: [PATCH 21/27] correct compilation of boost with python3 support --- .gitignore | 1 + CMakeLists.txt | 27 +++++++++++++++++++++++++++ README.md | 3 ++- compile.sh => compile_deps.sh | 17 ++++++----------- compile_openface.sh | 20 ++++++++++++++++++++ install.sh | 3 ++- 6 files changed, 58 insertions(+), 13 deletions(-) rename compile.sh => compile_deps.sh (76%) create mode 100755 compile_openface.sh diff --git a/.gitignore b/.gitignore index cd20cdf77..23b39aa52 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ opencv-4.1.0/* dlib-19.13.tar.bz2 cmake_tmp/* boost_1_71_0/* +processed/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f7be3e1d..d8605573c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,28 @@ else() MESSAGE("Python3 not found in the system.") endif() +#set(Python_ADDITIONAL_VERSIONS 3.5) +#find_package(PythonLibs 3 REQUIRED) +#if(${PythonLibs_FOUND}) +# MESSAGE("PythonLibs information:") +# MESSAGE(" PythonLibs_VERSION: ${PythonLibs_VERSION}") +# MESSAGE(" PythonLibs_INCLUDE_DIRS: ${PythonLibs_INCLUDE_DIRS}") +# MESSAGE(" PythonLibs_LIBRARIES: ${PythonLibs_LIBRARIES}") +# MESSAGE(" PythonLibs_LIBRARY_DIRS: ${PythonLibs_LIBRARY_DIRS}") +#else() +# MESSAGE("PythonLibs not found in the system.") +#endif() +#find_package(PythonInterp 3 REQUIRED) +#if(${PythonInterp_FOUND}) +# MESSAGE("PythonInterp information:") +# MESSAGE(" PythonInterp_VERSION: ${PythonInterp_VERSION}") +# MESSAGE(" PythonInterp_INCLUDE_DIRS: ${PythonInterp_INCLUDE_DIRS}") +# MESSAGE(" PythonInterp_LIBRARIES: ${PythonInterp_LIBRARIES}") +# MESSAGE(" PythonInterp_LIBRARY_DIRS: ${PythonInterp_LIBRARY_DIRS}") +#else() +# MESSAGE("PythonInterp not found in the system.") +#endif() + find_package( Boost 1.71.0 COMPONENTS filesystem system thread python REQUIRED) if(${Boost_FOUND}) MESSAGE("Boost information:") @@ -249,6 +271,11 @@ if(CMAKE_CURRENT_SOURCE_DIR STREQUAL "${CMAKE_SOURCE_DIR}") DESTINATION ${CONFIG_DEST_DIR}) endif() +#get_cmake_property(_variableNames VARIABLES) +#foreach (_variableName ${_variableNames}) +# message(STATUS "${_variableName}=${${_variableName}}") +#endforeach() + # executables add_subdirectory(exe/FaceLandmarkImg) add_subdirectory(exe/FaceLandmarkVid) diff --git a/README.md b/README.md index 1e5c3130b..7b0b82aea 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ If you need to debug or change configuration, use the scripts below: - `bash download_models.sh` : download and install models - `bash prepare.sh` : download all dependencies and install system libs -- `bash compile.sh` : compile everything +- `bash compile_deps.sh` : compile boost, opencv & dlib +- `bash compile_openface.sh` : compile openface For a hard reset of the folder, use `bash clear.sh` to remove boost, opencv, dlib and build folders. diff --git a/compile.sh b/compile_deps.sh similarity index 76% rename from compile.sh rename to compile_deps.sh index 430032591..2f1d030c7 100755 --- a/compile.sh +++ b/compile_deps.sh @@ -6,13 +6,16 @@ set -o pipefail if [ $# -ne 0 ] then - echo "Usage: recompile.sh" + echo "Usage: compile_deps.sh" exit 1 fi # boost 1.71.0 Dependency (https://github.com/zpoint/Boost-Python-Examples) echo "Installing boost 1.71.0..." cd boost_1_71_0 +echo "using mpi ; +using gcc : : g++ ; +using python : 3.5 : /usr/bin/python3 : /usr/include/python3.5m : /usr/local/lib ;" > ~/user-config.jam ./bootstrap.sh --with-python=/usr/bin/python3 --with-python-version=3.5 --with-python-root=/usr/local/lib/python3.5 --prefix=/usr/local sudo ./b2 install -a --with=all sudo ldconfig @@ -28,6 +31,7 @@ cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB make -j7 sudo make install cd ../.. +sudo ln -s /usr/lib/x86_64-linux-gnu/libboost_python-py35.so /usr/lib/x86_64-linux-gnu/libboost_python3.so echo "OpenCV installed." # dlib dependecy @@ -42,13 +46,4 @@ make -j4 sudo make install sudo ldconfig cd ../.. -echo "dlib installed" - -# OpenFace installation -echo "Installing OpenFace..." -mkdir -p build -cd build -cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. -make -cd .. -echo "OpenFace successfully installed." +echo "dlib installed" \ No newline at end of file diff --git a/compile_openface.sh b/compile_openface.sh new file mode 100755 index 000000000..d99d65246 --- /dev/null +++ b/compile_openface.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Exit script if any command fails +set -e +set -o pipefail + +if [ $# -ne 0 ] + then + echo "Usage: compile_openface.sh" + exit 1 +fi + +# OpenFace installation +echo "Installing OpenFace..." +mkdir -p build +cd build +cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. +make +cd .. +echo "OpenFace successfully installed." diff --git a/install.sh b/install.sh index 1b1136870..b32539208 100755 --- a/install.sh +++ b/install.sh @@ -2,4 +2,5 @@ bash download_models.sh bash prepare.sh -bash compile.sh \ No newline at end of file +bash compile_deps.sh +bash compile_openface.sh \ No newline at end of file From 0c6a18a66a7787788237b88fc82b4867cae39a34 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 03:16:22 +0200 Subject: [PATCH 22/27] fix on c++ class and functional python script --- README.md | 2 +- compile_openface.sh | 5 +- exe/FeatureExtractionPython/CMakeLists.txt | 1 + .../FeatureExtractionPython.cpp | 136 +++++++++++++----- .../src/LandmarkDetectorParameters.cpp | 16 ++- python_scripts/PyOpenfaceVideo_tester.py | 39 +++++ python_scripts/libPyOpenfaceVideo_tester.py | 21 --- 7 files changed, 155 insertions(+), 65 deletions(-) create mode 100644 python_scripts/PyOpenfaceVideo_tester.py delete mode 100644 python_scripts/libPyOpenfaceVideo_tester.py diff --git a/README.md b/README.md index 7b0b82aea..f2cbb6cf7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenFace 2.2.0: a facial behavior analysis toolkit -*Modified version of [original repository](https://github.com/TadasBaltrusaitis/OpenFace) to allow usage of webcam feature extraction from python. This is NOT a stable version.* +*Modified version of [original repository](https://github.com/TadasBaltrusaitis/OpenFace) to allow usage of webcam feature extraction from python, developped and tested on Linux Mint 18.2 Cinnamon 64bits.* part of the [ReVA-toolkit](https://github.com/numediart/ReVA-toolkit) diff --git a/compile_openface.sh b/compile_openface.sh index d99d65246..7d57b7f40 100755 --- a/compile_openface.sh +++ b/compile_openface.sh @@ -16,5 +16,8 @@ mkdir -p build cd build cmake -D CMAKE_CXX_COMPILER=g++-8 -D CMAKE_C_COMPILER=gcc-8 -D CMAKE_BUILD_TYPE=RELEASE .. make -cd .. +# copying models in FeatureExtractionPython folder +cp -r bin/model/ exe/FeatureExtractionPython/model/ +mkdir -p exe/FeatureExtractionPython/classifiers/ +cp ../lib/3rdParty/OpenCV/classifiers/haarcascade_frontalface_alt.xml exe/FeatureExtractionPython/classifiers/ echo "OpenFace successfully installed." diff --git a/exe/FeatureExtractionPython/CMakeLists.txt b/exe/FeatureExtractionPython/CMakeLists.txt index 83f146b8d..808cd5566 100644 --- a/exe/FeatureExtractionPython/CMakeLists.txt +++ b/exe/FeatureExtractionPython/CMakeLists.txt @@ -22,4 +22,5 @@ target_link_libraries(PyOpenfaceVideo GazeAnalyser) target_link_libraries(PyOpenfaceVideo Utilities) target_link_libraries(PyOpenfaceVideo ${Boost_LIBRARIES}) target_link_libraries(PyOpenfaceVideo ${Python3_LIBRARIES}) +set_target_properties(PyOpenfaceVideo PROPERTIES PREFIX "") install (TARGETS PyOpenfaceVideo DESTINATION bin) \ No newline at end of file diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 617d5cd78..5134fdc6e 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -271,6 +271,8 @@ int OpenfaceVideoWorker( } + return 0; + } struct OpenfaceV3 { @@ -319,6 +321,7 @@ typedef std::vector OpenfaceV3Vector; struct OpenfaceFrame { + uint32_t fID; bool updated; OpenfaceV3 gaze_0; OpenfaceV3 gaze_1; @@ -326,7 +329,7 @@ struct OpenfaceFrame { OpenfaceV3 head_pos; std::vector landmarks; - OpenfaceFrame(): updated(false) { + OpenfaceFrame(): updated(false), fID(0) { landmarks.resize(LANDMARKS_NUM); } @@ -339,6 +342,7 @@ struct OpenfaceFrame { void print() { std::cout << "OpenfaceFrame" << std::endl << + "\t" << "fID: " << fID << std::endl << "\t" << "updated: " << updated << std::endl << "\t" << "gaze_0: " << gaze_0.x << ", " << gaze_0.y << ", " << gaze_0.z << std::endl << "\t" << "gaze_1: " << gaze_1.x << ", " << gaze_1.y << ", " << gaze_1.z << std::endl << @@ -352,6 +356,7 @@ struct OpenfaceFrame { bool operator == ( const OpenfaceFrame& src ) { if ( + fID != src.fID || gaze_0 != src.gaze_0 || gaze_1 != src.gaze_1 || head_rot != src.head_rot || @@ -367,6 +372,7 @@ struct OpenfaceFrame { bool operator != ( const OpenfaceFrame& src ) { if ( + fID != src.fID || gaze_0 != src.gaze_0 || gaze_1 != src.gaze_1 || head_rot != src.head_rot || @@ -381,6 +387,7 @@ struct OpenfaceFrame { } void operator = ( const OpenfaceFrame& src ) { + fID = src.fID; updated = src.updated; gaze_0 = src.gaze_0; gaze_1 = src.gaze_1; @@ -397,6 +404,8 @@ class OpenfaceVideo { public: + OpenfaceFrame public_frame; + OpenfaceVideo() : worker(0) {} OpenfaceVideo( const OpenfaceVideo& src ) : worker(0) { @@ -413,27 +422,43 @@ class OpenfaceVideo { arguments.push_back(std::string(argv[i])); } //append_argument( "-2Dfp", "" ); - append_argument( "-3Dfp", "" ); - append_argument( "-pose", "" ); - append_argument( "-gaze", "" ); - append_argument( "-device", "0" ); + append_argument( "-3Dfp", "", false ); + append_argument( "-pose", "", false ); + append_argument( "-gaze", "", false ); + append_argument( "-device", "0", false ); } + int get_device() { + return atoi( get_argument( "-device" ).c_str() ); + } + void set_device( int d ) { - stop(); - - if ( arguments.empty() ) { - load_arguments(0,0); - } - for ( int i = 0, imax = arguments.size(); i < imax; ++i ) { - if ( arguments[i].compare( "-device" ) == 0 ) { - arguments[i+1] = std::to_string(d); - return; - } - } - append_argument( "-device", std::to_string(d) ); - + append_argument( "-device", std::to_string(d), true ); + } + + std::string get_landmark_model() { + return get_argument( "-mloc" ); + } + + void set_landmark_model( std::string path ) { + append_argument( "-mloc", path, true ); + } + + std::string get_HAAR() { + return get_argument( "-fdloc" ); + } + + void set_HAAR( const char* path ) { + append_argument( "-fdloc", std::string( path ), true ); + } + + std::string get_MTCNN() { + return get_argument( "-mtcnnloc" ); + } + + void set_MTCNN( const char* path ) { + append_argument( "-mtcnnloc", std::string( path ), true ); } bool start() { @@ -468,22 +493,21 @@ class OpenfaceVideo { const cv::Mat_& l3d = rec.get_landmarks_3D(); cv::Size s = l3d.size(); - std::cout << "new_frame " << rec.GetCSVFile() << ", rows: " << s.height << ", cols: " << s.width << std::endl; frame.gaze_0 = rec.get_gaze_direction(0); frame.gaze_1 = rec.get_gaze_direction(1); cv::Vec6f h = rec.get_head_pose(); - frame.head_rot.set( h[0], h[1], h[2] ); - frame.head_pos.set( h[3], h[4], h[5] ); + frame.head_pos.set( h[0], h[1], h[2] ); + frame.head_rot.set( h[3], h[4], h[5] ); for ( int c = 0; c < s.width; ++c ) { - frame.landmarks[c].x = l3d.at( c,0 ); - frame.landmarks[c].y = l3d.at( c,1 ); - frame.landmarks[c].z = l3d.at( c,2 ); + frame.landmarks[c].x = l3d.at( 0,c ); + frame.landmarks[c].y = l3d.at( 1,c ); + frame.landmarks[c].z = l3d.at( 2,c ); } - frame.updated = false; - frame.print(); + frame.fID++; + frame.updated = true; } @@ -500,10 +524,15 @@ class OpenfaceVideo { bool is_running() const { return worker != 0; } - - bool has_frame() const { + + bool has_frame() { boost::unique_lock< boost::shared_mutex > lock(_access); - return frame.updated; + if ( frame.updated ) { + public_frame = frame; + frame.updated = false; + return true; + } + return false; } OpenfaceFrame get_frame() { @@ -514,14 +543,26 @@ class OpenfaceVideo { private: - void append_argument( std::string a, std::string value ) { + void append_argument( std::string a, std::string value, bool empty_test ) { + + if ( empty_test && arguments.empty() ) { + load_arguments(0,0); + } + std::vector::iterator it = arguments.begin(); std::vector::iterator ite = arguments.end(); bool found = false; for ( ; it != ite; ++it ) { if ( (*it).compare( a ) == 0 ) { - std::cout << "arg " << a << " is already there" << std::endl; - found = true; break; + ++it; + if ( value.length() > 0 || it == ite ) { + std::cout << "arg " << a << " is already at " << value << std::endl; + found = true; break; + } else { + (*it) = value; + std::cout << "arg " << a << " > " << value << std::endl; + found = true; break; + } } } if ( !found ) { @@ -532,6 +573,21 @@ class OpenfaceVideo { } } + std::string get_argument( std::string a ) { + std::vector::iterator it = arguments.begin(); + std::vector::iterator ite = arguments.end(); + for ( ; it != ite; ++it ) { + if ( (*it).compare( a ) == 0 ) { + ++it; + if ( it == ite ) { + return ""; + } + return (*it); + } + } + return ""; + } + std::vector arguments; bool thread_running; bool _data_updated; @@ -597,6 +653,7 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { .def(vector_indexing_suite()); class_("OpenfaceFrame") + .def_readonly("ID", &OpenfaceFrame::fID) .def_readonly("gaze_0", &OpenfaceFrame::gaze_0) .def_readonly("gaze_1", &OpenfaceFrame::gaze_1) .def_readonly("head_rot", &OpenfaceFrame::head_rot) @@ -604,12 +661,15 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { .def_readonly("landmarks", &OpenfaceFrame::landmarks); class_("OpenfaceVideo") - .add_property("set_device", &OpenfaceVideo::set_device) - .add_property("start", &OpenfaceVideo::start) - .add_property("stop", &OpenfaceVideo::stop) - .add_property("is_running", &OpenfaceVideo::is_running) - .add_property("has_frame", &OpenfaceVideo::has_frame) - .add_property("get_frame", &OpenfaceVideo::get_frame); + .add_property("device", &OpenfaceVideo::get_device, &OpenfaceVideo::set_device) + .add_property("landmark_model", &OpenfaceVideo::get_landmark_model, &OpenfaceVideo::set_landmark_model) + .add_property("HAAR", &OpenfaceVideo::get_HAAR, &OpenfaceVideo::set_HAAR) + .add_property("MTCNN", &OpenfaceVideo::get_MTCNN, &OpenfaceVideo::set_MTCNN) + .def("start", &OpenfaceVideo::start) + .def("stop", &OpenfaceVideo::stop) + .def("is_running", &OpenfaceVideo::is_running) + .def("has_frame", &OpenfaceVideo::has_frame) + .def_readonly("frame", &OpenfaceVideo::public_frame); } int main(int argc, char **argv) { diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 960f73641..8064f7e65 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -88,6 +88,14 @@ FaceModelParameters::FaceModelParameters(std::vector &arguments) valid[i + 1] = false; i++; } + if (arguments[i].compare("-mtcnnloc") ==0) + { + std::string mtcnn_face_detector_loc = arguments[i + 1]; + mtcnn_face_detector_location = mtcnn_face_detector_loc; + valid[i] = false; + valid[i + 1] = false; + i++; + } if (arguments[i].compare("-sigma") == 0) { std::stringstream data(arguments[i + 1]); @@ -192,7 +200,7 @@ FaceModelParameters::FaceModelParameters(std::vector &arguments) } else { - std::cout << "Could not find the landmark detection model to load" << std::endl; + std::cout << "Could not find the landmark detection model to load " << model_path << std::endl; } if (model_path.stem().string().compare("main_ceclm_general") == 0) @@ -227,7 +235,7 @@ FaceModelParameters::FaceModelParameters(std::vector &arguments) } else { - std::cout << "Could not find the HAAR face detector location" << std::endl; + std::cout << "Could not find the HAAR face detector location " << haar_face_detector_location << std::endl; } // Make sure face detector location is valid @@ -247,7 +255,7 @@ FaceModelParameters::FaceModelParameters(std::vector &arguments) } else { - std::cout << "Could not find the MTCNN face detector location" << std::endl; + std::cout << "Could not find the MTCNN face detector location " << mtcnn_face_detector_location << std::endl; } check_model_path(root.string()); } @@ -274,7 +282,7 @@ void FaceModelParameters::check_model_path(const std::string& root) } else { - std::cout << "Could not find the landmark detection model to load" << std::endl; + std::cout << "Could not find the landmark detection model to load " << model_location << std::endl; } } diff --git a/python_scripts/PyOpenfaceVideo_tester.py b/python_scripts/PyOpenfaceVideo_tester.py new file mode 100644 index 000000000..6bb6e8757 --- /dev/null +++ b/python_scripts/PyOpenfaceVideo_tester.py @@ -0,0 +1,39 @@ +import os, sys +OFV_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/build/exe/FeatureExtractionPython' +sys.path.append(OFV_DIR) + +import time +import keyboard +from PyOpenfaceVideo import * + +ov = OpenfaceVideo() + +ov.device = 0 + +# set model path: mandatory to start tracking +ov.landmark_model = OFV_DIR + "/model/main_ceclm_general.txt" +ov.HAAR = OFV_DIR + "/classifiers/haarcascade_frontalface_alt.xml" +ov.MTCNN = OFV_DIR + "/model/mtcnn_detector/MTCNN_detector.txt" + +# start tracking +ov.start() + +while( ov.is_running() ): + + try: + if keyboard.is_pressed('q'): + ov.stop() + except: + pass + + if ov.has_frame(): + print( "new frame: ", ov.frame.ID ) + print( "\tgaze_0: ", ov.frame.gaze_0.x, ',', ov.frame.gaze_0.y, ',', ov.frame.gaze_0.z ) + print( "\tgaze_1: ", ov.frame.gaze_1.x, ',', ov.frame.gaze_1.y, ',', ov.frame.gaze_1.z ) + print( "\thead_rot: ", ov.frame.head_rot.x, ',', ov.frame.head_rot.y, ',', ov.frame.head_rot.z ) + print( "\thead_pos: ", ov.frame.head_pos.x, ',', ov.frame.head_pos.y, ',', ov.frame.head_pos.z ) + print( "\tlandmarks: " ) + for i in range( len(ov.frame.landmarks) ): + print( "\t\t", i, ':', ov.frame.landmarks[i].x, ',', ov.frame.landmarks[i].y, ',', ov.frame.landmarks[i].z ) + + time.sleep( 0.01 ) \ No newline at end of file diff --git a/python_scripts/libPyOpenfaceVideo_tester.py b/python_scripts/libPyOpenfaceVideo_tester.py deleted file mode 100644 index 97238d982..000000000 --- a/python_scripts/libPyOpenfaceVideo_tester.py +++ /dev/null @@ -1,21 +0,0 @@ -import os, sys -CURRENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/build/exe/FeatureExtractionPython' -print(CURRENT_DIR) -sys.path.append(CURRENT_DIR) - -import time -#import keyboard -from libPyOpenfaceVideo import * - -ov = libPyOpenfaceVideo.OpenfaceVideo() -ov.start() - -while( ov.is_running() ): - - #if keyboard.is_pressed('q'): - # ov.stop() - - if ov.has_frame(): - print( ov.get_frame() ) - - time.sleep( 0.01 ) \ No newline at end of file From 5d22d21f5903b747c36a2d0c7f8218ebb2cc0c43 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 03:19:54 +0200 Subject: [PATCH 23/27] python script ref --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f2cbb6cf7..d0c03190c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,12 @@ If you need to debug or change configuration, use the scripts below: For a hard reset of the folder, use `bash clear.sh` to remove boost, opencv, dlib and build folders. +Demo: got to **python_scripts** and launch: + +```bash +python3 PyOpenfaceVideo_tester.py +``` + ## Description Over the past few years, there has been an increased interest in automatic facial behavior analysis From 4a8892deb585ca0621ea3e58c208b8e31826f28e Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 16:27:49 +0200 Subject: [PATCH 24/27] huge sleep at the end of the script removed --- exe/FeatureExtractionPython/FeatureExtractionPython.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 5134fdc6e..808295ed0 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -514,7 +514,7 @@ class OpenfaceVideo { bool stop() { if ( thread_running ) { thread_running = false; - usleep( 1000 ); + usleep( 0.1 ); worker = 0; return true; } From 0ef7a5e9bea1f9047e921b0e3bc505420fefa990 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Thu, 29 Aug 2019 17:08:02 +0200 Subject: [PATCH 25/27] stopping thread is now efficient --- .../FeatureExtractionPython.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index 808295ed0..dc3db2e3c 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -211,8 +211,7 @@ int OpenfaceVideoWorker( char character_press = visualizer.ShowObservation(); // quit processing the current sequence (useful when in Webcam mode) - if (character_press == 'q') - { + if ( !running || character_press == 'q') { break; } @@ -254,22 +253,26 @@ int OpenfaceVideoWorker( open_face_rec.Close(); INFO_STREAM("Closing input reader"); sequence_reader.Close(); - INFO_STREAM("Closed successfully"); + /* if (recording_params.outputAUs()) { INFO_STREAM("Postprocessing the Action Unit predictions"); face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); - } + }*/ // Reset the models for the next video face_analyser.Reset(); face_model.Reset(); - - thread_group.remove_thread(this_thread); - delete this_thread; } + + thread_group.remove_thread(this_thread); + delete this_thread; + + this_thread = 0; + + INFO_STREAM("Closed successfully"); return 0; @@ -514,9 +517,11 @@ class OpenfaceVideo { bool stop() { if ( thread_running ) { thread_running = false; - usleep( 0.1 ); + boost::this_thread::sleep(boost::posix_time::milliseconds(10)); worker = 0; + std::cout << "OpenfaceVideo successfully closed" << std::endl; return true; + } return false; } @@ -536,9 +541,7 @@ class OpenfaceVideo { } OpenfaceFrame get_frame() { - boost::unique_lock< boost::shared_mutex > lock(_access); - frame.updated = false; - return frame; + return public_frame; } private: From 10aecde96e98cdd921cc65032432628cf784e57f Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Mon, 2 Sep 2019 01:25:46 +0200 Subject: [PATCH 26/27] info retrieval via callback (much more efficient) --- .../FeatureExtractionPython.cpp | 410 ++++++++++++------ .../src/FaceAnalyserParameters.cpp | 31 +- python_scripts/PyOpenfaceVideo_tester.py | 33 +- 3 files changed, 315 insertions(+), 159 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index dc3db2e3c..ea48e2186 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -33,6 +33,7 @@ /////////////////////////////////////////////////////////////////////////////// // tuned by frankiezafe, http://polymorph.cool +// https://stackoverflow.com/questions/41240610/how-to-call-python-from-a-boost-thread // FeatureExtraction.cpp : Defines the entry point for the feature extraction console application. @@ -44,6 +45,7 @@ #include #include #include +#include // Local includes #include "LandmarkCoreIncludes.h" @@ -83,7 +85,7 @@ printErrorAndAbort( std::string( "Fatal error: " ) + stream ) boost::shared_mutex _access; int OpenfaceVideoWorker( - std::vector& args, + std::vector& arguments, const bool& running, boost::function callback, boost::thread_group & thread_group, @@ -91,101 +93,69 @@ int OpenfaceVideoWorker( // Load the modules that are being used for tracking and face analysis // Load face landmark detector - LandmarkDetector::FaceModelParameters det_parameters(args); + LandmarkDetector::FaceModelParameters det_parameters(arguments); // Always track gaze in feature extraction LandmarkDetector::CLNF face_model(det_parameters.model_location); - - if (!face_model.loaded_successfully) - { + if (!face_model.loaded_successfully) { std::cout << "ERROR: Could not load the landmark detector" << std::endl; return 1; } - // Load facial feature extractor and AU analyser - FaceAnalysis::FaceAnalyserParameters face_analysis_params(args); + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - - if (!face_model.eye_model) - { + if (!face_model.eye_model) { std::cout << "WARNING: no eye model found" << std::endl; } - - if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0) - { + if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0) { std::cout << "WARNING: no Action Unit models found" << std::endl; } - Utilities::SequenceCapture sequence_reader; - // A utility for visualizing the results - Utilities::Visualizer visualizer(args); - + Utilities::Visualizer visualizer(arguments); // Tracking FPS for visualization Utilities::FpsTracker fps_tracker; fps_tracker.AddFrame(); - - while ( running ) // this is not a for loop as we might also be reading from a webcam - { + while ( running ) { // The sequence reader chooses what to open based on command line arguments provided - if (!sequence_reader.Open(args)) + if (!sequence_reader.Open(arguments)) break; - INFO_STREAM("Device or file opened"); - - if (sequence_reader.IsWebcam()) - { + if (sequence_reader.IsWebcam()) { INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); visualizer.vis_track = true; } - cv::Mat captured_image; - - Utilities::RecorderOpenFaceParameters recording_params(args, true, sequence_reader.IsWebcam(), + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); - if (!face_model.eye_model) - { + if (!face_model.eye_model) { recording_params.setOutputGaze(false); } - - Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, args); - + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); if (recording_params.outputGaze() && !face_model.eye_model) std::cout << "WARNING: no eye model defined, but outputting gaze" << std::endl; - captured_image = sequence_reader.GetNextFrame(); - // For reporting progress double reported_completion = 0; - INFO_STREAM("Starting tracking"); - while (!captured_image.empty()) - { + while (!captured_image.empty()) { // Converting to grayscale cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); - - // The actual facial landmark detection / tracking bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters, grayscale_image); - // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); - - if (detection_success && face_model.eye_model) - { + if (detection_success && face_model.eye_model) { GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); } - // Do face alignment cv::Mat sim_warped_img; cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; - // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization - if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus) - { + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus) { face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); @@ -396,11 +366,100 @@ struct OpenfaceFrame { gaze_1 = src.gaze_1; head_rot = src.head_rot; head_pos = src.head_pos; + landmarks.reserve(src.landmarks.size()); + copy(src.landmarks.begin(), src.landmarks.end(), landmarks.begin()); + } + + boost::python::dict to_python() { + + boost::python::dict pyframe; + + boost::python::list g0; + boost::python::list g1; + boost::python::list hr; + boost::python::list hp; + boost::python::list ls; + + g0.append( gaze_0.x ); + g0.append( gaze_0.y ); + g0.append( gaze_0.z ); + g1.append( gaze_1.x ); + g1.append( gaze_1.y ); + g1.append( gaze_1.z ); + hr.append( head_rot.x ); + hr.append( head_rot.y ); + hr.append( head_rot.z ); + hp.append( head_pos.x ); + hp.append( head_pos.y ); + hp.append( head_pos.z ); for ( int i = 0; i < LANDMARKS_NUM; ++i ) { - landmarks[i] = src.landmarks[i]; + boost::python::list vs; + vs.append( landmarks[i].x ); + vs.append( landmarks[i].y ); + vs.append( landmarks[i].z ); + ls.append( vs ); } + + pyframe["ID"] = fID; + pyframe["gaze_0"] = g0; + pyframe["gaze_1"] = g1; + pyframe["head_rot"] = hr; + pyframe["head_pos"] = hp; + pyframe["landmarks"] = ls; + + return pyframe; + + } + +}; + +class with_gil { +public: + with_gil() { state_ = PyGILState_Ensure(); } + ~with_gil() { PyGILState_Release(state_); } + with_gil(const with_gil&) = delete; + with_gil& operator=(const with_gil&) = delete; +private: + PyGILState_STATE state_; +}; + +class py_callable { +public: + + /// @brief Constructor that assumes the caller has the GIL locked. + py_callable(const boost::python::object& object) { + with_gil gil; + object_.reset( + // GIL locked, so it is safe to copy. + new boost::python::object{object}, + // Use a custom deleter to hold GIL when the object is deleted. + [](boost::python::object* object) { + with_gil gil; + delete object; + }); } + // Use default copy-constructor and assignment-operator. + py_callable(const py_callable&) = default; + py_callable& operator=(const py_callable&) = default; + + template + void operator()(Args... args) { + // Lock the GIL as the python object is going to be invoked. + with_gil gil; + (*object_)(std::forward(args)...); + } + + template + void call(Args... args) { + // Lock the GIL as the python object is going to be invoked. + with_gil gil; + (*object_)(std::forward(args)...); + } + +private: + std::shared_ptr object_; + }; class OpenfaceVideo { @@ -409,15 +468,18 @@ class OpenfaceVideo { OpenfaceFrame public_frame; - OpenfaceVideo() : worker(0) {} + OpenfaceVideo() : pycall(0) {} - OpenfaceVideo( const OpenfaceVideo& src ) : worker(0) { + OpenfaceVideo( const OpenfaceVideo& src ) : pycall(0) { std::cout << "OpenfaceVideo copy cstr is not implemented" << std::endl; } ~OpenfaceVideo() { // clean stop stop(); + if ( pycall != 0 ) { + delete pycall; + } } void load_arguments(int argc, char **argv) { @@ -464,46 +526,160 @@ class OpenfaceVideo { append_argument( "-mtcnnloc", std::string( path ), true ); } + std::string get_AU() { + return get_argument( "-au" ); + } + + void set_AU( const char* path ) { + append_argument( "-au", std::string( path ), true ); + } + bool start() { - if ( arguments.empty() ) { load_arguments(0,0); } - - stop(); - thread_running = true; - worker = new boost::thread(); - boost::function callback = boost::bind( &OpenfaceVideo::new_frame, this, _1 ); - *worker = boost::thread( - boost::bind( - &OpenfaceVideoWorker, - arguments, - thread_running, - callback, - boost::ref(openfacevideo_threads), - worker - ) - ); - openfacevideo_threads.add_thread(worker); - + // quite important to set to true! + _running = true; + // Load the modules that are being used for tracking and face analysis + // Load face landmark detector + LandmarkDetector::FaceModelParameters det_parameters(arguments); + // Always track gaze in feature extraction + LandmarkDetector::CLNF face_model(det_parameters.model_location); + if (!face_model.loaded_successfully) { + std::cout << "ERROR: Could not load the landmark detector" << std::endl; + return 1; + } + // Load facial feature extractor and AU analyser + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); + if (!face_model.eye_model) { + std::cout << "WARNING: no eye model found" << std::endl; + } + if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0) { + std::cout << "WARNING: no Action Unit models found" << std::endl; + } + Utilities::SequenceCapture sequence_reader; + // A utility for visualizing the results + Utilities::Visualizer visualizer(arguments); + // Tracking FPS for visualization + Utilities::FpsTracker fps_tracker; + fps_tracker.AddFrame(); + while ( _running ) { + // The sequence reader chooses what to open based on command line arguments provided + if (!sequence_reader.Open(arguments)) + break; + INFO_STREAM("Device or file opened"); + if (sequence_reader.IsWebcam()) { + INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); + INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); + visualizer.vis_track = true; + } + cv::Mat captured_image; + Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), + sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); + if (!face_model.eye_model) { + recording_params.setOutputGaze(false); + } + Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); + if (recording_params.outputGaze() && !face_model.eye_model) + std::cout << "WARNING: no eye model defined, but outputting gaze" << std::endl; + captured_image = sequence_reader.GetNextFrame(); + // For reporting progress + double reported_completion = 0; + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) { + // Converting to grayscale + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); + // The actual facial landmark detection / tracking + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters, grayscale_image); + // Gaze tracking, absolute gaze direction + cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); + if (detection_success && face_model.eye_model) { + GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); + GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); + gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); + } + // Do face alignment + cv::Mat sim_warped_img; + cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; + // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization + if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus) { + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); + face_analyser.GetLatestAlignedFace(sim_warped_img); + face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); + } + // Work out the pose of the head from the tracked model + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + // Keeping track of FPS + fps_tracker.AddFrame(); + // Displaying the tracking visualizations + visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetObservationFaceAlign(sim_warped_img); + visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); + visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); + visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); + visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); + visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + visualizer.SetFps(fps_tracker.GetFPS()); + // detect key presses + char character_press = visualizer.ShowObservation(); + // quit processing the current sequence (useful when in Webcam mode) + if ( !_running || character_press == 'q') { + break; + } + // Setting up the recorder output + // The number of channels in HOG is fixed at the moment, as using FHOG + open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); + open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); + open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); + open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), + face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); + open_face_rec.SetObservationPose(pose_estimate); + open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); + open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); + open_face_rec.SetObservationFaceID(0); + open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber()); + open_face_rec.SetObservationFaceAlign(sim_warped_img); + open_face_rec.WriteObservation(); + open_face_rec.WriteObservationTracked(); + // CALLING OPENVIDEO CALLBACK + new_frame( open_face_rec ); + // Reporting progress + if (sequence_reader.GetProgress() >= reported_completion / 10.0) { + std::cout << reported_completion * 10 << "% "; + if (reported_completion == 10) { + std::cout << std::endl; + } + reported_completion = reported_completion + 1; + } + // Grabbing the next frame in the sequence + captured_image = sequence_reader.GetNextFrame(); + } + INFO_STREAM("Closing output recorder"); + open_face_rec.Close(); + INFO_STREAM("Closing input reader"); + sequence_reader.Close(); + /* + if (recording_params.outputAUs()) + { + INFO_STREAM("Postprocessing the Action Unit predictions"); + face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); + }*/ + // Reset the models for the next video + face_analyser.Reset(); + face_model.Reset(); + } return true; - } void new_frame( Utilities::RecorderOpenFace& rec ) { - - boost::unique_lock< boost::shared_mutex > lock(_access); - const cv::Mat_& l3d = rec.get_landmarks_3D(); cv::Size s = l3d.size(); - frame.gaze_0 = rec.get_gaze_direction(0); frame.gaze_1 = rec.get_gaze_direction(1); - cv::Vec6f h = rec.get_head_pose(); frame.head_pos.set( h[0], h[1], h[2] ); frame.head_rot.set( h[3], h[4], h[5] ); - for ( int c = 0; c < s.width; ++c ) { frame.landmarks[c].x = l3d.at( 0,c ); frame.landmarks[c].y = l3d.at( 1,c ); @@ -512,26 +688,29 @@ class OpenfaceVideo { frame.fID++; frame.updated = true; + // if a callback function is defined, updating the local copy and passing it + // to the callback function + if ( pycall != 0 ) { + pyframe = frame.to_python(); + pycall->call( pyframe ); + } + } bool stop() { - if ( thread_running ) { - thread_running = false; - boost::this_thread::sleep(boost::posix_time::milliseconds(10)); - worker = 0; + if ( _running ) { + _running = false; std::cout << "OpenfaceVideo successfully closed" << std::endl; return true; - } return false; } bool is_running() const { - return worker != 0; + return _running; } bool has_frame() { - boost::unique_lock< boost::shared_mutex > lock(_access); if ( frame.updated ) { public_frame = frame; frame.updated = false; @@ -543,6 +722,13 @@ class OpenfaceVideo { OpenfaceFrame get_frame() { return public_frame; } + + void callback_frame( const boost::python::object& object ) { + if ( pycall != 0 ) { + delete pycall; + } + pycall = new py_callable{object}; + } private: @@ -592,54 +778,13 @@ class OpenfaceVideo { } std::vector arguments; - bool thread_running; + bool _running; bool _data_updated; - boost::thread_group openfacevideo_threads; - boost::thread* worker; OpenfaceFrame frame; -}; - -class with_gil { -public: - with_gil() { state_ = PyGILState_Ensure(); } - ~with_gil() { PyGILState_Release(state_); } - with_gil(const with_gil&) = delete; - with_gil& operator=(const with_gil&) = delete; -private: - PyGILState_STATE state_; -}; - -class py_callable { -public: - - /// @brief Constructor that assumes the caller has the GIL locked. - py_callable(const boost::python::object& object) { - with_gil gil; - object_.reset( - // GIL locked, so it is safe to copy. - new boost::python::object{object}, - // Use a custom deleter to hold GIL when the object is deleted. - [](boost::python::object* object) { - with_gil gil; - delete object; - }); - } - - // Use default copy-constructor and assignment-operator. - py_callable(const py_callable&) = default; - py_callable& operator=(const py_callable&) = default; - - template - void operator()(Args... args) { - // Lock the GIL as the python object is going to be invoked. - with_gil gil; - (*object_)(std::forward(args)...); - } - -private: - std::shared_ptr object_; + boost::python::dict pyframe; + py_callable* pycall; }; @@ -647,14 +792,15 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { using namespace boost::python; + PyEval_InitThreads(); + + /* class_("OpenfaceV3") .def_readonly("x", &OpenfaceV3::x) .def_readonly("y", &OpenfaceV3::y) .def_readonly("z", &OpenfaceV3::z); - class_("OpenfaceV3Vector") .def(vector_indexing_suite()); - class_("OpenfaceFrame") .def_readonly("ID", &OpenfaceFrame::fID) .def_readonly("gaze_0", &OpenfaceFrame::gaze_0) @@ -662,17 +808,19 @@ BOOST_PYTHON_MODULE(PyOpenfaceVideo) { .def_readonly("head_rot", &OpenfaceFrame::head_rot) .def_readonly("head_pos", &OpenfaceFrame::head_pos) .def_readonly("landmarks", &OpenfaceFrame::landmarks); + */ class_("OpenfaceVideo") .add_property("device", &OpenfaceVideo::get_device, &OpenfaceVideo::set_device) .add_property("landmark_model", &OpenfaceVideo::get_landmark_model, &OpenfaceVideo::set_landmark_model) .add_property("HAAR", &OpenfaceVideo::get_HAAR, &OpenfaceVideo::set_HAAR) .add_property("MTCNN", &OpenfaceVideo::get_MTCNN, &OpenfaceVideo::set_MTCNN) + .add_property("AU", &OpenfaceVideo::get_AU, &OpenfaceVideo::set_AU) .def("start", &OpenfaceVideo::start) .def("stop", &OpenfaceVideo::stop) .def("is_running", &OpenfaceVideo::is_running) - .def("has_frame", &OpenfaceVideo::has_frame) - .def_readonly("frame", &OpenfaceVideo::public_frame); + .def("callback_frame",&OpenfaceVideo::callback_frame); + } int main(int argc, char **argv) { diff --git a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp index fb5dd1087..e9d7ca653 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp @@ -67,6 +67,8 @@ FaceAnalyserParameters::FaceAnalyserParameters(std::vector &argumen bool scale_set = false; bool size_set = false; + + std::string au_custom_path = ""; for (size_t i = 1; i < arguments.size(); ++i) { @@ -103,6 +105,13 @@ FaceAnalyserParameters::FaceAnalyserParameters(std::vector &argumen size_set = true; i++; } + else if (arguments[i].compare("-au") == 0) + { + au_custom_path = arguments[i + 1]; + valid[i] = false; + valid[i + 1] = false; + i++; + } } for (int i = (int)arguments.size() - 1; i >= 0; --i) @@ -113,13 +122,21 @@ FaceAnalyserParameters::FaceAnalyserParameters(std::vector &argumen } } - if (dynamic) - { - this->model_location = "AU_predictors/main_dynamic_svms.txt"; - } - else - { - this->model_location = "AU_predictors/main_static_svms.txt"; + if ( au_custom_path.length() > 0 ) { + + this->model_location = au_custom_path; + + } else { + + if (dynamic) + { + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + } + else + { + this->model_location = "AU_predictors/main_static_svms.txt"; + } + } // If we set the size but not the scale, adapt the scale to the right size diff --git a/python_scripts/PyOpenfaceVideo_tester.py b/python_scripts/PyOpenfaceVideo_tester.py index 6bb6e8757..8655e4e3b 100644 --- a/python_scripts/PyOpenfaceVideo_tester.py +++ b/python_scripts/PyOpenfaceVideo_tester.py @@ -6,6 +6,14 @@ import keyboard from PyOpenfaceVideo import * +def frame_parsing( frame ): + print( '>>>> ', frame['ID'] ) + print( frame['gaze_0'] ) + print( frame['gaze_1'] ) + print( frame['head_rot'] ) + print( frame['head_pos'] ) + print( frame['landmarks'] ) + ov = OpenfaceVideo() ov.device = 0 @@ -15,25 +23,8 @@ ov.HAAR = OFV_DIR + "/classifiers/haarcascade_frontalface_alt.xml" ov.MTCNN = OFV_DIR + "/model/mtcnn_detector/MTCNN_detector.txt" -# start tracking -ov.start() +# setting the callback function +ov.callback_frame( frame_parsing ) -while( ov.is_running() ): - - try: - if keyboard.is_pressed('q'): - ov.stop() - except: - pass - - if ov.has_frame(): - print( "new frame: ", ov.frame.ID ) - print( "\tgaze_0: ", ov.frame.gaze_0.x, ',', ov.frame.gaze_0.y, ',', ov.frame.gaze_0.z ) - print( "\tgaze_1: ", ov.frame.gaze_1.x, ',', ov.frame.gaze_1.y, ',', ov.frame.gaze_1.z ) - print( "\thead_rot: ", ov.frame.head_rot.x, ',', ov.frame.head_rot.y, ',', ov.frame.head_rot.z ) - print( "\thead_pos: ", ov.frame.head_pos.x, ',', ov.frame.head_pos.y, ',', ov.frame.head_pos.z ) - print( "\tlandmarks: " ) - for i in range( len(ov.frame.landmarks) ): - print( "\t\t", i, ':', ov.frame.landmarks[i].x, ',', ov.frame.landmarks[i].y, ',', ov.frame.landmarks[i].z ) - - time.sleep( 0.01 ) \ No newline at end of file +# start tracking +ov.start() \ No newline at end of file From 04c1bc01328e229e97b3a2a3e260d0a30fce1423 Mon Sep 17 00:00:00 2001 From: frankiezafe Date: Mon, 2 Sep 2019 01:34:16 +0200 Subject: [PATCH 27/27] clean up --- .../FeatureExtractionPython.cpp | 166 ------------------ 1 file changed, 166 deletions(-) diff --git a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp index ea48e2186..f83893151 100644 --- a/exe/FeatureExtractionPython/FeatureExtractionPython.cpp +++ b/exe/FeatureExtractionPython/FeatureExtractionPython.cpp @@ -82,172 +82,6 @@ printErrorAndAbort( std::string( "Fatal error: " ) + stream ) #define LANDMARKS_NUM 68 -boost::shared_mutex _access; - -int OpenfaceVideoWorker( - std::vector& arguments, - const bool& running, - boost::function callback, - boost::thread_group & thread_group, - boost::thread * this_thread ) { - - // Load the modules that are being used for tracking and face analysis - // Load face landmark detector - LandmarkDetector::FaceModelParameters det_parameters(arguments); - // Always track gaze in feature extraction - LandmarkDetector::CLNF face_model(det_parameters.model_location); - if (!face_model.loaded_successfully) { - std::cout << "ERROR: Could not load the landmark detector" << std::endl; - return 1; - } - // Load facial feature extractor and AU analyser - FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); - FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - if (!face_model.eye_model) { - std::cout << "WARNING: no eye model found" << std::endl; - } - if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0) { - std::cout << "WARNING: no Action Unit models found" << std::endl; - } - Utilities::SequenceCapture sequence_reader; - // A utility for visualizing the results - Utilities::Visualizer visualizer(arguments); - // Tracking FPS for visualization - Utilities::FpsTracker fps_tracker; - fps_tracker.AddFrame(); - while ( running ) { - - // The sequence reader chooses what to open based on command line arguments provided - if (!sequence_reader.Open(arguments)) - break; - INFO_STREAM("Device or file opened"); - if (sequence_reader.IsWebcam()) { - INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode"); - INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)"); - visualizer.vis_track = true; - } - cv::Mat captured_image; - Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), - sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); - if (!face_model.eye_model) { - recording_params.setOutputGaze(false); - } - Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); - if (recording_params.outputGaze() && !face_model.eye_model) - std::cout << "WARNING: no eye model defined, but outputting gaze" << std::endl; - captured_image = sequence_reader.GetNextFrame(); - // For reporting progress - double reported_completion = 0; - INFO_STREAM("Starting tracking"); - while (!captured_image.empty()) { - // Converting to grayscale - cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); - // The actual facial landmark detection / tracking - bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters, grayscale_image); - // Gaze tracking, absolute gaze direction - cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); - if (detection_success && face_model.eye_model) { - GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true); - GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false); - gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1); - } - // Do face alignment - cv::Mat sim_warped_img; - cv::Mat_ hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0; - // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization - if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus) { - face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); - face_analyser.GetLatestAlignedFace(sim_warped_img); - face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); - } - - // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - - // Keeping track of FPS - fps_tracker.AddFrame(); - - // Displaying the tracking visualizations - visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - visualizer.SetObservationFaceAlign(sim_warped_img); - visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); - visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); - visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); - visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); - visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); - visualizer.SetFps(fps_tracker.GetFPS()); - - // detect key presses - char character_press = visualizer.ShowObservation(); - - // quit processing the current sequence (useful when in Webcam mode) - if ( !running || character_press == 'q') { - break; - } - - // Setting up the recorder output - open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG - open_face_rec.SetObservationVisualization(visualizer.GetVisImage()); - open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass()); - open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), - face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success); - open_face_rec.SetObservationPose(pose_estimate); - open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy)); - open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp); - open_face_rec.SetObservationFaceID(0); - open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber()); - open_face_rec.SetObservationFaceAlign(sim_warped_img); - open_face_rec.WriteObservation(); - open_face_rec.WriteObservationTracked(); - - // CALLING OPENVIDEO CALLBACK - callback( open_face_rec ); - - // Reporting progress - if (sequence_reader.GetProgress() >= reported_completion / 10.0) - { - std::cout << reported_completion * 10 << "% "; - if (reported_completion == 10) - { - std::cout << std::endl; - } - reported_completion = reported_completion + 1; - } - - // Grabbing the next frame in the sequence - captured_image = sequence_reader.GetNextFrame(); - - } - - INFO_STREAM("Closing output recorder"); - open_face_rec.Close(); - INFO_STREAM("Closing input reader"); - sequence_reader.Close(); - - /* - if (recording_params.outputAUs()) - { - INFO_STREAM("Postprocessing the Action Unit predictions"); - face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile()); - }*/ - - // Reset the models for the next video - face_analyser.Reset(); - face_model.Reset(); - - } - - thread_group.remove_thread(this_thread); - delete this_thread; - - this_thread = 0; - - INFO_STREAM("Closed successfully"); - - return 0; - -} - struct OpenfaceV3 { float x;