From 3f6032666154cda422ed8ca62454ea9ec1cc05f7 Mon Sep 17 00:00:00 2001 From: Branko Kokanovic Date: Thu, 19 Jul 2018 19:12:18 +0200 Subject: [PATCH 1/2] Support for raw chinese_whispers This commit introduces chinese_whispers method without any "helpers" on PHP side. Users needs to take care of everything (building edges from 128D face chip vector, for example), but this is exposed for people that need low-level call and want to calculate distances and build edges in PHP directly. --- README.md | 19 ++++ config.m4 | 1 + pdlib.cc | 2 + src/chinese_whispers.cc | 104 ++++++++++++++++++ src/chinese_whispers.h | 13 +++ tests/chinese_whispers_basic.phpt | 27 +++++ ...whispers_edge_associative_array_error.phpt | 13 +++ ...inese_whispers_edge_elements_not_long.phpt | 19 ++++ ...ese_whispers_edge_not_2_element_error.phpt | 20 ++++ ...chinese_whispers_edge_not_array_error.phpt | 21 ++++ ...chinese_whispers_wrong_arg_type_error.phpt | 15 +++ 11 files changed, 254 insertions(+) create mode 100644 src/chinese_whispers.cc create mode 100644 src/chinese_whispers.h create mode 100644 tests/chinese_whispers_basic.phpt create mode 100644 tests/chinese_whispers_edge_associative_array_error.phpt create mode 100644 tests/chinese_whispers_edge_elements_not_long.phpt create mode 100644 tests/chinese_whispers_edge_not_2_element_error.phpt create mode 100644 tests/chinese_whispers_edge_not_array_error.phpt create mode 100644 tests/chinese_whispers_wrong_arg_type_error.phpt diff --git a/README.md b/README.md index ad3114f..bb75092 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,28 @@ var_dump($landmarks); ``` +#### chinese whisers + +Provides raw access to dlib's `chinese_whispers` function. +Client need to build and provide edges. Edges are provided +as numeric array. Each element of this array should also be +numeric array with 2 elements of long type. + +Returned value is also numeric array, containing obtained labels. + +```php + +#include +#include + +using namespace dlib; +using namespace std; + +PHP_FUNCTION(dlib_chinese_whispers) +{ + zval *edges_arg; + HashTable *edges_arg_hash; + + std::vector edges; + std::vector labels; + + if(zend_parse_parameters(ZEND_NUM_ARGS(), "a", &edges_arg) == FAILURE){ + zend_throw_exception_ex( + zend_ce_exception, + 0 TSRMLS_CC, + "Unable to parse edges in dlib_chinese_whispers"); + return; + } + + edges_arg_hash = Z_ARRVAL_P(edges_arg); + + try { + HashPosition pos; + zval *edge; + HashTable *edge_hash; + + // Iterate for all given edges, check if they are valid and put them to edges + // + for ( + zend_hash_internal_pointer_reset_ex(edges_arg_hash, &pos); + (edge = zend_hash_get_current_data_ex(edges_arg_hash, &pos)) != nullptr; + zend_hash_move_forward_ex(edges_arg_hash, &pos)) { + // Check that each given edge is actually array + // + if (Z_TYPE_P(edge) != IS_ARRAY) { + zend_throw_exception_ex( + zend_ce_exception, + 0 TSRMLS_CC, + "Each edge provided in array needs to be numeric array of 2 elements"); + return; + } + + edge_hash = Z_ARRVAL_P(edge); + + // Check that there are two elements in this edge + // + if (zend_hash_num_elements(edge_hash) != 2) { + zend_throw_exception_ex( + zend_ce_exception, + 0 TSRMLS_CC, + "Edges need to contain exactly two elements"); + return; + } + + // Check that this is regular array with integer keys + // + if (!zend_hash_index_exists(edge_hash, 0) || + !zend_hash_index_exists(edge_hash, 1)) { + zend_throw_exception_ex( + zend_ce_exception, + 0 TSRMLS_CC, + "Edge should be numeric array with integer keys"); + return; + } + + zval *elem_i = zend_hash_index_find(edge_hash, 0); + zval *elem_j = zend_hash_index_find(edge_hash, 1); + + // Check that both elements in array are longs + if ((Z_TYPE_P(elem_i) != IS_LONG) || (Z_TYPE_P(elem_j) != IS_LONG)) { + zend_throw_exception_ex( + zend_ce_exception, + 0 TSRMLS_CC, + "Both elements in each edge must be of long type"); + return; + } + + // Finally, put extracted elements to edges + edges.push_back(sample_pair(Z_LVAL_P(elem_i), Z_LVAL_P(elem_j))); + } + + chinese_whispers(edges, labels); + + // Preparing and generating response array containing labels + // + array_init(return_value); + for (auto label = labels.begin(); label != labels.end(); label++) { + add_next_index_long(return_value, *label); + } + } catch (exception& e) + { + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what()); + return; + } +} + diff --git a/src/chinese_whispers.h b/src/chinese_whispers.h new file mode 100644 index 0000000..ec38e62 --- /dev/null +++ b/src/chinese_whispers.h @@ -0,0 +1,13 @@ +// +// Created by branko at kokanovic dot org on 2018/7/19. +// + +#ifndef PHP_DLIB_CHINESE_WHISPERS_H +#define PHP_DLIB_CHINESE_WHISPERS_H + +ZEND_BEGIN_ARG_INFO_EX(dlib_chinese_whispers_arginfo, 0, 0, 1) + ZEND_ARG_INFO(0, edges) +ZEND_END_ARG_INFO() +PHP_FUNCTION(dlib_chinese_whispers); + +#endif //PHP_DLIB_CHINESE_WHISPERS_H diff --git a/tests/chinese_whispers_basic.phpt b/tests/chinese_whispers_basic.phpt new file mode 100644 index 0000000..5f8e58d --- /dev/null +++ b/tests/chinese_whispers_basic.phpt @@ -0,0 +1,27 @@ +--TEST-- +Basic tests for chinese_whispers +--SKIPIF-- + +--FILE-- + +--EXPECT-- +array(1) { + [0]=> + int(0) +} +array(2) { + [0]=> + int(0) + [1]=> + int(1) +} +array(2) { + [0]=> + int(0) + [1]=> + int(0) +} diff --git a/tests/chinese_whispers_edge_associative_array_error.phpt b/tests/chinese_whispers_edge_associative_array_error.phpt new file mode 100644 index 0000000..ac7cd1d --- /dev/null +++ b/tests/chinese_whispers_edge_associative_array_error.phpt @@ -0,0 +1,13 @@ +--TEST-- +Edge given in edges array for chinese_whispers functions is associative array +--SKIPIF-- + +--FILE-- +0, "bar"=>1]]); +} catch (Exception $e) { + var_dump($e->getMessage()); +} +--EXPECT-- +string(46) "Edge should be numeric array with integer keys" diff --git a/tests/chinese_whispers_edge_elements_not_long.phpt b/tests/chinese_whispers_edge_elements_not_long.phpt new file mode 100644 index 0000000..9b06728 --- /dev/null +++ b/tests/chinese_whispers_edge_elements_not_long.phpt @@ -0,0 +1,19 @@ +--TEST-- +Edge elements given in edges array for chinese_whispers functions are not of long type +--SKIPIF-- + +--FILE-- +getMessage()); +} +try { + dlib_chinese_whispers([[0,0], [1, 1.1]]); +} catch (Exception $e) { + var_dump($e->getMessage()); +} +--EXPECT-- +string(47) "Both elements in each edge must be of long type" +string(47) "Both elements in each edge must be of long type" diff --git a/tests/chinese_whispers_edge_not_2_element_error.phpt b/tests/chinese_whispers_edge_not_2_element_error.phpt new file mode 100644 index 0000000..d46de75 --- /dev/null +++ b/tests/chinese_whispers_edge_not_2_element_error.phpt @@ -0,0 +1,20 @@ +--TEST-- +Edge given in edges array for chinese_whispers functions is not having all values to be arrays with 2 elements +--SKIPIF-- + +--FILE-- +getMessage()); +} +try { + dlib_chinese_whispers([[0,0], [1,1,1]]); +} catch (Exception $e) { + var_dump($e->getMessage()); +} +?> +--EXPECT-- +string(42) "Edges need to contain exactly two elements" +string(42) "Edges need to contain exactly two elements" diff --git a/tests/chinese_whispers_edge_not_array_error.phpt b/tests/chinese_whispers_edge_not_array_error.phpt new file mode 100644 index 0000000..01c4a8c --- /dev/null +++ b/tests/chinese_whispers_edge_not_array_error.phpt @@ -0,0 +1,21 @@ +--TEST-- +Edge given in edges array is not array for chinese_whispers functions +--SKIPIF-- + +--FILE-- +getMessage()); +} + +try { + dlib_chinese_whispers([[0,0], 1]); +} catch (Exception $e) { + var_dump($e->getMessage()); +} +?> +--EXPECT-- +string(67) "Each edge provided in array needs to be numeric array of 2 elements" +string(67) "Each edge provided in array needs to be numeric array of 2 elements" diff --git a/tests/chinese_whispers_wrong_arg_type_error.phpt b/tests/chinese_whispers_wrong_arg_type_error.phpt new file mode 100644 index 0000000..15fb21a --- /dev/null +++ b/tests/chinese_whispers_wrong_arg_type_error.phpt @@ -0,0 +1,15 @@ +--TEST-- +Args given to chinese_whispers functions is not correct +--SKIPIF-- + +--FILE-- +getMessage()); +} +?> +--EXPECT-- +Warning: dlib_chinese_whispers() expects parameter 1 to be array, string given in /home/branko/pdlib/tests/chinese_whispers_wrong_arg_type_error.php on line 3 +string(46) "Unable to parse edges in dlib_chinese_whispers" From 7de8d060b326512a007f36f18280784b1eb6ed4c Mon Sep 17 00:00:00 2001 From: Branko Kokanovic Date: Sat, 25 Aug 2018 00:51:46 +0200 Subject: [PATCH 2/2] Landmark detection (custom model and class-based) This change extends existing landmark detection in new ways: 1. Existing logic is hiding HOG model (frontal_face_detector) underneath and user cannot use other models (CNN model, for example). 2. Bounding box is exposed as additional argument, and user can define custom bounding box (which is needed, if image used to detect faces is changed (for example scaled), and we want to crop only face from original image to feed into shape predictor). 3. This approach is class-based, so no need for multiple loadings of shape predictor model (only once, in ctor) --- README.md | 16 ++- pdlib.cc | 43 ++++++- src/face_landmark_detection.cc | 110 ++++++++++++++++++ src/face_landmark_detection.h | 20 ++++ tests/face_landmark_detection_ctor_error.phpt | 21 ++++ 5 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 tests/face_landmark_detection_ctor_error.phpt diff --git a/README.md b/README.md index bb75092..ed42ab8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # PDlib - A PHP extension for Dlib -A PHP extension +A PHP extension ## Requirements - Dlib 19.13+ @@ -71,7 +71,19 @@ var_dump($landmarks); ``` -#### chinese whisers +Additionally, you can also use class-based approach: +```php +$rect = array("left"=>value, "top"=>value, "right"=>value, "bottom"=>value); +// You can download a trained facial shape predictor from: +// http://dlib.net/files/shape_predictor_5_face_landmarks.dat.bz2 +$fld = new FaceLandmarkDetection("path/to/shape/predictor/model"); +$parts = $fld->detect("path/to/image.jpg", $rect); +// $parts is integer array where keys are associative values with "x" and "y" for keys +``` + +Note that, if you use class-based approach, you need to feed bounding box rectangle with values obtained from `dlib_face_detection`. If you use `dlib_face_landmark_detection`, everything is already done for you (and you are using HOG face detection model). + +#### chinese whispers Provides raw access to dlib's `chinese_whispers` function. Client need to build and provide edges. Edges are provided diff --git a/pdlib.cc b/pdlib.cc index eeffc74..1fa615e 100644 --- a/pdlib.cc +++ b/pdlib.cc @@ -43,6 +43,9 @@ static int le_pdlib; static zend_class_entry *cnn_face_detection_ce = nullptr; static zend_object_handlers cnn_face_detection_obj_handlers; +static zend_class_entry *face_landmark_detection_ce = nullptr; +static zend_object_handlers face_landmark_detection_obj_handlers; + /* {{{ PHP_INI */ /* Remove comments and fill if you need to have entries in php.ini @@ -111,9 +114,32 @@ zend_object* php_cnn_face_detection_new(zend_class_entry *class_type TSRMLS_DC) static void php_cnn_face_detection_free(zend_object *object) { - cnn_face_detection *cfd = (cnn_face_detection*)((char*)object - XtOffsetOf(cnn_face_detection, std)); - delete cfd->net; - zend_object_std_dtor(object); + cnn_face_detection *cfd = (cnn_face_detection*)((char*)object - XtOffsetOf(cnn_face_detection, std)); + delete cfd->net; + zend_object_std_dtor(object); +} + +const zend_function_entry face_landmark_detection_class_methods[] = { + PHP_ME(FaceLandmarkDetection, __construct, face_landmark_detection_ctor_arginfo, ZEND_ACC_PUBLIC) + PHP_ME(FaceLandmarkDetection, detect, face_landmark_detection_detect_arginfo, ZEND_ACC_PUBLIC) + PHP_FE_END +}; + +zend_object* php_face_landmark_detection_new(zend_class_entry *class_type TSRMLS_DC) +{ + face_landmark_detection *fld = (face_landmark_detection*)ecalloc(1, sizeof(face_landmark_detection)); + zend_object_std_init(&fld->std, class_type TSRMLS_CC); + object_properties_init(&fld->std, class_type); + fld->std.handlers = &face_landmark_detection_obj_handlers; + + return &fld->std; +} + +static void php_face_landmark_detection_free(zend_object *object) +{ + face_landmark_detection *fld = (face_landmark_detection*)((char*)object - XtOffsetOf(face_landmark_detection, std)); + delete fld->sp; + zend_object_std_dtor(object); } /* {{{ PHP_MINIT_FUNCTION @@ -121,6 +147,8 @@ static void php_cnn_face_detection_free(zend_object *object) PHP_MINIT_FUNCTION(pdlib) { zend_class_entry ce; + // CnnFaceDetection class definition + // INIT_CLASS_ENTRY(ce, "CnnFaceDetection", cnn_face_detection_class_methods); cnn_face_detection_ce = zend_register_internal_class(&ce TSRMLS_CC); cnn_face_detection_ce->create_object = php_cnn_face_detection_new; @@ -128,6 +156,15 @@ PHP_MINIT_FUNCTION(pdlib) cnn_face_detection_obj_handlers.offset = XtOffsetOf(cnn_face_detection, std); cnn_face_detection_obj_handlers.free_obj = php_cnn_face_detection_free; + // FaceLandmarkDetection class definition + // + INIT_CLASS_ENTRY(ce, "FaceLandmarkDetection", face_landmark_detection_class_methods); + face_landmark_detection_ce = zend_register_internal_class(&ce TSRMLS_CC); + face_landmark_detection_ce->create_object = php_face_landmark_detection_new; + memcpy(&face_landmark_detection_obj_handlers, zend_get_std_object_handlers(), sizeof(zend_object_handlers)); + face_landmark_detection_obj_handlers.offset = XtOffsetOf(face_landmark_detection, std); + face_landmark_detection_obj_handlers.free_obj = php_face_landmark_detection_free; + /* If you have INI entries, uncomment these lines REGISTER_INI_ENTRIES(); */ diff --git a/src/face_landmark_detection.cc b/src/face_landmark_detection.cc index 25e85fe..3c205c7 100644 --- a/src/face_landmark_detection.cc +++ b/src/face_landmark_detection.cc @@ -2,6 +2,8 @@ #include "../php_pdlib.h" #include "face_landmark_detection.h" +#include + #include #include #include @@ -13,6 +15,12 @@ using namespace dlib; using namespace std; +static inline face_landmark_detection *php_face_landmark_detection_from_obj(zend_object *obj) { + return (face_landmark_detection*)((char*)(obj) - XtOffsetOf(face_landmark_detection, std)); +} + +#define Z_FACE_LANDMARK_DETECTION_P(zv) php_face_landmark_detection_from_obj(Z_OBJ_P((zv))) + PHP_FUNCTION(dlib_face_landmark_detection) { char *shape_predictor_file_path; @@ -60,4 +68,106 @@ PHP_FUNCTION(dlib_face_landmark_detection) { RETURN_FALSE; } +} + +PHP_METHOD(FaceLandmarkDetection, __construct) +{ + char *sz_shape_predictor_file_path; + size_t shape_predictor_file_path_len; + + face_landmark_detection *fld = Z_FACE_LANDMARK_DETECTION_P(getThis()); + + if (nullptr == fld) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Unable to find obj in FaceLandmarkDetection::__construct()"); + return; + } + + // Parse predictor model's path + if (zend_parse_parameters(ZEND_NUM_ARGS(), "s", + &sz_shape_predictor_file_path, &shape_predictor_file_path_len) == FAILURE){ + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse shape_predictor_file_path"); + return; + } + + // Load predictor model from given path + try { + string shape_predictor_file_path(sz_shape_predictor_file_path, shape_predictor_file_path_len); + fld->sp = new shape_predictor; + deserialize(shape_predictor_file_path) >> *(fld->sp); + } catch (exception& e) { + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what()); + return; + } +} + +// Helper macro to automatically have parsing of "top"/"bottom"/"left"/"right" +#define PARSE_BOUNDING_BOX_EDGE(side) \ + zval* data##side; \ + /* Tries to find given key in array */ \ + data##side = zend_hash_str_find(bounding_box_hash, #side, sizeof(#side)-1); \ + if (data##side == nullptr) { \ + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Bounding box (second argument) is missing " #side "key"); \ + return; \ + } \ + \ + /* We also need to check proper type of value in associative array */ \ + if (Z_TYPE_P(data##side) != IS_LONG) { \ + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Value of bounding box's (second argument) " #side " key is not long type"); \ + return; \ + } \ + zend_long side = Z_LVAL_P(data##side); \ + +PHP_METHOD(FaceLandmarkDetection, detect) +{ + char *img_path; + size_t img_path_len; + zval *bounding_box; + array2d img; + + // Parse path to image and bounding box. Bounding box is associative array of 4 elements - "top", "bottom", "left" and "right". + // + if (zend_parse_parameters(ZEND_NUM_ARGS(), "sa", &img_path, &img_path_len, &bounding_box) == FAILURE){ + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse detect arguments"); + return; + } + + // Check that bounding box have exactly 4 elements + HashTable *bounding_box_hash = Z_ARRVAL_P(bounding_box); + uint32_t bounding_box_num_elements = zend_hash_num_elements(bounding_box_hash); + if (bounding_box_num_elements != 4) { + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Bounding box (second argument) needs to have exactly 4 elements"); + return; + } + + // Retrieve all 4 edges of bounding box + // + PARSE_BOUNDING_BOX_EDGE(top) + PARSE_BOUNDING_BOX_EDGE(bottom) + PARSE_BOUNDING_BOX_EDGE(left) + PARSE_BOUNDING_BOX_EDGE(right) + + try { + // Load image and execute shape predictor on it. + // + face_landmark_detection *fld = Z_FACE_LANDMARK_DETECTION_P(getThis()); + load_image(img, img_path); + rectangle rct(left, top, right, bottom); + full_object_detection shape = fld->sp->operator()(img, rct); + + // Return value is regular array with integer keys. + // Each key is one part from shape. Value of each part is associative array of keys "x" and "y". + // + array_init(return_value); + for (int i = 0; i < shape.num_parts(); i++) { + zval part; + array_init(&part); + dlib::point p = shape.part(i); + add_assoc_long(&part, "x", p.x()); + add_assoc_long(&part, "y", p.y()); + add_next_index_zval(return_value, &part); + } + } catch (exception& e) { + zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what()); + return; + } } \ No newline at end of file diff --git a/src/face_landmark_detection.h b/src/face_landmark_detection.h index caff095..873ba49 100644 --- a/src/face_landmark_detection.h +++ b/src/face_landmark_detection.h @@ -5,10 +5,30 @@ #ifndef PDLIB_FACE_LANDMARK_DETECTION_H #define PDLIB_FACE_LANDMARK_DETECTION_H +#include + +using namespace dlib; + ZEND_BEGIN_ARG_INFO_EX(dlib_face_landmark_detection_arginfo, 0, 0, 1) ZEND_ARG_INFO(0, shape_predictor_file_path) ZEND_ARG_INFO(0, img_path) ZEND_END_ARG_INFO() PHP_FUNCTION(dlib_face_landmark_detection); +typedef struct _face_landmark_detection { + shape_predictor *sp; + zend_object std; +} face_landmark_detection; + +ZEND_BEGIN_ARG_INFO_EX(face_landmark_detection_ctor_arginfo, 0, 0, 1) + ZEND_ARG_INFO(0, shape_predictor_file_path) +ZEND_END_ARG_INFO() +PHP_METHOD(FaceLandmarkDetection, __construct); + +ZEND_BEGIN_ARG_INFO_EX(face_landmark_detection_detect_arginfo, 0, 0, 2) + ZEND_ARG_INFO(0, img_path) + ZEND_ARG_INFO(0, bounding_box) +ZEND_END_ARG_INFO() +PHP_METHOD(FaceLandmarkDetection, detect); + #endif //PDLIB_FACE_LANDMARK_DETECTION_H diff --git a/tests/face_landmark_detection_ctor_error.phpt b/tests/face_landmark_detection_ctor_error.phpt new file mode 100644 index 0000000..d01961d --- /dev/null +++ b/tests/face_landmark_detection_ctor_error.phpt @@ -0,0 +1,21 @@ +--TEST-- +Testing FaceLandmarkDetection constructor without arguments +--SKIPIF-- + +--FILE-- +getMessage()); +} +try { + new FaceLandmarkDetection("non-existent file"); +} catch (Exception $e) { + var_dump($e->getMessage()); +} +?> +--EXPECT-- +Warning: FaceLandmarkDetection::__construct() expects exactly 1 parameter, 0 given in /home/branko/pdlib/tests/face_landmark_detection_ctor_error.php on line 3 +string(41) "Unable to parse shape_predictor_file_path" +string(45) "Unable to open non-existent file for reading." \ No newline at end of file