diff --git a/configure.ac b/configure.ac index 4c252549..65e8606a 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ dnl please read gstreamer/docs/random/autotools before changing this file dnl initialize autoconf dnl releases only do -Wall, git and prerelease does -Werror too dnl use a three digit version number for releases, and four for git/pre -AC_INIT([GStreamer Inference],[0.8.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) +AC_INIT([GStreamer Inference],[0.9.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) AG_GST_INIT @@ -268,27 +268,55 @@ echo dnl *** opencv *** OPENCV_REQ=2.3.1 translit(dnm, m, l) AM_CONDITIONAL(USE_OPENCV, true) -AG_GST_CHECK_FEATURE(OPENCV, [OpenCV computer vision library], opencv, [ - AG_GST_PKG_CHECK_MODULES(OPENCV, opencv >= $OPENCV_REQ ) -]) -dnl define conditional for opencv versions 2.3.1 to 3.1.0 +AG_GST_CHECK_FEATURE(OPENCV, [OpenCV computer vision library], opencv, + [ + PKG_CHECK_MODULES(OPENCV, opencv4, + [ + HAVE_OPENCV="yes" + HAVE_OPENCV4="yes" + ], + [ + HAVE_OPENCV="no" + HAVE_OPENCV4="no" + AC_MSG_NOTICE($OPENCV_PKG_ERRORS) + PKG_CHECK_MODULES(OPENCV, opencv >= OPENCV_REQ , + [ + HAVE_OPENCV="yes" + HAVE_OPENCV4="no" + ], + [ + HAVE_OPENCV="no" + AC_MSG_NOTICE($OPENCV_PKG_ERRORS) + ]) + ]) + ]) + +dnl define conditional for opencv versions 2.3.1 to 4.0.0 if test "$USE_OPENCV" = "yes"; then OCV_BASE_VERSION_MAJOR=3 OCV_BASE_VERSION_MINOR=2 - OCV_VERSION_MAJOR=$(pkg-config --modversion opencv | awk -F. '{print $1}') - OCV_VERSION_MINOR=$(pkg-config --modversion opencv | awk -F. '{print $2}') - if test "$OCV_VERSION_MAJOR" -lt "$OCV_BASE_VERSION_MAJOR"; then - AC_DEFINE([OCV_VERSION_LT_3_2], [1], [Defined if OpenCV version is less than 3.2]) + if test "$HAVE_OPENCV4" = "yes"; then + AC_DEFINE([OCV_VERSION_4_0], [1], [Defined if OpenCV version is 4]) + OCV_VERSION_MAJOR=$(pkg-config --modversion opencv4 | awk -F. '{print $1}') + OCV_VERSION_MINOR=$(pkg-config --modversion opencv4 | awk -F. '{print $2}') else - if test "$OCV_VERSION_MINOR" -lt "$OCV_BASE_VERSION_MINOR"; then - AC_DEFINE([OCV_VERSION_LT_3_2], [1], [Defined if OpenCV version is less than 3.2]) + if test "$HAVE_OPENCV4" = "no"; then + OCV_VERSION_MAJOR=$(pkg-config --modversion opencv | awk -F. '{print $1}') + OCV_VERSION_MINOR=$(pkg-config --modversion opencv | awk -F. '{print $2}') + if test "$OCV_VERSION_MAJOR" -lt "$OCV_BASE_VERSION_MAJOR"; then + AC_DEFINE([OCV_VERSION_LT_3_2], [1], [Defined if OpenCV version is less than 3.2]) + else + if test "$OCV_VERSION_MINOR" -lt "$OCV_BASE_VERSION_MINOR"; then + AC_DEFINE([OCV_VERSION_LT_3_2], [1], [Defined if OpenCV version is less than 3.2]) + fi + fi fi fi AC_MSG_NOTICE([OpenCV version: $OCV_VERSION_MAJOR.$OCV_VERSION_MINOR]) fi dnl *** r2inference *** -R2INFERENCE_REQ=0.5.0 +R2INFERENCE_REQ=0.5.2 AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun\'s Inference Framework], r2inference, [ AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0 >= $R2INFERENCE_REQ) ],[],[],[ diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index 7762f6a6..f20e5919 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -13,6 +13,7 @@ libgstinferenceoverlayplugin_la_CFLAGS = \ $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_VIDEO_CFLAGS) \ $(R2INFERENCE_CFLAGS) \ + $(OPENCV_CFLAGS) \ -I$(top_srcdir)/gst-libs diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index 1422cb96..21d38599 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -89,7 +89,6 @@ plugin_init (GstPlugin * plugin) goto out; } - ret = gst_element_register (plugin, "facenetv1", GST_RANK_NONE, GST_TYPE_FACENETV1); if (!ret) { diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 7edba1ce..cd22ca2b 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -67,6 +67,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_tinyyolov2_debug_category); #define DEFAULT_IOU_THRESH 0.30 #define TOTAL_CLASSES 20 +#define TOTAL_BOXES 845 + const gfloat box_anchors[] = { 1.08, 1.19, 3.42, 4.41, 6.63, 11.38, 9.42, 5.11, 16.62, 10.52 }; @@ -317,7 +319,8 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov2 *tinyyolov2; - gdouble *probabilities = NULL; + gdouble **probabilities = g_malloc (sizeof (gdouble) * TOTAL_BOXES); + gint i; GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; @@ -329,9 +332,15 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, gst_create_boxes (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities, + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, probabilities, TOTAL_CLASSES); + /* Free probabilities */ + for (i = 0; i < detect_meta->num_boxes; i++) { + g_free (probabilities[i]); + } + g_free (probabilities); + gst_inference_print_boxes (vi, gst_tinyyolov2_debug_category, detect_meta); *valid_prediction = (detect_meta->num_boxes > 0) ? TRUE : FALSE; @@ -349,7 +358,7 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, GstInferenceMeta *imeta = NULL; BBox *boxes = NULL; gint num_boxes, i; - gdouble *probabilities = NULL; + gdouble **probabilities = g_malloc (sizeof (gdouble) * TOTAL_BOXES); g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (meta_model != NULL, FALSE); @@ -363,7 +372,7 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, /* Create boxes from prediction data */ gst_create_boxes (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities, + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, probabilities, TOTAL_CLASSES); GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); @@ -376,14 +385,14 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, - probabilities); - + probabilities[i]); gst_inference_prediction_append (imeta->prediction, pred); + g_free (probabilities[i]); } /* Free boxes after creation */ g_free (boxes); - + g_free (probabilities); /* Log predictions */ gst_inference_print_predictions (vi, gst_tinyyolov2_debug_category, imeta); diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index c2e8bbf3..9d28b5fe 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -64,6 +64,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_tinyyolov3_debug_category); #define DEFAULT_IOU_THRESH 0.40 #define TOTAL_CLASSES 80 +#define TOTAL_BOXES 2535 /* prototypes */ static void gst_tinyyolov3_set_property (GObject * object, @@ -284,7 +285,8 @@ gst_tinyyolov3_postprocess_old (GstVideoInference * vi, GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov3 *tinyyolov3; - gdouble *probabilities = NULL; + gdouble **probabilities = g_malloc (sizeof (gdouble) * TOTAL_BOXES); + gint i; GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; @@ -296,9 +298,15 @@ gst_tinyyolov3_postprocess_old (GstVideoInference * vi, gst_create_boxes_float (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, - tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, probabilities, TOTAL_CLASSES); + /* Free probabilities */ + for (i = 0; i < detect_meta->num_boxes; i++) { + g_free (probabilities[i]); + } + g_free (probabilities); + gst_inference_print_boxes (vi, gst_tinyyolov3_debug_category, detect_meta); *valid_prediction = (detect_meta->num_boxes > 0) ? TRUE : FALSE; @@ -316,7 +324,7 @@ gst_tinyyolov3_postprocess_new (GstVideoInference * vi, GstInferenceMeta *imeta = NULL; BBox *boxes = NULL; gint num_boxes, i; - gdouble *probabilities = NULL; + gdouble **probabilities = g_malloc (sizeof (gdouble) * TOTAL_BOXES); g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (meta_model != NULL, FALSE); @@ -330,7 +338,7 @@ gst_tinyyolov3_postprocess_new (GstVideoInference * vi, /* Create boxes from prediction data */ gst_create_boxes_float (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov3->obj_thresh, - tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, probabilities, TOTAL_CLASSES); GST_LOG_OBJECT (tinyyolov3, "Number of predictions: %d", num_boxes); @@ -344,12 +352,14 @@ gst_tinyyolov3_postprocess_new (GstVideoInference * vi, for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, - probabilities); + probabilities[i]); gst_inference_prediction_append (imeta->prediction, pred); + g_free (probabilities[i]); } /* Free boxes after creation */ g_free (boxes); + g_free (probabilities); /* Log predictions */ gst_inference_print_predictions (vi, gst_tinyyolov3_debug_category, imeta); diff --git a/gst-libs/Makefile.am b/gst-libs/Makefile.am index 99c71898..3d084fed 100644 --- a/gst-libs/Makefile.am +++ b/gst-libs/Makefile.am @@ -1,3 +1,2 @@ SUBDIRS=gst - diff --git a/gst-libs/gst/opencv/Makefile.am b/gst-libs/gst/opencv/Makefile.am index 8d83bb38..c385d7ab 100644 --- a/gst-libs/gst/opencv/Makefile.am +++ b/gst-libs/gst/opencv/Makefile.am @@ -8,19 +8,23 @@ libgstinferencebaseoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ + $(OPENCV_CFLAGS) \ $(R2INFERENCE_CFLAGS) libgstinferencebaseoverlay_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ - $(R2INFERENCE_CFLAGS) + $(OPENCV_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ + -std=c++11 libgstinferencebaseoverlay_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ + $(OPENCV_LIBS) \ $(R2INFERENCE_LIBS) gstinferencebaseoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc index 82697811..3ddfa720 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -40,6 +40,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_inference_base_overlay_debug_category); #define MAX_THICKNESS 100 #define DEFAULT_LABELS NULL #define DEFAULT_NUM_LABELS 0 +#define DEFAULT_ENABLE TRUE #define MIN_STYLE CLASSIC #define DEFAULT_STYLE CLASSIC @@ -56,7 +57,8 @@ enum PROP_THICKNESS, PROP_LABELS, PROP_STYLE, - PROP_ALPHA_OVERLAY + PROP_ALPHA_OVERLAY, + PROP_ENABLE, }; GType @@ -85,6 +87,7 @@ struct _GstInferenceBaseOverlayPrivate gint num_labels; LineStyleBoundingBox style; gdouble alpha_overlay; + gboolean enable; }; /* prototypes */ static void gst_inference_base_overlay_set_property (GObject * object, @@ -149,8 +152,14 @@ gst_inference_base_overlay_class_init (GstInferenceBaseOverlayClass * klass) "Line style to draw the bounding box", LINE_STYLE_BOUNDING_BOX, DEFAULT_STYLE, G_PARAM_READWRITE)); g_object_class_install_property (gobject_class, PROP_ALPHA_OVERLAY, - g_param_spec_double ("alpha_overlay", "alpha", "Overlay transparency", MIN_ALPHA_OVERLAY, - MAX_ALPHA_OVERLAY, DEFAULT_ALPHA_OVERLAY, G_PARAM_READWRITE)); + g_param_spec_double ("alpha_overlay", "alpha", "Overlay transparency", + MIN_ALPHA_OVERLAY, MAX_ALPHA_OVERLAY, DEFAULT_ALPHA_OVERLAY, + G_PARAM_READWRITE)); + + g_object_class_install_property (gobject_class, PROP_ENABLE, + g_param_spec_boolean ("enable", "Enable", + "Whether or not to overlay predictions on the buffers", + DEFAULT_ENABLE, G_PARAM_READWRITE)); base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inference_base_overlay_start); @@ -173,6 +182,7 @@ gst_inference_base_overlay_init (GstInferenceBaseOverlay * inference_overlay) priv->num_labels = DEFAULT_NUM_LABELS; priv->style = DEFAULT_STYLE; priv->alpha_overlay = DEFAULT_ALPHA_OVERLAY; + priv->enable = DEFAULT_ENABLE; } void @@ -217,8 +227,11 @@ gst_inference_base_overlay_set_property (GObject * object, guint property_id, break; case PROP_ALPHA_OVERLAY: priv->alpha_overlay = g_value_get_double (value); - GST_DEBUG_OBJECT (inference_overlay, "Changed overlay transparency to %lf", - priv->alpha_overlay); + GST_DEBUG_OBJECT (inference_overlay, + "Changed overlay transparency to %lf", priv->alpha_overlay); + break; + case PROP_ENABLE: + priv->enable = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -253,6 +266,9 @@ gst_inference_base_overlay_get_property (GObject * object, guint property_id, case PROP_ALPHA_OVERLAY: g_value_set_double (value, priv->alpha_overlay); break; + case PROP_ENABLE: + g_value_set_boolean (value, priv->enable); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -337,6 +353,12 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, g_return_val_if_fail (io_class->process_meta != NULL, GST_FLOW_ERROR); + if (FALSE == priv->enable) { + GST_LOG_OBJECT (trans, "Overlay disabled"); + ret = GST_FLOW_OK; + goto out; + } + meta = gst_buffer_get_meta (frame->buffer, io_class->meta_type); if (NULL == meta) { GST_LOG_OBJECT (trans, "No inference meta found"); diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index 2df4e7e0..18aa4855 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -22,9 +22,17 @@ #ifndef __GST_INFERENCE_BASE_OVERLAY_H__ #define __GST_INFERENCE_BASE_OVERLAY_H__ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + #include #include -#ifdef OCV_VERSION_LT_3_2 + +#ifdef OCV_VERSION_4_0 +#include "opencv4/opencv2/highgui.hpp" +#include "opencv4/opencv2/imgproc.hpp" +#elif OCV_VERSION_LT_3_2 #include "opencv2/highgui/highgui.hpp" #else #include "opencv2/imgproc.hpp" diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index a828d282..27a82f27 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -31,7 +31,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category -#define DOUBLE_PROPERTY_DEFAULT_VALUE 0.5 +#define DOUBLE_PROPERTY_DEFAULT_VALUE 0.0 class InferenceProperty { private: @@ -208,7 +208,7 @@ gst_backend_param_to_spec (r2i::ParameterMeta *param) { spec = g_param_spec_double (param->name.c_str (), param->name.c_str (), param->description.c_str (), - G_MINDOUBLE, + -G_MAXDOUBLE, G_MAXDOUBLE, DOUBLE_PROPERTY_DEFAULT_VALUE, (GParamFlags) gst_backend_param_flags (param->flags)); break; diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c index 280ba639..960e5ccc 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.c +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -31,6 +31,8 @@ static gchar *gst_child_inspector_type_int_to_string (GParamSpec * pspec, GValue * value); static gchar *gst_child_inspector_type_string_to_string (GParamSpec * pspec, GValue * value); +static gchar *gst_child_inspector_type_double_to_string (GParamSpec * pspec, + GValue * value); struct _GstChildInspectorFlag { @@ -53,14 +55,15 @@ static GstChildInspectorFlag flags[] = { static GstChildInspectorType types[] = { {G_TYPE_INT, gst_child_inspector_type_int_to_string}, {G_TYPE_STRING, gst_child_inspector_type_string_to_string}, + {G_TYPE_DOUBLE, gst_child_inspector_type_double_to_string}, {} }; static gchar * gst_child_inspector_type_string_to_string (GParamSpec * pspec, GValue * value) { - return g_strdup_printf ("String. Default: \"%s\"", - g_value_get_string (value)); + GParamSpecString *pstring = G_PARAM_SPEC_STRING (pspec); + return g_strdup_printf ("String. Default: \"%s\"", pstring->default_value); } static gchar * @@ -69,7 +72,16 @@ gst_child_inspector_type_int_to_string (GParamSpec * pspec, GValue * value) GParamSpecInt *pint = G_PARAM_SPEC_INT (pspec); return g_strdup_printf ("Integer. Range: %d - %d Default: %d", - pint->minimum, pint->maximum, g_value_get_int (value)); + pint->minimum, pint->maximum, pint->default_value); +} + +static gchar * +gst_child_inspector_type_double_to_string (GParamSpec * pspec, GValue * value) +{ + GParamSpecDouble *pdouble = G_PARAM_SPEC_DOUBLE (pspec); + + return g_strdup_printf ("Double. Range: %03.03e - %03.03e Default: %03.03e", + pdouble->minimum, pdouble->maximum, pdouble->default_value); } static const gchar * diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index dcd129bb..cd0f0fa6 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -37,11 +37,11 @@ static void gst_box_to_pixels (BBox * normalized_box, gint row, gint col, static gdouble gst_sigmoid (gdouble x); static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities, + gint grid_h, gint grid_w, gint boxes_size, gdouble ** probabilities, gint num_classes); static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint total_boxes, gdouble * probabilities, gint num_classes); + gint total_boxes, gdouble ** probabilities, gint num_classes); gboolean gst_fill_classification_meta (GstClassificationMeta * class_meta, @@ -170,7 +170,7 @@ gst_box_to_pixels (BBox * normalized_box, gint row, gint col, gint box) static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint grid_h, - gint grid_w, gint boxes_size, gdouble * probabilities, gint num_classes) + gint grid_w, gint boxes_size, gdouble ** probabilities, gint num_classes) { gint i, j, c, b; gint index; @@ -194,11 +194,12 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, obj_prob = ((gfloat *) prediction)[index + 4]; /* If the Objectness score is over the threshold add it to the boxes list */ if (obj_prob > obj_thresh) { + double *actual_probs = g_malloc (num_classes * sizeof (gdouble)); max_class_prob = 0; max_class_prob_index = 0; for (c = 0; c < num_classes; c++) { cur_class_prob = ((gfloat *) prediction)[index + box_dim + c]; - probabilities[c] = cur_class_prob; + actual_probs[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { max_class_prob = cur_class_prob; max_class_prob_index = c; @@ -217,6 +218,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, result.y = result.y - result.height * 0.5; boxes[counter] = result; counter = counter + 1; + probabilities[counter - 1] = actual_probs; } } } @@ -244,10 +246,9 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (probabilities != NULL, FALSE); *elements = 0; - *probabilities = g_malloc (num_classes * sizeof (gdouble)); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, - elements, grid_h, grid_w, boxes_size, *probabilities, num_classes); + elements, grid_h, grid_w, boxes_size, probabilities, num_classes); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); @@ -331,7 +332,7 @@ gst_create_class_from_prediction (GstVideoInference * vi, static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes, - gdouble * probabilities, gint num_classes) + gdouble ** probabilities, gint num_classes) { gint i, c; gint index; @@ -354,6 +355,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, /* If the objectness score is over the threshold add it to the boxes list */ if (obj_prob > obj_thresh) { + double *actual_probs = g_malloc (num_classes * sizeof (gdouble)); max_class_prob = 0; max_class_prob_index = 0; box_class_base = index + box_dim; @@ -361,7 +363,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, /* Iterate each class probability */ for (c = 0; c < num_classes; c++) { cur_class_prob = ((gfloat *) prediction)[box_class_base + c]; - probabilities[c] = cur_class_prob; + actual_probs[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { max_class_prob = cur_class_prob; max_class_prob_index = c; @@ -378,6 +380,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, result.height = ((gfloat *) prediction)[index + 3] - result.y; boxes[counter] = result; counter = counter + 1; + probabilities[counter - 1] = actual_probs; } } *elements = counter; @@ -390,7 +393,6 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh, gdouble ** probabilities, gint num_classes) { - gint total_boxes = 2535; BBox boxes[TOTAL_BOXES_15]; g_return_val_if_fail (vi != NULL, FALSE); @@ -400,10 +402,9 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (elements != NULL, FALSE); *elements = 0; - *probabilities = g_malloc (num_classes * sizeof (gdouble)); gst_get_boxes_from_prediction_float (obj_thresh, prob_thresh, prediction, - boxes, elements, total_boxes, *probabilities, num_classes); + boxes, elements, TOTAL_BOXES_15, probabilities, num_classes); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); diff --git a/gst/inferenceutils/Makefile.am b/gst/inferenceutils/Makefile.am index d64a9a3b..6f1b38e8 100644 --- a/gst/inferenceutils/Makefile.am +++ b/gst/inferenceutils/Makefile.am @@ -1,41 +1,43 @@ plugin_LTLIBRARIES = libgstinferenceutils.la -libgstinferenceutils_la_SOURCES = \ - gstinferenceutils.c \ - gstinferencefilter.c \ - gstinferencedebug.c \ - gstinferencecrop.cc \ - cropelement.cc \ - videocrop.cc +libgstinferenceutils_la_SOURCES = \ + gstinferenceutils.c \ + gstinferencefilter.c \ + gstinferencedebug.c \ + gstinferencecrop.cc \ + cropelement.cc \ + videocrop.cc \ + gstinferencebin.c -libgstinferenceutils_la_CFLAGS = \ - $(GST_CFLAGS) \ - $(GST_BASE_CFLAGS) \ - $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs +libgstinferenceutils_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs -libgstinferenceutils_la_CXXFLAGS = \ - $(GST_CFLAGS) \ - $(GST_BASE_CFLAGS) \ - $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs \ - -std=c++11 +libgstinferenceutils_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + -std=c++11 -libgstinferenceutils_la_LIBADD = \ - $(GST_LIBS) \ - $(GST_BASE_LIBS) \ - $(GST_PLUGINS_BASE_LIBS) \ - $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la +libgstinferenceutils_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la -libgstinferenceutils_la_LDFLAGS = \ - $(GST_PLUGIN_LDFLAGS) +libgstinferenceutils_la_LDFLAGS = \ + $(GST_PLUGIN_LDFLAGS) -libgstinferenceutils_la_LIBTOOLFLAGS = \ - $(GST_PLUGIN_LIBTOOLFLAGS) +libgstinferenceutils_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) -noinst_HEADERS = \ - gstinferencefilter.h \ - gstinferencedebug.h \ - gstinferencecrop.h \ - cropelement.h \ - videocrop.h +noinst_HEADERS = \ + gstinferencefilter.h \ + gstinferencedebug.h \ + gstinferencecrop.h \ + cropelement.h \ + videocrop.h \ + gstinferencebin.h diff --git a/gst/inferenceutils/gstinferencebin.c b/gst/inferenceutils/gstinferencebin.c new file mode 100644 index 00000000..1a3f9fc0 --- /dev/null +++ b/gst/inferenceutils/gstinferencebin.c @@ -0,0 +1,540 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstinferencebin + * + * Helper element that simplifies inference pipelines by creating a + * bin with the required elements in the typical inference + * configuration. + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! inferencebin arch=tinyyolov2 \ + * model-location=$MODEL_LOCATION backend=tensorflow input-layer=$INPUT_LAYER \ + * output-layer=$OUTPUT_LAYER labels="`cat labels.txt`" arch::iou-threshold=0.3 ! \ + * videoconvert ! ximagesink sync=false + * ]| + * Detects object in a camera stream + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencebin.h" + +/* generic templates */ +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +GST_DEBUG_CATEGORY_STATIC (gst_inference_bin_debug_category); +#define GST_CAT_DEFAULT gst_inference_bin_debug_category + +static void gst_inference_bin_finalize (GObject * object); +static void gst_inference_bin_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inference_bin_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static GstStateChangeReturn gst_inference_bin_change_state (GstElement * + element, GstStateChange transition); + +static gboolean gst_inference_bin_start (GstInferenceBin * self); +static gboolean gst_inference_bin_stop (GstInferenceBin * self); + +enum +{ + PROP_0, + PROP_ARCH, + PROP_BACKEND, + PROP_MODEL_LOCATION, + PROP_INPUT_LAYER, + PROP_OUTPUT_LAYER, + PROP_LABELS, + PROP_CROP, + PROP_OVERLAY, + PROP_FILTER, + PROP_SCALER, + PROP_CONVERTER, +}; + +#define PROP_ARCH_DEFAULT "tinyyolov2" +#define PROP_BACKEND_DEFAULT "tensorflow" +#define PROP_MODEL_LOCATION_DEFAULT NULL +#define PROP_INPUT_LAYER_DEFAULT NULL +#define PROP_OUTPUT_LAYER_DEFAULT NULL +#define PROP_LABELS_DEFAULT NULL +#define PROP_CROP_DEFAULT FALSE +#define PROP_OVERLAY_DEFAULT TRUE +#define PROP_FILTER_MIN -1 +#define PROP_FILTER_MAX G_MAXINT32 +#define PROP_FILTER_DEFAULT PROP_FILTER_MIN +#define PROP_SCALER_DEFAULT "videoscale" +#define PROP_CONVERTER_DEFAULT "videoconvert" + +struct _GstInferenceBin +{ + GstBin parent; + + gchar *arch; + gchar *backend; + gchar *model_location; + gchar *input_layer; + gchar *output_layer; + gchar *labels; + gboolean crop; + gboolean overlay; + guint filter; + gchar *scaler; + gchar *converter; + + GstPad *sinkpad; + GstPad *srcpad; +}; + +struct _GstInferenceBinClass +{ + GstBinClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferenceBin, gst_inference_bin, + GST_TYPE_BIN, + GST_DEBUG_CATEGORY_INIT (gst_inference_bin_debug_category, "inferencebin", + 0, "debug category for inferencebin element")); + +static void +gst_inference_bin_class_init (GstInferenceBinClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_template)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_template)); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "inferencebin", "Filter", + "A bin with the inference element in their typical configuration", + "Michael Gruner "); + + element_class->change_state = + GST_DEBUG_FUNCPTR (gst_inference_bin_change_state); + + object_class->finalize = gst_inference_bin_finalize; + object_class->set_property = gst_inference_bin_set_property; + object_class->get_property = gst_inference_bin_get_property; + + g_object_class_install_property (object_class, PROP_ARCH, + g_param_spec_string ("arch", "Architecture", + "The factory name of the network architecture to use for inference", + PROP_ARCH_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_BACKEND, + g_param_spec_string ("backend", "Backend", + "The backend to use as the inference engine", + PROP_BACKEND_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_MODEL_LOCATION, + g_param_spec_string ("model-location", "Model location", + "The location of the model to use for the inference", + PROP_MODEL_LOCATION_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_INPUT_LAYER, + g_param_spec_string ("input-layer", "Model input", + "The name of the input of the model", + PROP_INPUT_LAYER_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_OUTPUT_LAYER, + g_param_spec_string ("output-layer", "Model output", + "The name of the output of the model", + PROP_OUTPUT_LAYER_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_LABELS, + g_param_spec_string ("labels", "Model labels", + "The labels used to train the model", + PROP_LABELS_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_CROP, + g_param_spec_boolean ("crop", "Crop", + "Whether or not to crop out objects in the current prediction", + PROP_CROP_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_OVERLAY, + g_param_spec_boolean ("overlay", "Overlay", + "Whether or not to overlay predictions on the buffers", + PROP_OVERLAY_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_FILTER, + g_param_spec_int ("filter", "Inference Filter", + "The filter to apply to the inference (-1 disables).", + PROP_FILTER_MIN, PROP_FILTER_MAX, PROP_FILTER_DEFAULT, + G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_SCALER, + g_param_spec_string ("scaler", "Video Scaler", + "Bin description to use as video scaler", + PROP_SCALER_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_CONVERTER, + g_param_spec_string ("converter", "Color Space Converter", + "Bin description to use as color space converter", + PROP_CONVERTER_DEFAULT, G_PARAM_READWRITE)); +} + +static void +gst_inference_bin_init (GstInferenceBin * self) +{ + self->arch = g_strdup (PROP_ARCH_DEFAULT); + self->backend = g_strdup (PROP_BACKEND_DEFAULT); + self->model_location = g_strdup (PROP_MODEL_LOCATION_DEFAULT); + self->input_layer = g_strdup (PROP_INPUT_LAYER_DEFAULT); + self->output_layer = g_strdup (PROP_OUTPUT_LAYER_DEFAULT); + self->labels = g_strdup (PROP_LABELS_DEFAULT); + self->crop = PROP_CROP_DEFAULT; + self->overlay = PROP_OVERLAY_DEFAULT; + self->filter = PROP_FILTER_DEFAULT; + self->scaler = g_strdup (PROP_SCALER_DEFAULT); + self->converter = g_strdup (PROP_CONVERTER_DEFAULT); + + self->sinkpad = + gst_ghost_pad_new_no_target_from_template (NULL, + gst_static_pad_template_get (&sink_template)); + + gst_pad_set_active (self->sinkpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), self->sinkpad); + + self->srcpad = + gst_ghost_pad_new_no_target_from_template (NULL, + gst_static_pad_template_get (&src_template)); + gst_pad_set_active (self->srcpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), self->srcpad); +} + +static void +gst_inference_bin_finalize (GObject * object) +{ + GstInferenceBin *self = GST_INFERENCE_BIN (object); + + g_free (self->arch); + self->arch = NULL; + + g_free (self->backend); + self->backend = NULL; + + g_free (self->model_location); + self->model_location = NULL; + + g_free (self->input_layer); + self->input_layer = NULL; + + g_free (self->output_layer); + self->output_layer = NULL; + + g_free (self->labels); + self->labels = NULL; + + g_free (self->scaler); + self->scaler = NULL; + + g_free (self->converter); + self->converter = NULL; + + G_OBJECT_CLASS (gst_inference_bin_parent_class)->finalize (object); +} + +static void +gst_inference_bin_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferenceBin *self = GST_INFERENCE_BIN (object); + + GST_LOG_OBJECT (self, "set_property"); + + GST_OBJECT_LOCK (self); + + switch (property_id) { + case PROP_ARCH: + g_free (self->arch); + self->arch = g_value_dup_string (value); + break; + case PROP_BACKEND: + g_free (self->backend); + self->backend = g_value_dup_string (value); + break; + case PROP_MODEL_LOCATION: + g_free (self->model_location); + self->model_location = g_value_dup_string (value); + break; + case PROP_INPUT_LAYER: + g_free (self->input_layer); + self->input_layer = g_value_dup_string (value); + break; + case PROP_OUTPUT_LAYER: + g_free (self->output_layer); + self->output_layer = g_value_dup_string (value); + break; + case PROP_LABELS: + g_free (self->labels); + self->labels = g_value_dup_string (value); + break; + case PROP_CROP: + self->crop = g_value_get_boolean (value); + break; + case PROP_OVERLAY: + self->overlay = g_value_get_boolean (value); + break; + case PROP_FILTER: + self->filter = g_value_get_int (value); + break; + case PROP_SCALER: + g_free (self->scaler); + self->scaler = g_value_dup_string (value); + break; + case PROP_CONVERTER: + g_free (self->converter); + self->converter = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } + + GST_OBJECT_UNLOCK (self); +} + +static void +gst_inference_bin_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferenceBin *self = GST_INFERENCE_BIN (object); + + GST_LOG_OBJECT (self, "get_property"); + + GST_OBJECT_LOCK (self); + + switch (property_id) { + case PROP_ARCH: + g_value_set_string (value, self->arch); + break; + case PROP_BACKEND: + g_value_set_string (value, self->backend); + break; + case PROP_MODEL_LOCATION: + g_value_set_string (value, self->model_location); + break; + case PROP_INPUT_LAYER: + g_value_set_string (value, self->input_layer); + break; + case PROP_OUTPUT_LAYER: + g_value_set_string (value, self->output_layer); + break; + case PROP_LABELS: + g_value_set_string (value, self->labels); + break; + case PROP_CROP: + g_value_set_boolean (value, self->crop); + break; + case PROP_OVERLAY: + g_value_set_boolean (value, self->overlay); + break; + case PROP_FILTER: + g_value_set_int (value, self->filter); + break; + case PROP_SCALER: + g_value_set_string (value, self->scaler); + break; + case PROP_CONVERTER: + g_value_set_string (value, self->converter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } + + GST_OBJECT_UNLOCK (self); +} + +static gchar * +gst_inference_bin_build_pipe (GstInferenceBin * self) +{ + GString *desc = NULL; + const gchar *crop = NULL; + const gchar *overlay = NULL; + + g_return_val_if_fail (self, FALSE); + + crop = self->crop ? "true" : "false"; + overlay = self->overlay ? "true" : "false"; + + desc = g_string_new (NULL); + + g_string_append_printf (desc, "inferencefilter filter-class=%d name=filter " + "! ", self->filter); + g_string_append (desc, "inferencedebug name=debug_before ! "); + g_string_append_printf (desc, "%s name=converter_before ! ", self->converter); + g_string_append (desc, "tee name=tee "); + g_string_append (desc, + "tee. ! queue max-size-buffers=3 leaky=no name=queue_bypass ! " + "arch.sink_bypass "); + g_string_append_printf (desc, "tee. ! queue max-size-buffers=3 leaky=no " + "name=queue_sink ! inferencecrop enable=%s name=crop ! ", crop); + g_string_append_printf (desc, "%s name=scaler ! arch.sink_model ", + self->scaler); + g_string_append_printf (desc, "%s name=arch backend=%s model-location=%s ", + self->arch, self->backend, self->model_location); + + if (self->labels) { + g_string_append_printf (desc, "labels=%s ", self->labels); + } + + if (self->input_layer) { + g_string_append_printf (desc, "backend::input-layer=%s ", + self->input_layer); + } + + if (self->output_layer) { + g_string_append_printf (desc, "backend::output-layer=%s ", + self->output_layer); + } + + g_string_append (desc, "arch.src_bypass ! queue name=queue_output ! " + "inferencedebug name=debug_after "); + + g_string_append_printf (desc, "! inferenceoverlay enable=%s name=overlay ", + overlay); + + return g_string_free (desc, FALSE); +} + +static gboolean +gst_inference_bin_start (GstInferenceBin * self) +{ + gboolean ret = FALSE; + GstElement *bin = NULL; + GError *error = NULL; + GstPad *sinkpad = NULL; + GstPad *srcpad = NULL; + gchar *pipe = NULL; + + g_return_val_if_fail (self, FALSE); + + GST_OBJECT_LOCK (self); + pipe = gst_inference_bin_build_pipe (self); + GST_OBJECT_UNLOCK (self); + + GST_INFO_OBJECT (self, "Attempting to build \"%s\"", pipe); + + bin = + gst_parse_bin_from_description_full (pipe, TRUE, NULL, + GST_PARSE_FLAG_FATAL_ERRORS, &error); + + g_free (pipe); + + if (!bin) { + GST_ERROR_OBJECT (self, "Unable to create bin: %s", error->message); + g_error_free (error); + goto out; + } + + if (!gst_bin_add (GST_BIN (self), bin)) { + GST_ERROR_OBJECT (self, "Unable to add to bin"); + gst_object_unref (bin); + goto out; + } + + sinkpad = gst_element_get_static_pad (bin, "sink"); + srcpad = gst_element_get_static_pad (bin, "src"); + + g_return_val_if_fail (sinkpad, FALSE); + g_return_val_if_fail (srcpad, FALSE); + + gst_ghost_pad_set_target (GST_GHOST_PAD (self->sinkpad), sinkpad); + gst_ghost_pad_set_target (GST_GHOST_PAD (self->srcpad), srcpad); + + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + GST_INFO_OBJECT (self, "Created bin successfully"); + ret = TRUE; + +out: + return ret; +} + +static gboolean +gst_inference_bin_stop (GstInferenceBin * self) +{ + g_return_val_if_fail (self, FALSE); + + return TRUE; +} + +static GstStateChangeReturn +gst_inference_bin_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret; + GstInferenceBin *self = GST_INFERENCE_BIN (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + if (FALSE == gst_inference_bin_start (self)) { + GST_ERROR_OBJECT (self, "Failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + default: + break; + } + + ret = + GST_ELEMENT_CLASS (gst_inference_bin_parent_class)->change_state + (element, transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + if (FALSE == gst_inference_bin_stop (self)) { + GST_ERROR_OBJECT (self, "Failed to stop"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + default: + break; + } + +out: + return ret; +} diff --git a/gst/inferenceutils/gstinferencebin.h b/gst/inferenceutils/gstinferencebin.h new file mode 100644 index 00000000..0ead8aea --- /dev/null +++ b/gst/inferenceutils/gstinferencebin.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_BIN_H__ +#define __GST_INFERENCE_BIN_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_INFERENCE_BIN gst_inference_bin_get_type () +G_DECLARE_FINAL_TYPE (GstInferenceBin, gst_inference_bin, GST, INFERENCE_BIN, + GstBin) + +G_END_DECLS + +#endif //__GST_INFERENCE_BIN_H__ diff --git a/gst/inferenceutils/gstinferencecrop.cc b/gst/inferenceutils/gstinferencecrop.cc index 3b8f14f8..ba8a4df0 100644 --- a/gst/inferenceutils/gstinferencecrop.cc +++ b/gst/inferenceutils/gstinferencecrop.cc @@ -82,13 +82,15 @@ static void gst_inference_crop_find_predictions (GstInferenceCrop *self, gint *num_inferences, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); - #define PROP_CROP_RATIO_DEFAULT_WIDTH 1 #define PROP_CROP_RATIO_DEFAULT_HEIGHT 1 +#define PROP_ENABLE_DEFAULT TRUE -enum { +enum +{ PROP_0, PROP_CROP_ASPECT_RATIO, + PROP_ENABLE, }; struct _GstInferenceCrop { @@ -100,6 +102,7 @@ struct _GstInferenceCrop { gint height_ratio; gint width; gint height; + gboolean enable; }; struct _GstInferenceCropClass { @@ -119,9 +122,9 @@ gst_inference_crop_class_init (GstInferenceCropClass *klass) { GObjectClass *object_class = G_OBJECT_CLASS (klass); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_template)); + gst_static_pad_template_get (&sink_template)); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&src_template)); + gst_static_pad_template_get (&src_template)); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "Inference Crop", "Filter", @@ -143,6 +146,11 @@ gst_inference_crop_class_init (GstInferenceCropClass *klass) { G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, G_PARAM_READWRITE )); + + g_object_class_install_property (object_class, PROP_ENABLE, + g_param_spec_boolean ("enable", "Enable Crop", + "Whether or not no crop out subpredictions", + PROP_ENABLE_DEFAULT, G_PARAM_READWRITE)); } static void @@ -154,6 +162,8 @@ gst_inference_crop_init (GstInferenceCrop *self) { self->element = new VideoCrop (); self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; + self->enable = PROP_ENABLE; + if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); @@ -166,7 +176,7 @@ gst_inference_crop_init (GstInferenceCrop *self) { sinkpad = self->element->GetSinkPad (); g_return_if_fail (sinkpad); - self->sinkpad = GST_PAD(gst_object_ref(sinkpad)); + self->sinkpad = GST_PAD (gst_object_ref (sinkpad)); sinkgpad = gst_ghost_pad_new ("sink", sinkpad); gst_pad_set_active (sinkgpad, TRUE); @@ -180,7 +190,7 @@ gst_inference_crop_init (GstInferenceCrop *self) { srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); - self->srcpad = GST_PAD(gst_object_ref(srcpad)); + self->srcpad = GST_PAD (gst_object_ref (srcpad)); srcgpad = gst_ghost_pad_new ("src", srcpad); gst_pad_set_active (srcgpad, TRUE); @@ -194,7 +204,7 @@ static void gst_inference_crop_finalize (GObject *object) { GstInferenceCrop *self = GST_INFERENCE_CROP (object); - g_return_if_fail(self); + g_return_if_fail (self); delete (self->element); gst_object_unref (self->sinkpad); @@ -221,6 +231,11 @@ gst_inference_crop_set_property (GObject *object, guint property_id, } GST_OBJECT_UNLOCK (self); break; + case PROP_ENABLE: + GST_OBJECT_LOCK (self); + self->enable = g_value_get_boolean (value); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -240,6 +255,11 @@ gst_inference_crop_get_property (GObject *object, guint property_id, gst_value_set_fraction (value, self->width_ratio, self->height_ratio); GST_OBJECT_UNLOCK (self); break; + case PROP_ENABLE: + GST_OBJECT_LOCK (self); + g_value_set_boolean (value, self->enable); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -301,7 +321,7 @@ gst_inference_crop_set_caps (GstPad *pad, GParamSpec *unused, GST_INFO_OBJECT (self, "Set new caps to %" GST_PTR_FORMAT, caps); self->width = width; self->height = height; - gst_caps_unref(caps); + gst_caps_unref (caps); } static void @@ -315,10 +335,10 @@ gst_inference_crop_new_buffer_size (GstInferenceCrop *self, gint x, gint y, *right = self->width - x - width; if (width_ratio > 0 && height_ratio > 0) { - gint top_bottom_modify = round(((height_ratio * width) / width_ratio - height) / - 2); - gint left_right_modify = round(((width_ratio * height) / height_ratio - width) / - 2); + gint top_bottom_modify = + round (((height_ratio * width) / width_ratio - height) / 2); + gint left_right_modify = + round (((width_ratio * height) / height_ratio - width) / 2); if (width_ratio <= height_ratio) { if (width > height) { *top = *top - top_bottom_modify; @@ -369,15 +389,15 @@ gst_inference_crop_find_predictions (GstInferenceCrop *self, g_return_if_fail (list); g_return_if_fail (pred); - children_list = gst_inference_prediction_get_children(pred); + children_list = gst_inference_prediction_get_children (pred); - for (iter = children_list; iter != NULL; iter = g_slist_next(iter)) { - GstInferencePrediction *predict = (GstInferencePrediction *)iter->data; + for (iter = children_list; iter != NULL; iter = g_slist_next (iter)) { + GstInferencePrediction *predict = (GstInferencePrediction *) iter->data; gst_inference_crop_find_predictions (self, num_inferences, meta, list, predict ); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { + if (FALSE == G_NODE_IS_ROOT (pred->predictions) && TRUE == pred->enabled) { *list = g_list_append (*list, pred); *num_inferences = *num_inferences + 1; } @@ -397,19 +417,28 @@ gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GList *list = NULL; GList *iter = NULL; gboolean gap = TRUE; + gboolean enable = FALSE; GST_OBJECT_LOCK (self); crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; + enable = self->enable; GST_OBJECT_UNLOCK (self); + if (FALSE == enable) { + GST_LOG_OBJECT (self, "Cropping disabled, forwarding buffer"); + gap = FALSE; + ret = GST_PAD_PROBE_OK; + goto out; + } + buffer = gst_pad_probe_info_get_buffer (info); inference_meta = - (GstInferenceMeta *) gst_buffer_get_meta (buffer, - GST_INFERENCE_META_API_TYPE); + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + GST_INFERENCE_META_API_TYPE); - if (NULL == inference_meta ) { + if (NULL == inference_meta) { GST_LOG_OBJECT (self, "No meta found, dropping buffer"); goto out; } @@ -418,20 +447,20 @@ gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, gst_inference_crop_find_predictions (self, &num_inferences, inference_meta, &list, inference_meta->prediction); - for (iter = list; iter != NULL; iter = g_list_next(iter)) { - GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; + for (iter = list; iter != NULL; iter = g_list_next (iter)) { + GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; GstBuffer *croped_buffer; GstInferenceMeta *dmeta; gint top, bottom, right, left = 0; box = pred->bbox; GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, - box.height); + box.height); gst_inference_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, crop_width_ratio, crop_height_ratio, &top, &bottom, &right, &left); self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, - (gint) left); + (gint) left); croped_buffer = gst_buffer_copy (buffer); @@ -447,9 +476,9 @@ gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, dmeta->prediction->bbox.width = self->width - right - left; dmeta->prediction->bbox.height = self->height - top - bottom; - if (GST_FLOW_OK != gst_pad_chain(self->sinkpad, croped_buffer)) { - GST_ELEMENT_ERROR(self, CORE, FAILED, - ("Failed to push a new buffer into crop element"), (NULL)); + if (GST_FLOW_OK != gst_pad_chain (self->sinkpad, croped_buffer)) { + GST_ELEMENT_ERROR (self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); } gap = FALSE; } @@ -458,8 +487,9 @@ gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, out: if (gap) { - gst_pad_push_event (self->srcpad, gst_event_new_gap (GST_BUFFER_TIMESTAMP (buffer), - GST_BUFFER_DURATION (buffer))); + gst_pad_push_event (self->srcpad, + gst_event_new_gap (GST_BUFFER_TIMESTAMP (buffer), + GST_BUFFER_DURATION (buffer))); } return ret; diff --git a/gst/inferenceutils/gstinferenceutils.c b/gst/inferenceutils/gstinferenceutils.c index 460a0c92..a7b3f930 100644 --- a/gst/inferenceutils/gstinferenceutils.c +++ b/gst/inferenceutils/gstinferenceutils.c @@ -23,6 +23,7 @@ #include "config.h" #endif +#include "gstinferencebin.h" #include "gstinferencecrop.h" #include "gstinferencedebug.h" #include "gstinferencefilter.h" @@ -32,6 +33,13 @@ plugin_init (GstPlugin * plugin) { gboolean ret = TRUE; + ret = + gst_element_register (plugin, "inferencebin", GST_RANK_NONE, + GST_TYPE_INFERENCE_BIN); + if (!ret) { + goto out; + } + ret = gst_element_register (plugin, "inferencecrop", GST_RANK_NONE, GST_TYPE_INFERENCE_CROP);