diff --git a/.gitignore b/.gitignore index 6bcf7c25..fb032452 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ m4/ .libs/ stamp-h* autoregen.sh -gstinference-*.pc +gst-inference-*.pc # Build outputs .deps/ diff --git a/COPYING b/COPYING index ee391442..fb6e92e3 100644 --- a/COPYING +++ b/COPYING @@ -1,16 +1,3 @@ -GstInference copyright (C) 2019 RidgeRun LLC - -This GStreamer plug-in is free software; you can redistribute it -and/or modify it under the terms of the GNU Lesser General Public -License version 2.1 as published by the Free Software Foundation. - -This GStreamer plug-in is distributed in the hope that it will be -useful, but WITHOUT ANY WARRANTY; without even the implied -warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -See the GNU Lesser General Public License below for more details. - ----------------------------------------------------------------------- - GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 diff --git a/Makefile.am b/Makefile.am index a33a6965..fcdc4dde 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,6 +2,7 @@ DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc ALWAYS_SUBDIRS = \ gst-libs \ + gst \ ext \ tests \ common \ @@ -54,3 +55,8 @@ CRUFT_DIRS = include $(top_srcdir)/common/cruft.mak all-local: check-cruft + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = gst-inference-@GST_API_VERSION@.pc + +EXTRA_DIST = gst-inference.pc.in diff --git a/configure.ac b/configure.ac index d5441457..87a0819f 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ dnl please read gstreamer/docs/random/autotools before changing this file dnl initialize autoconf dnl releases only do -Wall, git and prerelease does -Werror too dnl use a three digit version number for releases, and four for git/pre -AC_INIT([GStreamer Inference],[0.6.1.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) +AC_INIT([GStreamer Inference],[0.7.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) AG_GST_INIT @@ -20,7 +20,6 @@ AS_NANO(GST_GIT="no", GST_GIT="yes") dnl can autoconf find the source ? AC_CONFIG_SRCDIR([ext/r2inference/gstinference.c]) -AC_CONFIG_SRCDIR([ext/opencv/gstinferenceoverlay.c]) dnl define the output header for config AC_CONFIG_HEADERS([config.h]) @@ -195,7 +194,7 @@ AC_ARG_ENABLE(Bsymbolic, dnl *** set variables based on configure arguments *** dnl set license and copyright notice -GST_LICENSE="Proprietary" +GST_LICENSE="LGPL" AC_DEFINE_UNQUOTED(GST_LICENSE, "$GST_LICENSE", [GStreamer Inference license]) AC_SUBST(GST_LICENSE) @@ -289,8 +288,9 @@ if test "$USE_OPENCV" = "yes"; then fi dnl *** r2inference *** +R2INFERENCE_REQ=0.4.0 AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun\'s Inference Framework], r2inference, [ - AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0) + AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0 >= $R2INFERENCE_REQ) ],[],[],[ AC_MSG_ERROR([Please install R2Inference from https://github.com/RidgeRun/r2inference.git]) ]) @@ -367,6 +367,9 @@ AC_SUBST(GST_PLUGIN_LDFLAGS) dnl *** output files *** +# Our Include path, to be shared in multiple makefiles +AC_SUBST([GSTINCLUDEDIR],[$includedir/gstreamer-1.0]) + dnl keep this alphabetic per directory, please AC_CONFIG_FILES( Makefile @@ -378,12 +381,17 @@ docs/version.entities ext/Makefile ext/opencv/Makefile ext/r2inference/Makefile +gst/Makefile +gst/inferencecrop/Makefile +gst/inferencefilter/Makefile +gst/inferencedebug/Makefile gst-libs/Makefile gst-libs/gst/Makefile gst-libs/gst/opencv/Makefile gst-libs/gst/r2inference/Makefile m4/Makefile tests/check/Makefile +gst-inference-${GST_API_VERSION}.pc:gst-inference.pc.in tests/Makefile tests/examples/classification/Makefile tests/examples/detection/Makefile @@ -394,4 +402,3 @@ tests/files/Makefile AC_OUTPUT AG_GST_OUTPUT_PLUGINS - diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index 8c001671..7762f6a6 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -1,12 +1,13 @@ -plugin_LTLIBRARIES = libgstinferenceoverlay.la +plugin_LTLIBRARIES = libgstinferenceoverlayplugin.la -libgstinferenceoverlay_la_SOURCES = \ +libgstinferenceoverlayplugin_la_SOURCES = \ gstclassificationoverlay.cc \ gstdetectionoverlay.cc \ - gstinferenceoverlay.c \ + gstplugin.cc \ + gstinferenceoverlay.cc \ gstembeddingoverlay.cc -libgstinferenceoverlay_la_CFLAGS = \ +libgstinferenceoverlayplugin_la_CFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -15,7 +16,7 @@ libgstinferenceoverlay_la_CFLAGS = \ -I$(top_srcdir)/gst-libs -libgstinferenceoverlay_la_CXXFLAGS = \ +libgstinferenceoverlayplugin_la_CXXFLAGS = \ $(GST_CXXFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -25,7 +26,7 @@ libgstinferenceoverlay_la_CXXFLAGS = \ -std=c++11 -libgstinferenceoverlay_la_LIBADD = \ +libgstinferenceoverlayplugin_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ @@ -33,15 +34,16 @@ libgstinferenceoverlay_la_LIBADD = \ $(OPENCV_LIBS) \ $(R2INFERENCE_LIBS) \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la \ - $(top_builddir)/gst-libs/gst/opencv/libgstinferenceoverlay-@GST_API_VERSION@.la + $(top_builddir)/gst-libs/gst/opencv/libgstinferencebaseoverlay-@GST_API_VERSION@.la -libgstinferenceoverlay_la_LDFLAGS = \ +libgstinferenceoverlayplugin_la_LDFLAGS = \ $(GST_PLUGIN_LDFLAGS) -libgstinferenceoverlay_la_LIBTOOLFLAGS = \ +libgstinferenceoverlayplugin_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ gstclassificationoverlay.h \ gstdetectionoverlay.h \ + gstinferenceoverlay.h \ gstembeddingoverlay.h diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 456a49a0..8b54c2f6 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,30 +18,28 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstclassificationoverlay.h" #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif +/* *INDENT-OFF* */ static const cv::Scalar black = cv::Scalar (0, 0, 0, 0); static const cv::Scalar white = cv::Scalar (255, 255, 255, 255); +/* *INDENT-ON* */ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); #define GST_CAT_DEFAULT gst_classification_overlay_debug_category /* prototypes */ static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceOverlay * - inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels); +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * + inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style); enum { @@ -50,18 +48,18 @@ enum struct _GstClassificationOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; struct _GstClassificationOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_classification_overlay_debug_category, "classificationoverlay", 0, "debug category for classification_overlay element")); @@ -69,7 +67,8 @@ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, static void gst_classification_overlay_class_init (GstClassificationOverlayClass * klass) { - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "classificationoverlay", "Filter", @@ -93,29 +92,17 @@ gst_classification_overlay_init (GstClassificationOverlay * } static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceOverlay * - inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels) +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * + inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { GstClassificationMeta *class_meta; - gint index, i, width, height, channels; + gint index, i; gdouble max, current; - cv::Mat cv_mat; cv::String str; cv::Size size; - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - class_meta = (GstClassificationMeta *) meta; /* Get the most probable label */ @@ -133,16 +120,15 @@ gst_classification_overlay_process_meta (GstInferenceOverlay * } else { str = cv::format ("Label #%d prob:%f", index, max); } - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); + /* Put string on screen * 10*font_scale+16 aproximates text's rendered size on screen as a * lineal function to avoid using cv::getTextSize */ - cv::putText (cv_mat, str, cv::Point (0, 10*font_scale+16), cv::FONT_HERSHEY_PLAIN, - font_scale, white, thickness + (thickness*0.5)); - cv::putText (cv_mat, str, cv::Point (0, 10*font_scale+16), cv::FONT_HERSHEY_PLAIN, - font_scale, black, thickness); + cv::putText (cv_mat, str, cv::Point (0, 10 * font_scale + 16), + cv::FONT_HERSHEY_PLAIN, font_scale, white, thickness + (thickness * 0.5)); + cv::putText (cv_mat, str, cv::Point (0, 10 * font_scale + 16), + cv::FONT_HERSHEY_PLAIN, font_scale, black, thickness); return GST_FLOW_OK; } diff --git a/ext/opencv/gstclassificationoverlay.h b/ext/opencv/gstclassificationoverlay.h index 214554cd..fe31be7d 100644 --- a/ext/opencv/gstclassificationoverlay.h +++ b/ext/opencv/gstclassificationoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -22,12 +22,12 @@ #ifndef _GST_CLASSIFICATION_OVERLAY_H_ #define _GST_CLASSIFICATION_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_CLASSIFICATION_OVERLAY (gst_classification_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstClassificationOverlay, gst_classification_overlay, GST, CLASSIFICATION_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstClassificationOverlay, gst_classification_overlay, GST, CLASSIFICATION_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index 164a1bbb..1ee31c5b 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,19 +18,15 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstdetectionoverlay.h" #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif +/* *INDENT-OFF* */ static const cv::Scalar colors[] = { cv::Scalar (254, 254, 254), cv::Scalar (239, 211, 127), cv::Scalar (225, 169, 0), cv::Scalar (211, 127, 254), @@ -43,16 +39,18 @@ static const cv::Scalar colors[] = { cv::Scalar (28, 42, 127), cv::Scalar (14, 84, 0), cv::Scalar (0, 254, 254), cv::Scalar (14, 211, 127) }; +/* *INDENT-ON* */ + #define N_C (sizeof (colors)/sizeof (cv::Scalar)) GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); #define GST_CAT_DEFAULT gst_detection_overlay_debug_category /* prototypes */ -static GstFlowReturn -gst_detection_overlay_process_meta (GstInferenceOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels); +static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, + gint num_labels, LineStyleBoundingBox style); enum { @@ -61,25 +59,26 @@ enum struct _GstDetectionOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; struct _GstDetectionOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstDetectionOverlay, gst_detection_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_detection_overlay_debug_category, "detectionoverlay", 0, "debug category for detection_overlay element")); static void gst_detection_overlay_class_init (GstDetectionOverlayClass * klass) { - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "detectionoverlay", "Filter", @@ -102,32 +101,18 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) } static GstFlowReturn -gst_detection_overlay_process_meta (GstInferenceOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels) +gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { GstDetectionMeta *detect_meta; - gint i, width, height, channels; - cv::Mat cv_mat; + gint i; cv::Size size; cv::String str; BBox box; - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - detect_meta = (GstDetectionMeta *) meta; - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); for (i = 0; i < detect_meta->num_boxes; ++i) { box = detect_meta->boxes[i]; if (num_labels > box.label) { diff --git a/ext/opencv/gstdetectionoverlay.h b/ext/opencv/gstdetectionoverlay.h index ee1b8629..c5878fdd 100644 --- a/ext/opencv/gstdetectionoverlay.h +++ b/ext/opencv/gstdetectionoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -22,12 +22,12 @@ #ifndef _GST_DETECTION_OVERLAY_H_ #define _GST_DETECTION_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_DETECTION_OVERLAY (gst_detection_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstDetectionOverlay, gst_detection_overlay, GST, DETECTION_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstDetectionOverlay, gst_detection_overlay, GST, DETECTION_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 2e9f3f93..5d7638c9 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,18 +18,14 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstembeddingoverlay.h" + #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif #define DEFAULT_EMBEDDINGS NULL #define DEFAULT_NUM_EMBEDDINGS 0 @@ -37,9 +33,11 @@ #define MAX_LIKENESS_THRESH G_MAXDOUBLE #define DEFAULT_LIKENESS_THRESH 1.0 +/* *INDENT-OFF* */ static const cv::Scalar forest_green = cv::Scalar (11, 102, 35); static const cv::Scalar chilli_red = cv::Scalar (194, 24, 7); static const cv::Scalar white = cv::Scalar (255, 255, 255, 255); +/* *INDENT-ON* */ GST_DEBUG_CATEGORY_STATIC (gst_embedding_overlay_debug_category); #define GST_CAT_DEFAULT gst_embedding_overlay_debug_category @@ -50,13 +48,12 @@ static void gst_embedding_overlay_set_property (GObject * object, static void gst_embedding_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); static void gst_embedding_overlay_finalize (GObject * object); -static GstFlowReturn -gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels); -static gboolean -gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, - const GValue * value); +static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, + gint num_labels, LineStyleBoundingBox style); +static gboolean gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * + embedding_overlay, const GValue * value); enum { @@ -67,7 +64,7 @@ enum struct _GstEmbeddingOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; gchar *embeddings; gchar **embeddings_list; gint num_embeddings; @@ -77,13 +74,13 @@ struct _GstEmbeddingOverlay struct _GstClassificationOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstEmbeddingOverlay, gst_embedding_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_embedding_overlay_debug_category, "embeddingoverlay", 0, "debug category for embedding_overlay element")); @@ -91,7 +88,8 @@ static void gst_embedding_overlay_class_init (GstEmbeddingOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gobject_class->set_property = gst_embedding_overlay_set_property; gobject_class->get_property = gst_embedding_overlay_get_property; @@ -143,20 +141,20 @@ gst_embedding_overlay_set_property (GObject * object, guint property_id, switch (property_id) { case PROP_EMBEDDINGS: - if (gst_embedding_overlay_set_embeddings(embedding_overlay, value)){ + if (gst_embedding_overlay_set_embeddings (embedding_overlay, value)) { GST_DEBUG_OBJECT (embedding_overlay, "Changed inference labels %s", - embedding_overlay->embeddings); + embedding_overlay->embeddings); } else { GST_ERROR_OBJECT (embedding_overlay, "Failed setting embeddings"); } break; case PROP_LIKENESS_THRESH: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); embedding_overlay->likeness_thresh = g_value_get_double (value); GST_DEBUG_OBJECT (embedding_overlay, "Changed likeness threshold to %lf", embedding_overlay->likeness_thresh); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -174,14 +172,14 @@ gst_embedding_overlay_get_property (GObject * object, guint property_id, switch (property_id) { case PROP_EMBEDDINGS: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); g_value_set_string (value, embedding_overlay->embeddings); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; case PROP_LIKENESS_THRESH: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); g_value_set_double (value, embedding_overlay->likeness_thresh); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -208,15 +206,16 @@ gst_embedding_overlay_finalize (GObject * object) } static GstFlowReturn -gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels) +gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { - GstEmbeddingOverlay *embedding_overlay = GST_EMBEDDING_OVERLAY (inference_overlay); + GstEmbeddingOverlay *embedding_overlay = + GST_EMBEDDING_OVERLAY (inference_overlay); GstClassificationMeta *class_meta; - gint i, j, width, height, channels; + gint i, j, width, height; gdouble current, diff; - cv::Mat cv_mat; cv::String str; cv::Size size; cv::Scalar tmp_color; @@ -227,36 +226,23 @@ gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, gint embedding_size; gdouble likeness_thresh; - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); embeddings_list = g_strdupv (embedding_overlay->embeddings_list); num_embeddings = embedding_overlay->num_embeddings; embedding_size = embedding_overlay->embedding_size; likeness_thresh = embedding_overlay->likeness_thresh; - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); - if (num_embeddings == 0){ + if (num_embeddings == 0) { GST_WARNING_OBJECT (embedding_overlay, "Please set at least one valid embedding using the 'embeddings'" "property"); goto end; } - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - class_meta = (GstClassificationMeta *) meta; - if (class_meta->num_labels != embedding_size){ + if (class_meta->num_labels != embedding_size) { GST_WARNING_OBJECT (embedding_overlay, "Provided embeddings and inference output have different sizes"); goto end; @@ -270,8 +256,7 @@ gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, diff = 0.0; for (j = 0; j < embedding_size; ++j) { current = class_meta->label_probs[j]; - current -= - atof (embeddings_list[i * embedding_size + j]); + current -= atof (embeddings_list[i * embedding_size + j]); current = current * current; diff = diff + current; } @@ -314,22 +299,25 @@ gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, break; } - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); + width = cv_mat.cols; + height = cv_mat.rows; + /* Put string on screen * 10*font_scale+16 aproximates text's rendered size on screen as a * lineal function to avoid using cv::getTextSize. * 5*thickness adds the border offset */ - cv::putText (cv_mat, str, cv::Point (5*thickness, 5*thickness+10*font_scale+16), - cv::FONT_HERSHEY_PLAIN, font_scale, white, thickness + (thickness*0.5)); - cv::putText (cv_mat, str, cv::Point (5*thickness, 5*thickness+10*font_scale+16), - cv::FONT_HERSHEY_PLAIN, font_scale, color, thickness); + cv::putText (cv_mat, str, cv::Point (5 * thickness, + 5 * thickness + 10 * font_scale + 16), cv::FONT_HERSHEY_PLAIN, + font_scale, white, thickness + (thickness * 0.5)); + cv::putText (cv_mat, str, cv::Point (5 * thickness, + 5 * thickness + 10 * font_scale + 16), cv::FONT_HERSHEY_PLAIN, + font_scale, color, thickness); cv::rectangle (cv_mat, cv::Point (0, 0), cv::Point (width, height), color, - 10*thickness); + 10 * thickness); end: - g_strfreev(embeddings_list); + g_strfreev (embeddings_list); return GST_FLOW_OK; } @@ -340,7 +328,7 @@ gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, g_return_val_if_fail (embedding_overlay, FALSE); g_return_val_if_fail (value, FALSE); - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); if (embedding_overlay->embeddings != NULL) { g_free (embedding_overlay->embeddings); } @@ -358,7 +346,7 @@ gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, embedding_overlay->embedding_size = g_strv_length (embedding_overlay->embeddings_list) / embedding_overlay->num_embeddings; - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); return TRUE; } diff --git a/ext/opencv/gstembeddingoverlay.h b/ext/opencv/gstembeddingoverlay.h index 4915435d..3a2b503e 100644 --- a/ext/opencv/gstembeddingoverlay.h +++ b/ext/opencv/gstembeddingoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -22,12 +22,12 @@ #ifndef _GST_EMBEDDING_OVERLAY_H_ #define _GST_EMBEDDING_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_EMBEDDING_OVERLAY (gst_embedding_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstEmbeddingOverlay, gst_embedding_overlay, GST, EMBEDDING_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstEmbeddingOverlay, gst_embedding_overlay, GST, EMBEDDING_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc new file mode 100644 index 00000000..81e5cd2d --- /dev/null +++ b/ext/opencv/gstinferenceoverlay.cc @@ -0,0 +1,332 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferenceoverlay.h" +#include "gst/r2inference/gstinferencemeta.h" + +#define DEFAULT_LABELS NULL + +enum +{ + PROP_0, + PROP_LABELS +}; + +/* *INDENT-OFF* */ +static const cv::Scalar colors[] = { + cv::Scalar (254, 254, 254), cv::Scalar (239, 211, 127), + cv::Scalar (225, 169, 0), cv::Scalar (211, 127, 254), + cv::Scalar (197, 84, 127), cv::Scalar (183, 42, 0), + cv::Scalar (169, 0.0, 254), cv::Scalar (155, 42, 127), + cv::Scalar (141, 84, 0), cv::Scalar (127, 254, 254), + cv::Scalar (112, 211, 127), cv::Scalar (98, 169, 0), + cv::Scalar (84, 127, 254), cv::Scalar (70, 84, 127), + cv::Scalar (56, 42, 0), cv::Scalar (42, 0, 254), + cv::Scalar (28, 42, 127), cv::Scalar (14, 84, 0), + cv::Scalar (0, 254, 254), cv::Scalar (14, 211, 127) +}; +/* *INDENT-ON* */ + +#define N_C (sizeof (colors)/sizeof (cv::Scalar)) + +#define CHOSEN_COLOR 14 +#define OVERLAY_HEIGHT 50 +#define OVERLAY_WIDTH 30 +#define LINES_GAP 20 + +GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); +#define GST_CAT_DEFAULT gst_inference_overlay_debug_category + +/* prototypes */ +static void gst_inference_overlay_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inference_overlay_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, + gint num_labels, LineStyleBoundingBox style); +static void draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, + cv::Scalar color, gint thickness, LineStyleBoundingBox style, gint gap); + +struct _GstInferenceOverlay +{ + GstInferenceBaseOverlay parent; +}; + +struct _GstInferenceOverlayClass +{ + GstInferenceBaseOverlay parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferenceOverlay, gst_inference_overlay, + GST_TYPE_INFERENCE_BASE_OVERLAY, + GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, + "inferenceoverlay", 0, "debug category for inference_overlay element")); + +static void +gst_inference_overlay_class_init (GstInferenceOverlayClass * klass) +{ + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = gst_inference_overlay_set_property; + gobject_class->get_property = gst_inference_overlay_get_property; + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "inferenceoverlay", "Filter", + "Overlays Inferece metadata on input buffer", + "Lenin Torres "); + g_object_class_install_property (gobject_class, PROP_LABELS, + g_param_spec_string ("labels", "labels", + "(Deprecated) Semicolon separated string containing inference labels.", + DEFAULT_LABELS, G_PARAM_READWRITE)); + + io_class->process_meta = + GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); + io_class->meta_type = GST_INFERENCE_META_API_TYPE; +} + +static void +gst_inference_overlay_init (GstInferenceOverlay * inference_overlay) +{ +} + +void +gst_inference_overlay_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + + GST_DEBUG_OBJECT (inference_overlay, "set_property"); + + switch (property_id) { + case PROP_LABELS: + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inference_overlay_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + + GST_DEBUG_OBJECT (inference_overlay, "get_property"); + + switch (property_id) { + case PROP_LABELS: + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + gint thickness, LineStyleBoundingBox style, gint gap) +{ + gfloat dx = pt1.x - pt2.x; + gfloat dy = pt1.y - pt2.y; + + gfloat dist = sqrt (dx * dx + dy * dy); + + std::vector < cv::Point > pts; + for (gint i = 0; i < dist; i += gap) { + gfloat r = (gfloat) i / dist; + gint x = gint ((pt1.x * (1.0 - r) + pt2.x * r) + .5); + gint y = gint ((pt1.y * (1.0 - r) + pt2.y * r) + .5); + cv::Point p = cv::Point (x, y); + pts.push_back (p); + } + + gint pts_size = pts.size (); + + if (DOTTED == style) { + for (gint i = 0; i < pts_size; i++) { + cv::circle (img, pts[i], thickness, color, -1); + } + } else { + cv::Point s = pts[0]; + cv::Point e = pts[0]; + + for (gint i = 0; i < pts_size; i++) { + s = e; + e = pts[i]; + + if (1 == i % 2) { + cv::line (img, s, e, color, thickness); + } + } + } +} + +static void +gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) +{ + cv::Size size; + cv::String label; + GList *iter = NULL; + GSList *list = NULL; + GSList *tree_iter = NULL; + cv::String prob; + BoundingBox box; + gint classes = 0; + gdouble alpha = 0.5; + cv::Mat alpha_overlay; + gint width, height, x, y = 0; + + g_return_if_fail (pred != NULL); + + list = gst_inference_prediction_get_children (pred); + + for (tree_iter = list; tree_iter != NULL; + tree_iter = g_slist_next (tree_iter)) { + GstInferencePrediction *predict = + (GstInferencePrediction *) tree_iter->data; + + gst_get_meta (predict, cv_mat, font_scale, thickness, + labels_list, num_labels, style); + } + + if (!pred->enabled) { + /* Ignore overlay if the prediction is disabled */ + return; + } + + box = pred->bbox; + + for (iter = pred->classifications; iter != NULL; iter = g_list_next (iter)) { + GstInferenceClassification *classification = (GstInferenceClassification *) + iter->data; + + classes++; + if (classification->num_classes > classification->class_id + && classification->class_label != NULL) { + label = + cv::format ("%s : %0.3f", + classification->class_label, + classification->class_prob); + } else { + label = cv::format ("Label #%d : %0.3f", classification->class_id, + classification->class_prob); + } + cv::putText (cv_mat, label, cv::Point (box.x + box.width, + box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, + font_scale, cv::Scalar::all (0), thickness); + } + + cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, + thickness, 0); + + if ((box.x + box.width) < 0) { + x = 0; + } else if ((int) (box.x + box.width) >= cv_mat.cols) { + x = cv_mat.cols - 1; + } else { + x = box.x + box.width; + } + + if ((int) (x + box.width + text.width) >= cv_mat.cols) { + width = cv_mat.cols - x - 1; + } else { + width = text.width; + } + + if ((int) (box.y + OVERLAY_HEIGHT * classes) >= cv_mat.rows) { + y = cv_mat.rows - 1; + } else if ((int) box.y < 0) { + y = 1; + } else { + y = box.y; + } + if ((int) (y + (OVERLAY_HEIGHT * classes)) >= cv_mat.rows) { + height = cv_mat.rows - y - 1; + } else { + height = OVERLAY_HEIGHT * classes; + } + + if (width && height) { + cv::Mat rectangle (height, width, cv_mat.type (), cv::Scalar (255, 255, + 255)); + cv::Rect roi (x, y, width, height); + cv::addWeighted (cv_mat (roi), alpha, rectangle, 1.0 - alpha, 0.0, + cv_mat (roi)); + } + + if (FALSE == G_NODE_IS_ROOT (pred->predictions)) { + if (0 == style) { + cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness); + } else { + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + draw_line (cv_mat, cv::Point (box.x + box.width, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + draw_line (cv_mat, cv::Point (box.x, box.y + box.height), + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + } + } + + if (NULL != list) { + g_slist_free (list); + } +} + +static GstFlowReturn +gst_inference_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) +{ + GstInferenceMeta *detect_meta; + + g_return_val_if_fail (inference_overlay != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (frame != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (meta != NULL, GST_FLOW_ERROR); + + detect_meta = (GstInferenceMeta *) meta; + + gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, + labels_list, num_labels, style); + + return GST_FLOW_OK; +} diff --git a/ext/opencv/gstinferenceoverlay.h b/ext/opencv/gstinferenceoverlay.h new file mode 100644 index 00000000..d89f2009 --- /dev/null +++ b/ext/opencv/gstinferenceoverlay.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_INFERENCE_OVERLAY_H_ +#define _GST_INFERENCE_OVERLAY_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_INFERENCE_OVERLAY (gst_inference_overlay_get_type()) +G_DECLARE_FINAL_TYPE (GstInferenceOverlay, gst_inference_overlay, GST, INFERENCE_OVERLAY, GstInferenceBaseOverlay) + +G_END_DECLS + +#endif diff --git a/ext/opencv/gstinferenceoverlay.c b/ext/opencv/gstplugin.cc similarity index 86% rename from ext/opencv/gstinferenceoverlay.c rename to ext/opencv/gstplugin.cc index 52b8129c..4bd1ba1f 100644 --- a/ext/opencv/gstinferenceoverlay.c +++ b/ext/opencv/gstplugin.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -26,6 +26,7 @@ #include "gstclassificationoverlay.h" #include "gstdetectionoverlay.h" #include "gstembeddingoverlay.h" +#include "gstinferenceoverlay.h" static gboolean plugin_init (GstPlugin * plugin) @@ -52,7 +53,12 @@ plugin_init (GstPlugin * plugin) if (!ret) { goto out; } - + ret = + gst_element_register (plugin, "inferenceoverlay", GST_RANK_NONE, + GST_TYPE_INFERENCE_OVERLAY); + if (!ret) { + goto out; + } out: return ret; @@ -60,6 +66,6 @@ plugin_init (GstPlugin * plugin) GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, - inferenceoverlay, + inferenceoverlayplugin, "Create overlays on incomming image frames with proper inference metadata", plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/ext/r2inference/gstfacenetv1.c b/ext/r2inference/gstfacenetv1.c index 258fa2e4..7e004e13 100644 --- a/ext/r2inference/gstfacenetv1.c +++ b/ext/r2inference/gstfacenetv1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -62,8 +62,16 @@ static void gst_facenetv1_finalize (GObject * object); static gboolean gst_facenetv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_facenetv1_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_facenetv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_facenetv1_start (GstVideoInference * vi); static gboolean gst_facenetv1_stop (GstVideoInference * vi); @@ -205,22 +213,74 @@ gst_facenetv1_preprocess (GstVideoInference * vi, } static gboolean -gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_facenetv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); - gst_inference_print_embedding (vi, gst_facenetv1_debug_category, class_meta, - prediction, gst_debug_level); + gst_inference_print_highest_probability (vi, gst_facenetv1_debug_category, + class_meta, prediction, gst_debug_level); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_facenetv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_facenetv1_debug_category, imeta); + *valid_prediction = TRUE; return TRUE; } +static gboolean +gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_facenetv1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_facenetv1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_facenetv1_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstfacenetv1.h b/ext/r2inference/gstfacenetv1.h index 7ed9cb05..c91989b8 100644 --- a/ext/r2inference/gstfacenetv1.h +++ b/ext/r2inference/gstfacenetv1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv1.c b/ext/r2inference/gstinceptionv1.c index aaed7dd8..d6924889 100644 --- a/ext/r2inference/gstinceptionv1.c +++ b/ext/r2inference/gstinceptionv1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -57,8 +57,16 @@ GST_DEBUG_CATEGORY_STATIC (gst_inceptionv1_debug_category); static gboolean gst_inceptionv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_inceptionv1_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv1_start (GstVideoInference * vi); static gboolean gst_inceptionv1_stop (GstVideoInference * vi); @@ -144,12 +152,13 @@ gst_inceptionv1_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_inceptionv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -161,6 +170,56 @@ gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv1_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv1_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv1.h b/ext/r2inference/gstinceptionv1.h index 412143ae..6018f30a 100644 --- a/ext/r2inference/gstinceptionv1.h +++ b/ext/r2inference/gstinceptionv1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index c9d99e2c..e54323cb 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -65,8 +65,16 @@ static void gst_inceptionv2_finalize (GObject * object); static gboolean gst_inceptionv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv2_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_inceptionv2_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv2_start (GstVideoInference * vi); static gboolean gst_inceptionv2_stop (GstVideoInference * vi); @@ -211,12 +219,13 @@ gst_inceptionv2_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_inceptionv2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -228,6 +237,56 @@ gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv2_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv2_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv2.h b/ext/r2inference/gstinceptionv2.h index ba4aa4dc..8b444e79 100644 --- a/ext/r2inference/gstinceptionv2.h +++ b/ext/r2inference/gstinceptionv2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv3.c b/ext/r2inference/gstinceptionv3.c index 582906f3..7f89b0a3 100644 --- a/ext/r2inference/gstinceptionv3.c +++ b/ext/r2inference/gstinceptionv3.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -56,8 +56,16 @@ GST_DEBUG_CATEGORY_STATIC (gst_inceptionv3_debug_category); static gboolean gst_inceptionv3_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_inceptionv3_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv3_start (GstVideoInference * vi); static gboolean gst_inceptionv3_stop (GstVideoInference * vi); @@ -143,12 +151,13 @@ gst_inceptionv3_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_inceptionv3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -160,6 +169,56 @@ gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv3_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv3_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv3_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv3_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv3.h b/ext/r2inference/gstinceptionv3.h index 1978acc0..e504624b 100644 --- a/ext/r2inference/gstinceptionv3.h +++ b/ext/r2inference/gstinceptionv3.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv4.c b/ext/r2inference/gstinceptionv4.c index eae66ca0..24081dc4 100644 --- a/ext/r2inference/gstinceptionv4.c +++ b/ext/r2inference/gstinceptionv4.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -64,8 +64,9 @@ static void gst_inceptionv4_finalize (GObject * object); static gboolean gst_inceptionv4_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv4_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction); + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv4_start (GstVideoInference * vi); static gboolean gst_inceptionv4_stop (GstVideoInference * vi); @@ -210,12 +211,13 @@ gst_inceptionv4_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_inceptionv4_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -227,6 +229,56 @@ gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv4_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv4_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv4_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv4_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv4_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv4.h b/ext/r2inference/gstinceptionv4.h index f4726f9b..f5a4a450 100644 --- a/ext/r2inference/gstinceptionv4.h +++ b/ext/r2inference/gstinceptionv4.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index d53be1d8..1422cb96 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstmobilenetv2.c b/ext/r2inference/gstmobilenetv2.c index 05fa37da..7b9b44e0 100644 --- a/ext/r2inference/gstmobilenetv2.c +++ b/ext/r2inference/gstmobilenetv2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -55,8 +55,16 @@ GST_DEBUG_CATEGORY_STATIC (gst_mobilenetv2_debug_category); static gboolean gst_mobilenetv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_mobilenetv2_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_mobilenetv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_mobilenetv2_start (GstVideoInference * vi); static gboolean gst_mobilenetv2_stop (GstVideoInference * vi); @@ -142,12 +150,13 @@ gst_mobilenetv2_preprocess (GstVideoInference * vi, } static gboolean -gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_mobilenetv2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -159,6 +168,56 @@ gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_mobilenetv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_mobilenetv2_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_mobilenetv2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_mobilenetv2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_mobilenetv2_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstmobilenetv2.h b/ext/r2inference/gstmobilenetv2.h index ce78f9be..6e2d24cd 100644 --- a/ext/r2inference/gstmobilenetv2.h +++ b/ext/r2inference/gstmobilenetv2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstresnet50v1.c b/ext/r2inference/gstresnet50v1.c index 9c86edf7..b52a905a 100644 --- a/ext/r2inference/gstresnet50v1.c +++ b/ext/r2inference/gstresnet50v1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -59,8 +59,16 @@ GST_DEBUG_CATEGORY_STATIC (gst_resnet50v1_debug_category); static gboolean gst_resnet50v1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); +static gboolean gst_resnet50v1_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_resnet50v1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_resnet50v1_start (GstVideoInference * vi); static gboolean gst_resnet50v1_stop (GstVideoInference * vi); @@ -148,12 +156,13 @@ gst_resnet50v1_preprocess (GstVideoInference * vi, } static gboolean -gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_resnet50v1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -165,6 +174,56 @@ gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_resnet50v1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_resnet50v1_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_resnet50v1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_resnet50v1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_resnet50v1_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstresnet50v1.h b/ext/r2inference/gstresnet50v1.h index 83390c58..9cb41a01 100644 --- a/ext/r2inference/gstresnet50v1.h +++ b/ext/r2inference/gstresnet50v1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index c33f3799..33aace84 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -81,8 +81,17 @@ static gboolean gst_tinyyolov2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction); + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean +gst_tinyyolov2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_tinyyolov2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov2_start (GstVideoInference * vi); static gboolean gst_tinyyolov2_stop (GstVideoInference * vi); @@ -280,18 +289,46 @@ gst_tinyyolov2_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + g_return_val_if_fail (vi, FALSE); + g_return_val_if_fail (prediction, FALSE); + g_return_val_if_fail (meta_model, FALSE); + g_return_val_if_fail (info_model, FALSE); + g_return_val_if_fail (valid_prediction, FALSE); + + ret &= + gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_tinyyolov2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return TRUE; +} + +static gboolean +gst_tinyyolov2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov2 *tinyyolov2; + gdouble *probabilities = NULL; + GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; + + g_return_val_if_fail (detect_meta, FALSE); + GST_LOG_OBJECT (vi, "Postprocess"); detect_meta->num_boxes = 0; tinyyolov2 = GST_TINYYOLOV2 (vi); - gst_create_boxes (vi, prediction, detect_meta, info_model, valid_prediction, + gst_create_boxes (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh); + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); gst_inference_print_boxes (vi, gst_tinyyolov2_debug_category, detect_meta); @@ -300,6 +337,58 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_tinyyolov2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstTinyyolov2 *tinyyolov2 = NULL; + GstInferenceMeta *imeta = NULL; + BBox *boxes = NULL; + gint num_boxes, i; + gdouble *probabilities = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + imeta = (GstInferenceMeta *) meta_model; + tinyyolov2 = GST_TINYYOLOV2 (vi); + + GST_LOG_OBJECT (tinyyolov2, "Postprocess Meta"); + + /* Create boxes from prediction data */ + gst_create_boxes (vi, prediction, valid_prediction, + &boxes, &num_boxes, tinyyolov2->obj_thresh, + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); + + GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); + + if (NULL == imeta->prediction) { + BoundingBox bbox = { 0, 0, info_model->width, info_model->height }; + imeta->prediction = gst_inference_prediction_new_full (&bbox); + } + + for (i = 0; i < num_boxes; i++) { + GstInferencePrediction *pred = + gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, + probabilities); + + gst_inference_prediction_append (imeta->prediction, pred); + } + + /* Free boxes after creation */ + g_free (boxes); + + /* Log predictions */ + gst_inference_print_predictions (vi, gst_tinyyolov2_debug_category, imeta); + + *valid_prediction = (num_boxes > 0) ? TRUE : FALSE; + + return TRUE; +} + static gboolean gst_tinyyolov2_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gsttinyyolov2.h b/ext/r2inference/gsttinyyolov2.h index f67904b2..434f1740 100644 --- a/ext/r2inference/gsttinyyolov2.h +++ b/ext/r2inference/gsttinyyolov2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index f7a53ebb..e7ff510d 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -73,8 +73,17 @@ static gboolean gst_tinyyolov3_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction); + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean +gst_tinyyolov3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_tinyyolov3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov3_start (GstVideoInference * vi); static gboolean gst_tinyyolov3_stop (GstVideoInference * vi); @@ -246,18 +255,46 @@ gst_tinyyolov3_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + g_return_val_if_fail (vi, FALSE); + g_return_val_if_fail (prediction, FALSE); + g_return_val_if_fail (meta_model, FALSE); + g_return_val_if_fail (info_model, FALSE); + g_return_val_if_fail (valid_prediction, FALSE); + + ret &= + gst_tinyyolov3_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_tinyyolov3_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return TRUE; +} + +static gboolean +gst_tinyyolov3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov3 *tinyyolov3; + gdouble *probabilities = NULL; + GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; + + g_return_val_if_fail (detect_meta, FALSE); + GST_LOG_OBJECT (vi, "Postprocess"); detect_meta->num_boxes = 0; tinyyolov3 = GST_TINYYOLOV3 (vi); - gst_create_boxes_float (vi, prediction, detect_meta, info_model, - valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, - tinyyolov3->obj_thresh, tinyyolov3->prob_thresh, tinyyolov3->iou_thresh); + gst_create_boxes (vi, prediction, valid_prediction, + &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); gst_inference_print_boxes (vi, gst_tinyyolov3_debug_category, detect_meta); @@ -266,6 +303,58 @@ gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_tinyyolov3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstTinyyolov3 *tinyyolov3 = NULL; + GstInferenceMeta *imeta = NULL; + BBox *boxes = NULL; + gint num_boxes, i; + gdouble *probabilities = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + imeta = (GstInferenceMeta *) meta_model; + tinyyolov3 = GST_TINYYOLOV3 (vi); + + GST_LOG_OBJECT (tinyyolov3, "Postprocess Meta"); + + /* Create boxes from prediction data */ + gst_create_boxes (vi, prediction, valid_prediction, + &boxes, &num_boxes, tinyyolov3->obj_thresh, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); + + GST_LOG_OBJECT (tinyyolov3, "Number of predictions: %d", num_boxes); + + if (NULL == imeta->prediction) { + imeta->prediction = gst_inference_prediction_new (); + imeta->prediction->bbox.width = info_model->width; + imeta->prediction->bbox.height = info_model->height; + } + + for (i = 0; i < num_boxes; i++) { + GstInferencePrediction *pred = + gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, + probabilities); + gst_inference_prediction_append (imeta->prediction, pred); + } + + /* Free boxes after creation */ + g_free (boxes); + + /* Log predictions */ + gst_inference_print_predictions (vi, gst_tinyyolov3_debug_category, imeta); + + *valid_prediction = (num_boxes > 0) ? TRUE : FALSE; + + return TRUE; +} + static gboolean gst_tinyyolov3_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gsttinyyolov3.h b/ext/r2inference/gsttinyyolov3.h index 08f3328e..ab7b087a 100644 --- a/ext/r2inference/gsttinyyolov3.h +++ b/ext/r2inference/gsttinyyolov3.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-inference.pc.in b/gst-inference.pc.in new file mode 100644 index 00000000..a753a936 --- /dev/null +++ b/gst-inference.pc.in @@ -0,0 +1,12 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@GSTINCLUDEDIR@ + +Name: GstInference +Description: GStreamer framework for deep learning inference. +URL: https://github.com/RidgeRun/gst-inference/ +Version: @VERSION@ +Requires: gstreamer-1.0 gstreamer-plugins-base-1.0 gstreamer-video-1.0 r2inference-0.0 +Libs: -L${libdir} -lgstinference-@GST_API_VERSION@ +Cflags: -I${includedir} diff --git a/gst-libs/gst/opencv/Makefile.am b/gst-libs/gst/opencv/Makefile.am index ffd1ba4c..8d83bb38 100644 --- a/gst-libs/gst/opencv/Makefile.am +++ b/gst-libs/gst/opencv/Makefile.am @@ -1,23 +1,29 @@ -lib_LTLIBRARIES = libgstinferenceoverlay-@GST_API_VERSION@.la +lib_LTLIBRARIES = libgstinferencebaseoverlay-@GST_API_VERSION@.la -libgstinferenceoverlay_@GST_API_VERSION@_la_SOURCES= \ - gstinferenceoverlay.c +libgstinferencebaseoverlay_@GST_API_VERSION@_la_SOURCES= \ + gstinferencebaseoverlay.cc -libgstinferenceoverlay_@GST_API_VERSION@_la_CFLAGS= \ +libgstinferencebaseoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ $(R2INFERENCE_CFLAGS) -libgstinferenceoverlay_@GST_API_VERSION@_la_LIBADD= \ +libgstinferencebaseoverlay_@GST_API_VERSION@_la_CXXFLAGS= \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) + +libgstinferencebaseoverlay_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) -gstinferenceoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ +gstinferencebaseoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ -gstinferenceoverlayinclude_HEADERS= \ - gstinferenceoverlay.h +gstinferencebaseoverlayinclude_HEADERS= \ + gstinferencebaseoverlay.h diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc new file mode 100644 index 00000000..3bde2118 --- /dev/null +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -0,0 +1,358 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstinferencebaseoverlay.h" + +/* pad templates */ + +#define VIDEO_SRC_CAPS \ + GST_VIDEO_CAPS_MAKE("{RGB, RGBx, RGBA, BGR, BGRx, BGRA, xRGB, ARGB, xBGR, ABGR}") + +#define VIDEO_SINK_CAPS \ + GST_VIDEO_CAPS_MAKE("{RGB, RGBx, RGBA, BGR, BGRx, BGRA, xRGB, ARGB, xBGR, ABGR}") + +GST_DEBUG_CATEGORY_STATIC (gst_inference_base_overlay_debug_category); +#define GST_CAT_DEFAULT gst_inference_base_overlay_debug_category + +#define MIN_FONT_SCALE 0 +#define DEFAULT_FONT_SCALE 2 +#define MAX_FONT_SCALE 100 +#define MIN_THICKNESS 1 +#define DEFAULT_THICKNESS 2 +#define MAX_THICKNESS 100 +#define DEFAULT_LABELS NULL +#define DEFAULT_NUM_LABELS 0 + +#define MIN_STYLE CLASSIC +#define DEFAULT_STYLE CLASSIC +#define MAX_STYLE DASHED + +enum +{ + PROP_0, + PROP_FONT_SCALE, + PROP_THICKNESS, + PROP_LABELS, + PROP_STYLE +}; + +GType +line_style_bounding_box_get_type (void) +{ + static GType type = G_TYPE_INVALID; + if (G_UNLIKELY (type == G_TYPE_INVALID)) { + static const GEnumValue values[] = { + {CLASSIC, "CLASSIC", "classic",}, + {DOTTED, "DOTTED", "dotted",}, + {DASHED, "DASHED", "dashed",}, + {4, NULL, NULL,}, + }; + type = g_enum_register_static ("LineStyleBoundingBox", values); + } + return type; +} + +typedef struct _GstInferenceBaseOverlayPrivate GstInferenceBaseOverlayPrivate; +struct _GstInferenceBaseOverlayPrivate +{ + gdouble font_scale; + gint thickness; + gchar *labels; + gchar **labels_list; + gint num_labels; + LineStyleBoundingBox style; +}; +/* prototypes */ +static void gst_inference_base_overlay_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inference_base_overlay_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_inference_base_overlay_dispose (GObject * object); +static void gst_inference_base_overlay_finalize (GObject * object); + +static gboolean gst_inference_base_overlay_start (GstBaseTransform * trans); +static gboolean gst_inference_base_overlay_stop (GstBaseTransform * trans); +static GstFlowReturn +gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, + GstVideoFrame * frame); + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferenceBaseOverlay, gst_inference_base_overlay, + GST_TYPE_VIDEO_FILTER, + GST_DEBUG_CATEGORY_INIT (gst_inference_base_overlay_debug_category, + "inferencebaseoverlay", 0, "debug category for inferenceoverlay class"); + G_ADD_PRIVATE (GstInferenceBaseOverlay)); + +#define GST_INFERENCE_BASE_OVERLAY_PRIVATE(self) \ + (GstInferenceBaseOverlayPrivate *)(gst_inference_base_overlay_get_instance_private (self)) + +static void +gst_inference_base_overlay_class_init (GstInferenceBaseOverlayClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass); + + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string (VIDEO_SRC_CAPS))); + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string (VIDEO_SINK_CAPS))); + + gobject_class->set_property = gst_inference_base_overlay_set_property; + gobject_class->get_property = gst_inference_base_overlay_get_property; + gobject_class->dispose = gst_inference_base_overlay_dispose; + gobject_class->finalize = gst_inference_base_overlay_finalize; + + g_object_class_install_property (gobject_class, PROP_FONT_SCALE, + g_param_spec_double ("font-scale", "font", "Font scale", MIN_FONT_SCALE, + MAX_FONT_SCALE, DEFAULT_FONT_SCALE, G_PARAM_READWRITE)); + + g_object_class_install_property (gobject_class, PROP_THICKNESS, + g_param_spec_int ("thickness", "thickness", + "Box line thickness in pixels", MIN_THICKNESS, MAX_THICKNESS, + DEFAULT_THICKNESS, G_PARAM_READWRITE)); + + g_object_class_install_property (gobject_class, PROP_LABELS, + g_param_spec_string ("labels", "labels", + "Semicolon separated string containing inference labels", + DEFAULT_LABELS, G_PARAM_READWRITE)); + g_object_class_install_property (gobject_class, PROP_STYLE, + g_param_spec_enum ("style", "style", + "Line style to draw the bounding box", LINE_STYLE_BOUNDING_BOX, + DEFAULT_STYLE, G_PARAM_READWRITE)); + + base_transform_class->start = + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_start); + base_transform_class->stop = + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_stop); + video_filter_class->transform_frame_ip = + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_transform_frame_ip); + +} + +static void +gst_inference_base_overlay_init (GstInferenceBaseOverlay * inference_overlay) +{ + GstInferenceBaseOverlayPrivate *priv = + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); + priv->font_scale = DEFAULT_FONT_SCALE; + priv->thickness = DEFAULT_THICKNESS; + priv->labels = DEFAULT_LABELS; + priv->labels_list = DEFAULT_LABELS; + priv->num_labels = DEFAULT_NUM_LABELS; + priv->style = DEFAULT_STYLE; +} + +void +gst_inference_base_overlay_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); + + GST_DEBUG_OBJECT (inference_overlay, "set_property"); + + switch (property_id) { + case PROP_FONT_SCALE: + priv->font_scale = g_value_get_double (value); + GST_DEBUG_OBJECT (inference_overlay, "Changed font scale to %lf", + priv->font_scale); + break; + case PROP_THICKNESS: + priv->thickness = g_value_get_int (value); + GST_DEBUG_OBJECT (inference_overlay, "Changed box thickness to %d", + priv->thickness); + break; + case PROP_LABELS: + if (priv->labels != NULL) { + g_free (priv->labels); + } + if (priv->labels_list != NULL) { + g_strfreev (priv->labels_list); + } + priv->labels = g_value_dup_string (value); + priv->labels_list = g_strsplit (g_value_get_string (value), ";", 0); + priv->num_labels = g_strv_length (priv->labels_list); + GST_DEBUG_OBJECT (inference_overlay, "Changed inference labels %s", + priv->labels); + break; + case PROP_STYLE: + priv->style = (LineStyleBoundingBox) g_value_get_enum (value); + GST_DEBUG_OBJECT (inference_overlay, "Changed box style to %d", + priv->style); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inference_base_overlay_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); + + GST_DEBUG_OBJECT (inference_overlay, "get_property"); + + switch (property_id) { + case PROP_FONT_SCALE: + g_value_set_double (value, priv->font_scale); + break; + case PROP_THICKNESS: + g_value_set_int (value, priv->thickness); + break; + case PROP_LABELS: + g_value_set_string (value, priv->labels); + break; + case PROP_STYLE: + g_value_set_enum (value, priv->style); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inference_base_overlay_dispose (GObject * object) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); + + GST_DEBUG_OBJECT (inference_overlay, "dispose"); + + /* clean up as possible. may be called multiple times */ + if (priv->labels_list != NULL) { + g_strfreev (priv->labels_list); + } + if (priv->labels != NULL) { + g_free (priv->labels); + } + + G_OBJECT_CLASS (gst_inference_base_overlay_parent_class)->dispose (object); +} + +void +gst_inference_base_overlay_finalize (GObject * object) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); + + GST_DEBUG_OBJECT (inference_overlay, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS (gst_inference_base_overlay_parent_class)->finalize (object); +} + +static gboolean +gst_inference_base_overlay_start (GstBaseTransform * trans) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); + + GST_DEBUG_OBJECT (inference_overlay, "start"); + + return TRUE; +} + +static gboolean +gst_inference_base_overlay_stop (GstBaseTransform * trans) +{ + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); + + GST_DEBUG_OBJECT (inference_overlay, "stop"); + + return TRUE; +} + +/* transform */ +static GstFlowReturn +gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, + GstVideoFrame * frame) +{ + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_GET_CLASS (trans); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); + GstInferenceBaseOverlayPrivate *priv = + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); + GstMeta *meta; + GstFlowReturn ret = GST_FLOW_ERROR; + gint width, height, channels; + cv::Mat cv_mat; + gint sizes[2] = { 0, 0 }; + gsize steps[2] = { 0, 0 }; + const gint num_dims = 2; + gchar *data = NULL; + + g_return_val_if_fail (io_class->process_meta != NULL, GST_FLOW_ERROR); + + meta = gst_buffer_get_meta (frame->buffer, io_class->meta_type); + if (NULL == meta) { + GST_LOG_OBJECT (trans, "No inference meta found"); + ret = GST_FLOW_OK; + goto out; + } + + GST_LOG_OBJECT (trans, "Valid inference meta found"); + + /* Use pixel stride instead of num components because RGBx reports 3 channels */ + channels = GST_VIDEO_FRAME_COMP_PSTRIDE (frame, 0); + width = GST_VIDEO_FRAME_WIDTH (frame); + height = GST_VIDEO_FRAME_HEIGHT (frame); + data = (gchar *) GST_VIDEO_FRAME_PLANE_DATA (frame, 0); + + sizes[0] = height; + sizes[1] = width; + + /* This is not a mistake, it's oddly inverted */ + steps[1] = channels; + steps[0] = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); + + GST_LOG_OBJECT (trans, "width: %d, height: %d, stride: %" G_GSIZE_FORMAT + ", channels: %d", width, height, steps[0], channels); + + cv_mat = + cv::Mat (num_dims, (gint *) sizes, CV_MAKETYPE (CV_8U, channels), data, + (gsize *) steps); + + ret = + io_class->process_meta (inference_overlay, cv_mat, frame, meta, + priv->font_scale, priv->thickness, priv->labels_list, priv->num_labels, + priv->style); + +out: + return ret; +} diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h new file mode 100644 index 00000000..32a6effe --- /dev/null +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -0,0 +1,69 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_BASE_OVERLAY_H__ +#define __GST_INFERENCE_BASE_OVERLAY_H__ + +#include +#include +#ifdef OCV_VERSION_LT_3_2 +#include "opencv2/highgui/highgui.hpp" +#else +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#endif + +G_BEGIN_DECLS + +#define LINE_STYLE_BOUNDING_BOX (line_style_bounding_box_get_type ()) +/** + * LineStyleBoundingBox: + * @CLASSIC : Draw lines without any style + * @DOTTED : Draw lines with dots + * @DASHED : Draw lines with dashed lines + * + **/ +typedef enum +{ + CLASSIC, + DOTTED, + DASHED, +} LineStyleBoundingBox; + +GType line_style_bounding_box_get_type (void) G_GNUC_CONST; +#define GST_TYPE_INFERENCE_BASE_OVERLAY gst_inference_base_overlay_get_type () +G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST, + INFERENCE_BASE_OVERLAY, GstVideoFilter); + +struct _GstInferenceBaseOverlayClass +{ + GstVideoFilterClass parent_class; + + GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, + cv::Mat &mat, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, + gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style); + + GType meta_type; +}; + +G_END_DECLS + +#endif //__GST_INFERENCE_OVERLAY_H__ diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.c b/gst-libs/gst/opencv/gstinferenceoverlay.c deleted file mode 100644 index 6c6bfae3..00000000 --- a/gst-libs/gst/opencv/gstinferenceoverlay.c +++ /dev/null @@ -1,284 +0,0 @@ -/* - * GStreamer - * Copyright (C) 2019 RidgeRun - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307, USA. - * - */ - -#include "gstinferenceoverlay.h" - -/* pad templates */ - -#define VIDEO_SRC_CAPS \ - GST_VIDEO_CAPS_MAKE("{RGB, RGBx, RGBA, BGR, BGRx, BGRA, xRGB, ARGB, xBGR, ABGR}") - -#define VIDEO_SINK_CAPS \ - GST_VIDEO_CAPS_MAKE("{RGB, RGBx, RGBA, BGR, BGRx, BGRA, xRGB, ARGB, xBGR, ABGR}") - -GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); -#define GST_CAT_DEFAULT gst_inference_overlay_debug_category - -#define MIN_FONT_SCALE 0 -#define DEFAULT_FONT_SCALE 2 -#define MAX_FONT_SCALE 100 -#define MIN_THICKNESS 1 -#define DEFAULT_THICKNESS 2 -#define MAX_THICKNESS 100 -#define DEFAULT_LABELS NULL -#define DEFAULT_NUM_LABELS 0 - -enum -{ - PROP_0, - PROP_FONT_SCALE, - PROP_THICKNESS, - PROP_LABELS -}; - -typedef struct _GstInferenceOverlayPrivate GstInferenceOverlayPrivate; -struct _GstInferenceOverlayPrivate -{ - gdouble font_scale; - gint thickness; - gchar *labels; - gchar **labels_list; - gint num_labels; -}; -/* prototypes */ -static void gst_inference_overlay_set_property (GObject * object, - guint property_id, const GValue * value, GParamSpec * pspec); -static void gst_inference_overlay_get_property (GObject * object, - guint property_id, GValue * value, GParamSpec * pspec); -static void gst_inference_overlay_dispose (GObject * object); -static void gst_inference_overlay_finalize (GObject * object); - -static gboolean gst_inference_overlay_start (GstBaseTransform * trans); -static gboolean gst_inference_overlay_stop (GstBaseTransform * trans); -static GstFlowReturn -gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, - GstVideoFrame * frame); - -/* class initialization */ - -G_DEFINE_TYPE_WITH_CODE (GstInferenceOverlay, gst_inference_overlay, - GST_TYPE_VIDEO_FILTER, - GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, - "inferenceoverlay", 0, "debug category for inferenceoverlay class"); - G_ADD_PRIVATE (GstInferenceOverlay)); - -#define GST_INFERENCE_OVERLAY_PRIVATE(self) \ - (GstInferenceOverlayPrivate *)(gst_inference_overlay_get_instance_private (self)) - -static void -gst_inference_overlay_class_init (GstInferenceOverlayClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstBaseTransformClass *base_transform_class = - GST_BASE_TRANSFORM_CLASS (klass); - GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass); - - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SRC_CAPS))); - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SINK_CAPS))); - - gobject_class->set_property = gst_inference_overlay_set_property; - gobject_class->get_property = gst_inference_overlay_get_property; - gobject_class->dispose = gst_inference_overlay_dispose; - gobject_class->finalize = gst_inference_overlay_finalize; - - g_object_class_install_property (gobject_class, PROP_FONT_SCALE, - g_param_spec_double ("font-scale", "font", "Font scale", MIN_FONT_SCALE, - MAX_FONT_SCALE, DEFAULT_FONT_SCALE, G_PARAM_READWRITE)); - - g_object_class_install_property (gobject_class, PROP_THICKNESS, - g_param_spec_int ("thickness", "thickness", - "Box line thickness in pixels", MIN_THICKNESS, MAX_THICKNESS, - DEFAULT_THICKNESS, G_PARAM_READWRITE)); - - g_object_class_install_property (gobject_class, PROP_LABELS, - g_param_spec_string ("labels", "labels", - "Semicolon separated string containing inference labels", - DEFAULT_LABELS, G_PARAM_READWRITE)); - - base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inference_overlay_start); - base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_inference_overlay_stop); - video_filter_class->transform_frame_ip = - GST_DEBUG_FUNCPTR (gst_inference_overlay_transform_frame_ip); - -} - -static void -gst_inference_overlay_init (GstInferenceOverlay * inference_overlay) -{ - GstInferenceOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); - priv->font_scale = DEFAULT_FONT_SCALE; - priv->thickness = DEFAULT_THICKNESS; - priv->labels = DEFAULT_LABELS; - priv->labels_list = DEFAULT_LABELS; - priv->num_labels = DEFAULT_NUM_LABELS; -} - -void -gst_inference_overlay_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); - - GST_DEBUG_OBJECT (inference_overlay, "set_property"); - - switch (property_id) { - case PROP_FONT_SCALE: - priv->font_scale = g_value_get_double (value); - GST_DEBUG_OBJECT (inference_overlay, "Changed font scale to %lf", - priv->font_scale); - break; - case PROP_THICKNESS: - priv->thickness = g_value_get_int (value); - GST_DEBUG_OBJECT (inference_overlay, "Changed box thickness to %d", - priv->thickness); - break; - case PROP_LABELS: - if (priv->labels != NULL) { - g_free (priv->labels); - } - if (priv->labels_list != NULL) { - g_strfreev (priv->labels_list); - } - priv->labels = g_value_dup_string (value); - priv->labels_list = g_strsplit (g_value_get_string (value), ";", 0); - priv->num_labels = g_strv_length (priv->labels_list); - GST_DEBUG_OBJECT (inference_overlay, "Changed inference labels %s", - priv->labels); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - break; - } -} - -void -gst_inference_overlay_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); - - GST_DEBUG_OBJECT (inference_overlay, "get_property"); - - switch (property_id) { - case PROP_FONT_SCALE: - g_value_set_double (value, priv->font_scale); - break; - case PROP_THICKNESS: - g_value_set_int (value, priv->thickness); - break; - case PROP_LABELS: - g_value_set_string (value, priv->labels); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - break; - } -} - -void -gst_inference_overlay_dispose (GObject * object) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); - - GST_DEBUG_OBJECT (inference_overlay, "dispose"); - - /* clean up as possible. may be called multiple times */ - if (priv->labels_list != NULL) { - g_strfreev (priv->labels_list); - } - if (priv->labels != NULL) { - g_free (priv->labels); - } - - G_OBJECT_CLASS (gst_inference_overlay_parent_class)->dispose (object); -} - -void -gst_inference_overlay_finalize (GObject * object) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - - GST_DEBUG_OBJECT (inference_overlay, "finalize"); - - /* clean up object here */ - - G_OBJECT_CLASS (gst_inference_overlay_parent_class)->finalize (object); -} - -static gboolean -gst_inference_overlay_start (GstBaseTransform * trans) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); - - GST_DEBUG_OBJECT (inference_overlay, "start"); - - return TRUE; -} - -static gboolean -gst_inference_overlay_stop (GstBaseTransform * trans) -{ - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); - - GST_DEBUG_OBJECT (inference_overlay, "stop"); - - return TRUE; -} - -/* transform */ -static GstFlowReturn -gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, - GstVideoFrame * frame) -{ - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_GET_CLASS (trans); - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); - GstInferenceOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); - GstMeta *meta; - GstFlowReturn ret = GST_FLOW_ERROR; - - meta = gst_buffer_get_meta (frame->buffer, io_class->meta_type); - if (NULL == meta) { - GST_LOG_OBJECT (trans, "No inference meta found"); - ret = GST_FLOW_OK; - } else { - GST_LOG_OBJECT (trans, "Valid inference meta found"); - if (io_class->process_meta != NULL) { - ret = - io_class->process_meta (inference_overlay, frame, meta, - priv->font_scale, priv->thickness, priv->labels_list, - priv->num_labels); - } - } - - return ret; -} diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 7616d45f..f99d2062 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -9,9 +9,12 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstbackend.cc \ gstncsdk.cc \ gsttensorflow.cc \ - gstinferencepreprocess.c \ + gsttflite.cc \ + gstinferencepreprocess.c \ gstinferencepostprocess.c \ - gstinferencedebug.c + gstinferencedebug.c \ + gstinferenceprediction.c \ + gstinferenceclassification.c libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ @@ -28,8 +31,8 @@ libgstinference_@GST_API_VERSION@_la_CFLAGS= \ libgstinference_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ - $(GST_BASE_LIBS) \ - -lgstvideo-@GST_API_VERSION@ \ + $(GST_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) @@ -42,7 +45,11 @@ gstinferenceinclude_HEADERS= \ gstbackend.h \ gstbackendsubclass.h \ gstncsdk.h \ - gsttensorflow.h \ - gstinferencepreprocess.h \ + gsttensorflow.h \ + gsttflite.h \ + gstinferencepreprocess.h \ gstinferencepostprocess.h \ - gstinferencedebug.h + gstinferencedebug.h \ + gstinferencemeta.h \ + gstinferenceprediction.h \ + gstinferenceclassification.h diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 633957b2..018acb3b 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -395,6 +395,27 @@ gst_backend_stop (GstBackend *self, GError **err) { return FALSE; } +static r2i::ImageFormat::Id +gst_backend_cast_format (GstVideoFormat format) { + r2i::ImageFormat::Id image_format; + + switch (format) { + case GST_VIDEO_FORMAT_RGB: + image_format = r2i::ImageFormat::Id::RGB; + break; + case GST_VIDEO_FORMAT_BGR: + image_format = r2i::ImageFormat::Id::BGR; + break; + case GST_VIDEO_FORMAT_GRAY8: + image_format = r2i::ImageFormat::Id::GRAY8; + break; + default: + image_format = r2i::ImageFormat::Id::RGB; + break; + } + return image_format; +} + gboolean gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, gpointer *prediction_data, gsize *prediction_size, GError **err) { @@ -419,7 +440,8 @@ gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, error = frame->Configure (input_frame->data[0], input_frame->info.width, - input_frame->info.height, r2i::ImageFormat::Id::RGB); + input_frame->info.height, + gst_backend_cast_format(input_frame->info.finfo->format)); if (error.IsError ()) { goto error; } diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 933bb21f..eb354813 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstbackendsubclass.h b/gst-libs/gst/r2inference/gstbackendsubclass.h index 37aef953..8cef2dac 100644 --- a/gst-libs/gst/r2inference/gstbackendsubclass.h +++ b/gst-libs/gst/r2inference/gstbackendsubclass.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef __GST_BACKENDSUBCLASS_H__ #define __GST_BACKENDSUBCLASS_H__ diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c index 932c69fd..280ba639 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.c +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstchildinspector.h b/gst-libs/gst/r2inference/gstchildinspector.h index 28392601..6b0166d3 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.h +++ b/gst-libs/gst/r2inference/gstchildinspector.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index 324d23c9..a1f1de8c 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -23,6 +23,7 @@ #include "gstchildinspector.h" #include "gstncsdk.h" #include "gsttensorflow.h" +#include "gsttflite.h" #include "gstbackend.h" #include #include @@ -34,6 +35,7 @@ static std::unordered_map backend_types ({ {r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, {r2i::FrameworkCode::TENSORFLOW, GST_TYPE_TENSORFLOW}, + {r2i::FrameworkCode::TFLITE, GST_TYPE_TFLITE}, {r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} }); @@ -51,6 +53,8 @@ gst_inference_backends_get_type (void) {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", "tensorflow"}, + {r2i::FrameworkCode::TFLITE, "Tensorflow Lite Machine Learning Framework", + "tflite"}, {0, NULL, NULL} }; if (!backend_type) { diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h index 8d8bc8c9..da3ab634 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.h +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c new file mode 100644 index 00000000..0b26e853 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -0,0 +1,237 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstinferenceclassification.h" + +#include + +#define DEFAULT_CLASS_ID -1 +#define DEFAULT_CLASS_PROB 0.0f +#define DEFAULT_CLASS_LABEL NULL +#define DEFAULT_NUM_CLASSES 0 +#define DEFAULT_PROBABILITIES NULL +#define DEFAULT_LABELS NULL + +static GType gst_inference_classification_get_type (void); +GST_DEFINE_MINI_OBJECT_TYPE (GstInferenceClassification, + gst_inference_classification); + +static void classification_free (GstInferenceClassification * self); +static void classification_reset (GstInferenceClassification * self); + +static gdouble *probabilities_copy (const gdouble * from, gint num_classes); + +static guint64 get_new_id (void); + +static guint64 +get_new_id (void) +{ + static guint64 _id = G_GUINT64_CONSTANT (0); + static GMutex _id_mutex; + static guint64 ret = 0; + + g_mutex_lock (&_id_mutex); + ret = _id++; + g_mutex_unlock (&_id_mutex); + + return ret; +} + +static void +classification_reset (GstInferenceClassification * self) +{ + g_return_if_fail (self); + + self->classification_id = get_new_id (); + self->class_id = DEFAULT_CLASS_ID; + self->class_prob = DEFAULT_CLASS_PROB; + self->num_classes = DEFAULT_NUM_CLASSES; + + if (self->class_label) { + g_free (self->class_label); + } + self->class_label = DEFAULT_CLASS_LABEL; + + if (self->probabilities) { + g_free (self->probabilities); + } + self->probabilities = DEFAULT_PROBABILITIES; + + if (self->labels) { + g_strfreev (self->labels); + } + self->labels = DEFAULT_LABELS; +} + +GstInferenceClassification * +gst_inference_classification_new (void) +{ + GstInferenceClassification *self = g_slice_new (GstInferenceClassification); + + gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, + gst_inference_classification_get_type (), + (GstMiniObjectCopyFunction) gst_inference_classification_copy, NULL, + (GstMiniObjectFreeFunction) classification_free); + + g_mutex_init (&self->mutex); + + self->class_label = NULL; + self->probabilities = NULL; + self->labels = NULL; + + classification_reset (self); + + return self; +} + +static gdouble * +probabilities_copy (const gdouble * from, gint num_classes) +{ + gsize size = 0; + gdouble *to = NULL; + + g_return_val_if_fail (from, NULL); + g_return_val_if_fail (num_classes > 0, NULL); + + size = num_classes * sizeof (double); + to = g_malloc0 (size); + + memcpy (to, from, size); + + return to; +} + +GstInferenceClassification * +gst_inference_classification_new_full (gint class_id, gdouble class_prob, + const gchar * class_label, gint num_classes, const gdouble * probabilities, + gchar ** labels) +{ + GstInferenceClassification *self = gst_inference_classification_new (); + + GST_INFERENCE_CLASSIFICATION_LOCK (self); + + self->class_id = class_id; + self->class_prob = class_prob; + self->num_classes = num_classes; + + if (class_label) { + self->class_label = g_strdup (class_label); + } + + if (probabilities && num_classes > 0) { + self->probabilities = probabilities_copy (probabilities, num_classes); + } + + if (labels) { + self->labels = g_strdupv (labels); + } + + GST_INFERENCE_CLASSIFICATION_UNLOCK (self); + + return self; +} + +GstInferenceClassification * +gst_inference_classification_ref (GstInferenceClassification * self) +{ + g_return_val_if_fail (self, NULL); + + return (GstInferenceClassification *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); +} + +void +gst_inference_classification_unref (GstInferenceClassification * self) +{ + g_return_if_fail (self); + + gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); +} + +GstInferenceClassification * +gst_inference_classification_copy (const GstInferenceClassification * self) +{ + GstInferenceClassification *other = NULL; + + g_return_val_if_fail (self, NULL); + + other = gst_inference_classification_new (); + + GST_INFERENCE_CLASSIFICATION_LOCK ((GstInferenceClassification *) self); + + other->classification_id = self->classification_id; + other->class_id = self->class_id; + other->class_prob = self->class_prob; + other->num_classes = self->num_classes; + + if (self->class_label) { + other->class_label = g_strdup (self->class_label); + } + + if (self->probabilities) { + other->probabilities = + probabilities_copy (self->probabilities, self->num_classes); + } + + if (self->labels) { + other->labels = g_strdupv (self->labels); + } + + GST_INFERENCE_CLASSIFICATION_UNLOCK ((GstInferenceClassification *) self); + + return other; +} + +gchar * +gst_inference_classification_to_string (GstInferenceClassification * self, + gint level) +{ + gint indent = level * 2; + gchar *serial = NULL; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_CLASSIFICATION_LOCK (self); + + serial = g_strdup_printf ("{\n" + "%*s Id : %" G_GUINT64_FORMAT "\n" + "%*s Class : %d\n" + "%*s Label : %s\n" + "%*s Probability : %f\n" + "%*s Classes : %d\n" + "%*s}", + indent, "", self->classification_id, + indent, "", self->class_id, + indent, "", self->class_label, + indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); + + GST_INFERENCE_CLASSIFICATION_UNLOCK (self); + + return serial; +} + +static void +classification_free (GstInferenceClassification * self) +{ + classification_reset (self); + + g_mutex_clear (&self->mutex); +} diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h new file mode 100644 index 00000000..3b3cb95a --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -0,0 +1,159 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_CLASSIFICATION__ +#define __GST_INFERENCE_CLASSIFICATION__ + +#include + +G_BEGIN_DECLS + +/** + * GstInferenceClassification: + * @classification_id: a unique id associated to this classification + * @class_id: the numerical id associated to the assigned class + * @class_prob: the resulting probability of the assigned + * class. Typically between 0 and 1 + * @class_label: the label associated to this class or NULL if not + * available + * @num_classes: the amount of classes of the entire prediction + * @probabilities: the entire array of probabilities of the prediction + * @labels: the entire array of labels of the prediction or NULL if + * not available + */ +typedef struct _GstInferenceClassification GstInferenceClassification; +struct _GstInferenceClassification +{ + /**/ + GstMiniObject base; + GMutex mutex; + + /**/ + guint64 classification_id; + gint class_id; + gdouble class_prob; + gchar *class_label; + gint num_classes; + gdouble *probabilities; + gchar **labels; +}; + +/** + * gst_inference_classification_new: + * + * Creates a new GstInferenceClassification. + * + * Returns: A newly allocated and initialized GstInferenceClassification. + */ +GstInferenceClassification * gst_inference_classification_new (void); + +/** + * gst_inference_classification_new_full: + * @class_id: the numerical id associated to the assigned class + * @class_prob: the resulting probability of the assigned + * class. Typically between 0 and 1 + * @class_label: the label associated to this class or NULL if not + * available. A copy of the label is made if available. + * @num_classes: the amount of classes of the entire prediction + * @probabilities: the entire array of probabilities of the + * prediction. A copy of the array is made. + * @labels: the entire array of labels of the prediction or NULL if + * not available. A copy is made, if available. + * + * Creates a new GstInferenceClassification and assigns its members. + * + * Returns: A newly allocated and initialized GstInferenceClassification. + */ +GstInferenceClassification * gst_inference_classification_new_full (gint class_id, gdouble class_prob, + const gchar * class_label, gint num_classes, const gdouble * probabilities, + gchar ** labels); + +/** + * gst_inference_classification_reset: + * @self: the classification to reset + * + * Clears a classification, effectively freeing all associated memory. + */ +void gst_inference_classification_reset (GstInferenceClassification * self); + +/** + * gst_inference_classification_copy: + * @self: the classification to copy + * + * Copies a classification into a newly allocated one. This is a deep + * copy, meaning that all arrays are copied as well. No pointers are + * shared. + * + * Returns: a newly allocated copy of the original classification + */ +GstInferenceClassification * gst_inference_classification_copy (const GstInferenceClassification * self); + +/** + * gst_inference_classification_ref: + * @self: the classification to ref + * + * Increase the reference counter of the classification. + * + * Returns: the same classification, for convenience purposes. + */ +GstInferenceClassification * gst_inference_classification_ref (GstInferenceClassification * self); + +/** + * gst_inference_classification_unref: + * @self: the classification to unref + * + * Decreases the reference counter of the classification. When the + * reference counter hits zero, the classification is freed. + */ +void gst_inference_classification_unref (GstInferenceClassification * self); + +/** + * gst_inference_classification_to_string: + * @self: the classification to serialize + * + * Serializes the classification into a JSON-like string. The full + * arrays are not included. Free this string after usage using + * g_free() + * + * Returns: a string representing the classification. + */ +gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level); + +/** + * GST_INFERENCE_CLASSIFICATION_LOCK: + * @c: The GstInferenceClassification to lock + * + * Locks the classification to avoid concurrent access from different + * threads. + */ +#define GST_INFERENCE_CLASSIFICATION_LOCK(c) g_mutex_lock (&((c)->mutex)) + +/** + * GST_INFERENCE_CLASSIFICATION_UNLOCK: + * @c: The GstInferenceClassification to unlock + * + * Unlocks the prediction to yield the access to other threads. + */ +#define GST_INFERENCE_CLASSIFICATION_UNLOCK(c) g_mutex_unlock (&((c)->mutex)) + +G_END_DECLS + +#endif // __GST_INFERENCE_CLASSIFICATION__ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index ea84737b..facf0c86 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencedebug.h" void @@ -92,3 +93,22 @@ gst_inference_print_boxes (GstVideoInference * vi, GstDebugCategory * category, detect_meta->boxes[index].height, detect_meta->boxes[index].prob); } } + +void +gst_inference_print_predictions (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta) +{ + GstInferencePrediction *pred = NULL; + gchar *spred = NULL; + + g_return_if_fail (vi != NULL); + g_return_if_fail (category != NULL); + g_return_if_fail (inference_meta != NULL); + + pred = inference_meta->prediction; + spred = gst_inference_prediction_to_string (pred); + + GST_CAT_LOG (category, "\n%s", spred); + + g_free (spred); +} diff --git a/gst-libs/gst/r2inference/gstinferencedebug.h b/gst-libs/gst/r2inference/gstinferencedebug.h index d4e4acde..9c88d745 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.h +++ b/gst-libs/gst/r2inference/gstinferencedebug.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef GST_INFERENCE_DEBUG_H #define GST_INFERENCE_DEBUG_H @@ -25,7 +26,6 @@ #include G_BEGIN_DECLS - /** * \brief Display the vector with the predictions * @@ -35,8 +35,9 @@ G_BEGIN_DECLS * \param prediction Value of the prediction * \param gstlevel Level of debuging */ - -void gst_inference_print_embedding(GstVideoInference * vi, GstDebugCategory *category, GstClassificationMeta *class_meta, const gpointer prediction, GstDebugLevel gstlevel); +void gst_inference_print_embedding (GstVideoInference * vi, + GstDebugCategory * category, GstClassificationMeta * class_meta, + const gpointer prediction, GstDebugLevel gstlevel); /** * \brief Display the vector with the predictions @@ -48,7 +49,9 @@ void gst_inference_print_embedding(GstVideoInference * vi, GstDebugCategory *cat * \param gstlevel Level of debuging */ -void gst_inference_print_highest_probability(GstVideoInference * vi, GstDebugCategory *category, GstClassificationMeta *class_meta, const gpointer prediction, GstDebugLevel gstlevel); +void gst_inference_print_highest_probability (GstVideoInference * vi, + GstDebugCategory * category, GstClassificationMeta * class_meta, + const gpointer prediction, GstDebugLevel gstlevel); /** * \brief Display the vector with the predictions @@ -58,7 +61,30 @@ void gst_inference_print_highest_probability(GstVideoInference * vi, GstDebugCat * \param detect_meta Meta detected */ -void gst_inference_print_boxes(GstVideoInference * vi, GstDebugCategory *category, GstDetectionMeta *detect_meta); +void gst_inference_print_boxes (GstVideoInference * vi, + GstDebugCategory * category, GstDetectionMeta * detect_meta); + +/** + * \brief Display the predictions in the inference meta + * + * \param vi Father object of every architecture + * \param category The debug category + * \param inference_meta Inference Meta + */ + +void gst_inference_print_predictions (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta); + +/** + * \brief Display the classes in the inference meta + * + * \param vi Father object of every architecture + * \param category The debug category + * \param inference_meta Inference Meta + */ + +void gst_inference_print_classes (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta); G_END_DECLS #endif // GST_INFERENCE_DEBUG_H diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 8b3a50a8..b4b44b1e 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -1,19 +1,34 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #include "gstinferencemeta.h" #include #include +static gboolean gst_inference_meta_init (GstMeta * meta, + gpointer params, GstBuffer * buffer); +static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); +static gboolean gst_inference_meta_transform (GstBuffer * transbuf, + GstMeta * meta, GstBuffer * buffer, GQuark type, gpointer data); static gboolean gst_classification_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -32,6 +47,36 @@ static gboolean gst_detection_meta_scale (GstBuffer * transbuf, static gboolean gst_classification_meta_copy (GstBuffer * transbuf, GstMeta * meta, GstBuffer * buffer); +GType +gst_inference_meta_api_get_type (void) +{ + static volatile GType type = 0; + static const gchar *tags[] = { GST_META_TAG_VIDEO_STR, NULL + }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstInferenceMetaAPI", tags); + g_once_init_leave (&type, _type); + } + return type; +} + +/* inference metadata */ +const GstMetaInfo * +gst_inference_meta_get_info (void) +{ + static const GstMetaInfo *inference_meta_info = NULL; + + if (g_once_init_enter (&inference_meta_info)) { + const GstMetaInfo *meta = gst_meta_register (GST_INFERENCE_META_API_TYPE, + "GstInferenceMeta", sizeof (GstInferenceMeta), + gst_inference_meta_init, gst_inference_meta_free, + gst_inference_meta_transform); + g_once_init_leave (&inference_meta_info, meta); + } + return inference_meta_info; +} + GType gst_embedding_meta_api_get_type (void) { @@ -97,9 +142,7 @@ GType gst_detection_meta_api_get_type (void) { static volatile GType type = 0; - static const gchar *tags[] = - { GST_META_TAG_VIDEO_STR, GST_META_TAG_VIDEO_ORIENTATION_STR, - GST_META_TAG_VIDEO_SIZE_STR, NULL + static const gchar *tags[] = { GST_META_TAG_VIDEO_STR, NULL }; if (g_once_init_enter (&type)) { @@ -151,6 +194,135 @@ gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer) } } +static gboolean +gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstInferenceMeta *dmeta, *smeta; + GstInferencePrediction *pred = NULL; + gboolean ret = TRUE; + gboolean needs_scale = FALSE; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); + + smeta = (GstInferenceMeta *) meta; + dmeta = + (GstInferenceMeta *) gst_buffer_get_meta (dest, + gst_inference_meta_api_get_type ()); + + g_return_val_if_fail (dmeta, FALSE); + + pred = + gst_inference_prediction_find (dmeta->prediction, + smeta->prediction->prediction_id); + + if (!pred) { + GST_ERROR + ("Predictions between metas do not match. Something really wrong happened"); + g_return_val_if_reached (FALSE); + } + + needs_scale = gst_inference_prediction_merge (smeta->prediction, pred); + + /* Transfer Stream ID */ + g_free (dmeta->stream_id); + dmeta->stream_id = g_strdup (smeta->stream_id); + + if (GST_META_TRANSFORM_IS_COPY (type)) { + GST_LOG ("Copy inference metadata"); + + /* The merge already handled the copy */ + goto out; + } + + if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { + if (needs_scale) { + GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + gst_inference_prediction_scale_ip (pred, trans->out_info, trans->in_info); + + } + goto out; + } + + /* Invalid transformation */ + gst_buffer_remove_meta (dest, (GstMeta *) dmeta); + ret = FALSE; + +out: + gst_inference_prediction_unref (pred); + return ret; +} + +static gboolean +gst_inference_meta_transform_new_meta (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstInferenceMeta *dmeta, *smeta; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); + + smeta = (GstInferenceMeta *) meta; + dmeta = + (GstInferenceMeta *) gst_buffer_add_meta (dest, GST_INFERENCE_META_INFO, + NULL); + if (!dmeta) { + GST_ERROR ("Unable to add meta to buffer"); + return FALSE; + } + + gst_inference_prediction_unref (dmeta->prediction); + + /* Transfer Stream ID */ + g_free (dmeta->stream_id); + dmeta->stream_id = g_strdup (smeta->stream_id); + + if (GST_META_TRANSFORM_IS_COPY (type)) { + GST_LOG ("Copy inference metadata"); + + dmeta->prediction = gst_inference_prediction_copy (smeta->prediction); + return TRUE; + } + + if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { + GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + + dmeta->prediction = + gst_inference_prediction_scale (smeta->prediction, trans->out_info, + trans->in_info); + return TRUE; + } + + /* Invalid transformation */ + gst_buffer_remove_meta (dest, (GstMeta *) dmeta); + return FALSE; +} + +static gboolean +gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstMeta *dmeta; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); + + GST_LOG ("Transforming inference metadata"); + + dmeta = gst_buffer_get_meta (dest, gst_inference_meta_api_get_type ()); + if (!dmeta) { + return gst_inference_meta_transform_new_meta (dest, meta, buffer, type, + data); + } else { + return gst_inference_meta_transform_existing_meta (dest, meta, buffer, type, + data); + } +} + static gboolean gst_detection_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) { @@ -312,10 +484,36 @@ gst_classification_meta_transform (GstBuffer * dest, GstMeta * meta, { GST_LOG ("Transforming detection metadata"); - if (GST_META_TRANSFORM_IS_COPY (type)) { - return gst_classification_meta_copy (dest, meta, buffer); - } + /* TODO: Eventually check for the specific transformation here. + Fail if its an unsupported transformation */ + return gst_classification_meta_copy (dest, meta, buffer); +} - /* No transform supported */ - return FALSE; +/* inference metadata functions */ +static gboolean +gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) +{ + GstInferenceMeta *imeta = (GstInferenceMeta *) meta; + GstInferencePrediction *root; + + /* Create root Prediction */ + root = gst_inference_prediction_new (); + + imeta->prediction = root; + imeta->stream_id = NULL; + + return TRUE; +} + +static void +gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstInferenceMeta *imeta = NULL; + + g_return_if_fail (meta != NULL); + g_return_if_fail (buffer != NULL); + + imeta = (GstInferenceMeta *) meta; + gst_inference_prediction_unref (imeta->prediction); + g_free (imeta->stream_id); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 76bdc92d..59902943 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -1,19 +1,31 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #ifndef GST_INFERENCE_META_H #define GST_INFERENCE_META_H #include +#include + G_BEGIN_DECLS #define GST_EMBEDDING_META_API_TYPE (gst_embedding_meta_api_get_type()) #define GST_EMBEDDING_META_INFO (gst_embedding_meta_get_info()) @@ -21,6 +33,9 @@ G_BEGIN_DECLS #define GST_CLASSIFICATION_META_INFO (gst_classification_meta_get_info()) #define GST_DETECTION_META_API_TYPE (gst_detection_meta_api_get_type()) #define GST_DETECTION_META_INFO (gst_detection_meta_get_info()) +#define GST_INFERENCE_META_API_TYPE (gst_inference_meta_api_get_type()) +#define GST_INFERENCE_META_INFO (gst_inference_meta_get_info()) + /** * Basic bounding box structure for detection */ @@ -35,6 +50,19 @@ struct _BBox gdouble height; }; +/** + * Implements the placeholder for inference information. + */ +typedef struct _GstInferenceMeta GstInferenceMeta; +struct _GstInferenceMeta +{ + GstMeta meta; + + GstInferencePrediction *prediction; + + gchar *stream_id; +}; + /** * Implements the placeholder for embedding information. */ @@ -68,6 +96,9 @@ struct _GstDetectionMeta BBox *boxes; }; +GType gst_inference_meta_api_get_type (void); +const GstMetaInfo *gst_inference_meta_get_info (void); + GType gst_embedding_meta_api_get_type (void); const GstMetaInfo *gst_embedding_meta_get_info (void); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index baebb32e..fa3cc0aa 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencepostprocess.h" #include #include @@ -25,6 +26,8 @@ #define TOTAL_BOXES_5 845 #define TOTAL_BOXES_15 2535 +#define TOTAL_CLASSES 20 + /* Functions declaration*/ static gdouble gst_intersection_over_union (BBox box_1, BBox box_2); @@ -36,7 +39,7 @@ static void gst_box_to_pixels (BBox * normalized_box, gint row, gint col, static gdouble gst_sigmoid (gdouble x); static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint grid_h, gint grid_w, gint boxes_size); + gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities); static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes); @@ -168,7 +171,7 @@ gst_box_to_pixels (BBox * normalized_box, gint row, gint col, gint box) static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint grid_h, - gint grid_w, gint boxes_size) + gint grid_w, gint boxes_size, gdouble * probabilities) { gint i, j, c, b; gint index; @@ -177,7 +180,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gint max_class_prob_index; gint counter = 0; gint box_dim = 5; - gint classes = 20; + gint classes = TOTAL_CLASSES; g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); @@ -196,6 +199,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, max_class_prob_index = 0; for (c = 0; c < classes; c++) { cur_class_prob = ((gfloat *) prediction)[index + box_dim + c]; + probabilities[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { max_class_prob = cur_class_prob; max_class_prob_index = c; @@ -224,33 +228,107 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta * detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh) + gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, + gdouble ** probabilities) { gint grid_h = 13; gint grid_w = 13; gint boxes_size = 5; BBox boxes[TOTAL_BOXES_5]; - *elements = 0; g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); - g_return_val_if_fail (detect_meta != NULL, FALSE); - g_return_val_if_fail (info_model != NULL, FALSE); g_return_val_if_fail (valid_prediction != NULL, FALSE); g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); + g_return_val_if_fail (probabilities != NULL, FALSE); + + *elements = 0; + *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, - elements, grid_h, grid_w, boxes_size); + elements, grid_h, grid_w, boxes_size, *probabilities); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); memcpy (*resulting_boxes, boxes, *elements * sizeof (BBox)); + return TRUE; } +GstInferencePrediction * +gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, + gchar ** labels_list, gint num_labels, const gdouble * probabilities) +{ + GstInferencePrediction *predict = NULL; + GstInferenceClassification *c = NULL; + gchar *label = NULL; + BoundingBox bbox; + + g_return_val_if_fail (vi != NULL, NULL); + g_return_val_if_fail (box != NULL, NULL); + g_return_val_if_fail (probabilities != NULL, NULL); + + bbox.x = box->x; + bbox.y = box->y; + bbox.width = box->width; + bbox.height = box->height; + + predict = gst_inference_prediction_new_full (&bbox); + + if (num_labels > box->label) { + label = labels_list[box->label]; + } + c = gst_inference_classification_new_full (box->label, box->prob, label, + num_labels, probabilities, labels_list); + gst_inference_prediction_append_classification (predict, c); + + return predict; +} + +GstInferenceClassification * +gst_create_class_from_prediction (GstVideoInference * vi, + const gpointer prediction, gsize predsize, gchar ** labels_list, + gint num_labels) +{ + gdouble max = -1; + gint index = 0; + gdouble *probs = NULL; + gint num_classes = 0; + const gchar *label = NULL; + + g_return_val_if_fail (vi != NULL, NULL); + + num_classes = predsize; + + /* FIXME: This is just dumb */ + if (sizeof (gfloat) != sizeof (gdouble)) { + probs = g_malloc (num_classes * sizeof (gdouble)); + + for (gint i = 0; i < num_classes; ++i) { + probs[i] = (gdouble) ((gfloat *) prediction)[i]; + } + } else { + probs = (gdouble *) prediction; + } + + /* Obtain the highest probability and set it as class */ + for (gint i = 0; i < num_classes; ++i) { + gdouble current = probs[i]; + if (current > max) { + max = current; + index = i; + } + } + + if (num_labels > index) { + label = labels_list[index]; + } + return gst_inference_classification_new_full (index, max, label, num_classes, + probs, labels_list); +} + static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes) @@ -307,7 +385,6 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta * detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh) @@ -318,8 +395,6 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); - g_return_val_if_fail (detect_meta != NULL, FALSE); - g_return_val_if_fail (info_model != NULL, FALSE); g_return_val_if_fail (valid_prediction != NULL, FALSE); g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index a691cfed..779f8286 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -26,7 +26,6 @@ #define __GST_INFERENCE_POSTPROCESS_H__ G_BEGIN_DECLS - /** * \brief Fill all the classification meta with predictions * @@ -34,17 +33,14 @@ G_BEGIN_DECLS * \param prediction Value of the prediction * \param predsize Size of the prediction */ - -gboolean gst_fill_classification_meta(GstClassificationMeta *class_meta, const gpointer prediction, - gsize predsize); + gboolean gst_fill_classification_meta (GstClassificationMeta * class_meta, + const gpointer prediction, gsize predsize); /** * \brief Fill all the detection meta with the boxes * * \param vi Father object of every architecture * \param prediction Value of the prediction - * \param detect_meta Meta to fill - * \param info_model Info about the model to use * \param valid_prediction Check if the prediction is valid * \param resulting_boxes The output boxes of the prediction * \param elements The number of objects @@ -53,17 +49,14 @@ gboolean gst_fill_classification_meta(GstClassificationMeta *class_meta, const g * \param iou_thresh Intersection over union threshold */ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta *detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh); + gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, gdouble ** probabilities); /** * \brief Fill all the detection meta with the boxes * * \param vi Father object of every architecture * \param prediction Value of the prediction - * \param detect_meta Meta to fill - * \param info_model Info about the model to use * \param valid_prediction Check if the prediction is valid * \param resulting_boxes The output boxes of the prediction * \param elements The number of objects @@ -71,11 +64,33 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, * \param prob_thresh Class probability threshold * \param iou_thresh Intersection over union threshold */ -gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta *detect_meta, GstVideoInfo * info_model, - gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh); +gboolean gst_create_boxes_float (GstVideoInference * vi, + const gpointer prediction, gboolean * valid_prediction, + BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, + gdouble prob_thresh, gdouble iou_thresh); -G_END_DECLS +/** + * \brief Create Prediction from box + * + * \param vi Father object of every architecture + * \param box Box used to fill Prediction + * \param labels_list List with all possible lables + * \param num_labels The number of posibble labels + */ +GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, + BBox * box, gchar **labels_list, gint num_labels, const gdouble * probabilities); +/** + * \brief Create Classification from prediction data + * + * \param vi Father object of every architecture + * \param prediction Value of the prediction + * \param predsize Size of the prediction + * \param labels_list List with all possible lables + * \param num_labels The number of posibble labels + */ +GstInferenceClassification *gst_create_class_from_prediction (GstVideoInference * vi, + const gpointer prediction, gsize predsize, gchar **labels_list, gint num_labels); + +G_END_DECLS #endif diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c new file mode 100644 index 00000000..35bda23c --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -0,0 +1,801 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstinferenceprediction.h" + +static GType gst_inference_prediction_get_type (void); +GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); + +typedef struct _PredictionScaleData PredictionScaleData; +struct _PredictionScaleData +{ + GstVideoInfo *from; + GstVideoInfo *to; +}; + +typedef struct _PredictionFindData PredictionFindData; +struct _PredictionFindData +{ + GstInferencePrediction *prediction; + guint64 prediction_id; +}; + +static void gst_inference_prediction_free (GstInferencePrediction * self); +static GstInferencePrediction *prediction_copy (const GstInferencePrediction * + self); +static void prediction_free (GstInferencePrediction * obj); +static gboolean prediction_find (GstInferencePrediction * obj, + PredictionFindData * found); +static GstInferencePrediction *prediction_find_unlocked (GstInferencePrediction + * self, guint64 id); +static void prediction_reset (GstInferencePrediction * self); +static gchar *prediction_to_string (GstInferencePrediction * self, gint level); +static gchar *prediction_children_to_string (GstInferencePrediction * self, + gint level); +static gchar *prediction_classes_to_string (GstInferencePrediction * self, + gint level); +static GstInferencePrediction *prediction_scale (const GstInferencePrediction * + self, GstVideoInfo * to, GstVideoInfo * from); +static void prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); +static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); +static gboolean prediction_merge (GstInferencePrediction * src, + GstInferencePrediction * dst); +static void prediction_get_enabled (GstInferencePrediction * self, + GList ** enabled); + +static GstInferenceClassification + * classification_copy (GstInferenceClassification * from, gpointer data); +static void classification_merge (GList * src, GList ** dst); +static gint classification_compare (gconstpointer a, gconstpointer b); + +static void bounding_box_reset (BoundingBox * bbox); +static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); + +static void node_get_children (GNode * node, gpointer data); +static gpointer node_copy (gconstpointer node, gpointer data); +static gboolean node_scale_ip (GNode * node, gpointer data); +static gpointer node_scale (gconstpointer, gpointer data); +static gboolean node_assign (GNode * node, gpointer data); +static gboolean node_find (GNode * node, gpointer data); +static gboolean node_get_enabled (GNode * node, gpointer data); + +static void compute_factors (GstVideoInfo * from, GstVideoInfo * to, + gdouble * hfactor, gdouble * vfactor); +static guint64 get_new_id (void); + +static guint64 +get_new_id (void) +{ + static guint64 _id = G_GUINT64_CONSTANT (0); + static GMutex _id_mutex; + static guint64 ret = 0; + + g_mutex_lock (&_id_mutex); + ret = _id++; + g_mutex_unlock (&_id_mutex); + + return ret; +} + +GstInferencePrediction * +gst_inference_prediction_new (void) +{ + GstInferencePrediction *self = g_slice_new (GstInferencePrediction); + + gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, + gst_inference_prediction_get_type (), + (GstMiniObjectCopyFunction) gst_inference_prediction_copy, NULL, + (GstMiniObjectFreeFunction) gst_inference_prediction_free); + + g_mutex_init (&self->mutex); + + self->predictions = NULL; + self->classifications = NULL; + + prediction_reset (self); + + return self; +} + +GstInferencePrediction * +gst_inference_prediction_new_full (BoundingBox * bbox) +{ + GstInferencePrediction *self = NULL; + + g_return_val_if_fail (bbox, NULL); + + self = gst_inference_prediction_new (); + + GST_INFERENCE_PREDICTION_LOCK (self); + self->bbox = *bbox; + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return self; +} + +GstInferencePrediction * +gst_inference_prediction_ref (GstInferencePrediction * self) +{ + g_return_val_if_fail (self, NULL); + + return (GstInferencePrediction *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); +} + +void +gst_inference_prediction_unref (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); +} + +void +gst_inference_prediction_append (GstInferencePrediction * self, + GstInferencePrediction * child) +{ + g_return_if_fail (self); + g_return_if_fail (child); + + GST_INFERENCE_PREDICTION_LOCK (self); + GST_INFERENCE_PREDICTION_LOCK (child); + g_node_append (self->predictions, child->predictions); + GST_INFERENCE_PREDICTION_UNLOCK (child); + GST_INFERENCE_PREDICTION_UNLOCK (self); +} + +static GstInferenceClassification * +classification_copy (GstInferenceClassification * from, gpointer data) +{ + return gst_inference_classification_copy (from); +} + +static GstInferencePrediction * +prediction_copy (const GstInferencePrediction * self) +{ + GstInferencePrediction *other = NULL; + + g_return_val_if_fail (self, NULL); + + other = gst_inference_prediction_new (); + + other->prediction_id = self->prediction_id; + other->enabled = self->enabled; + other->bbox = self->bbox; + + other->classifications = + g_list_copy_deep (self->classifications, (GCopyFunc) classification_copy, + NULL); + + return other; +} + +static gpointer +node_copy (gconstpointer node, gpointer data) +{ + GstInferencePrediction *self = (GstInferencePrediction *) node; + + g_return_val_if_fail (node, NULL); + + return prediction_copy (self); +} + +static gboolean +node_assign (GNode * node, gpointer data) +{ + GstInferencePrediction *pred = (GstInferencePrediction *) node->data; + + g_return_val_if_fail (node, FALSE); + + pred->predictions = node; + + return FALSE; +} + +GstInferencePrediction * +gst_inference_prediction_copy (const GstInferencePrediction * self) +{ + GNode *other = NULL; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_PREDICTION_LOCK ((GstInferencePrediction *) self); + + /* Copy the binary tree */ + other = g_node_copy_deep (self->predictions, node_copy, NULL); + + /* Now finish assigning the nodes to the predictions */ + g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + + GST_INFERENCE_PREDICTION_UNLOCK ((GstInferencePrediction *) self); + + return (GstInferencePrediction *) other->data; +} + +static gchar * +bounding_box_to_string (BoundingBox * bbox, gint level) +{ + gint indent = level * 2; + + g_return_val_if_fail (bbox, NULL); + + return g_strdup_printf ("{\n" + "%*s x : %d\n" + "%*s y : %d\n" + "%*s width : %u\n" + "%*s height : %u\n" + "%*s}", + indent, "", bbox->x, + indent, "", bbox->y, + indent, "", bbox->width, indent, "", bbox->height, indent, ""); +} + +static gchar * +prediction_children_to_string (GstInferencePrediction * self, gint level) +{ + GSList *subpreds = NULL; + GSList *iter = NULL; + GString *string = NULL; + + g_return_val_if_fail (self, NULL); + + /* Build the child predictions using a GString */ + string = g_string_new (NULL); + + subpreds = prediction_get_children_unlocked (self); + + for (iter = subpreds; iter != NULL; iter = g_slist_next (iter)) { + GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; + gchar *child = prediction_to_string (pred, level + 1); + + g_string_append_printf (string, "%s, ", child); + g_free (child); + } + + g_slist_free (subpreds); + + return g_string_free (string, FALSE); +} + +static gchar * +prediction_classes_to_string (GstInferencePrediction * self, gint level) +{ + GList *iter = NULL; + GString *string = NULL; + + g_return_val_if_fail (self, NULL); + + /* Build the classes for this predictions using a GString */ + string = g_string_new (NULL); + + for (iter = self->classifications; iter != NULL; iter = g_list_next (iter)) { + GstInferenceClassification *c = (GstInferenceClassification *) iter->data; + gchar *sclass = gst_inference_classification_to_string (c, level + 1); + + g_string_append_printf (string, "%s, ", sclass); + g_free (sclass); + } + + return g_string_free (string, FALSE); +} + +static gchar * +prediction_to_string (GstInferencePrediction * self, gint level) +{ + gint indent = level * 2; + gchar *bbox = NULL; + gchar *children = NULL; + gchar *classes = NULL; + gchar *prediction = NULL; + + g_return_val_if_fail (self, NULL); + + bbox = bounding_box_to_string (&self->bbox, level + 1); + classes = prediction_classes_to_string (self, level + 1); + children = prediction_children_to_string (self, level + 1); + + prediction = g_strdup_printf ("{\n" + "%*s id : %" G_GUINT64_FORMAT ",\n" + "%*s enabled : %s,\n" + "%*s bbox : %s,\n" + "%*s classes : [\n" + "%*s %s\n" + "%*s ],\n" + "%*s predictions : [\n" + "%*s %s\n" + "%*s ]\n" + "%*s}", + indent, "", self->prediction_id, + indent, "", self->enabled ? "True" : "False", + indent, "", bbox, + indent, "", indent, "", classes, indent, "", + indent, "", indent, "", children, indent, "", indent, ""); + + g_free (bbox); + g_free (children); + g_free (classes); + + return prediction; +} + +gchar * +gst_inference_prediction_to_string (GstInferencePrediction * self) +{ + gchar *serial = NULL; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_PREDICTION_LOCK (self); + serial = prediction_to_string (self, 0); + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return serial; +} + +static void +node_get_children (GNode * node, gpointer data) +{ + GSList **children = (GSList **) data; + GstInferencePrediction *prediction; + + g_return_if_fail (node); + g_return_if_fail (children); + + prediction = (GstInferencePrediction *) node->data; + + *children = g_slist_append (*children, prediction); +} + +static GSList * +prediction_get_children_unlocked (GstInferencePrediction * self) +{ + GSList *children = NULL; + + g_return_val_if_fail (self, NULL); + + if (self->predictions) { + g_node_children_foreach (self->predictions, G_TRAVERSE_ALL, + node_get_children, &children); + } + + return children; +} + +GSList * +gst_inference_prediction_get_children (GstInferencePrediction * self) +{ + GSList *children = NULL; + + GST_INFERENCE_PREDICTION_LOCK (self); + children = prediction_get_children_unlocked (self); + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return children; +} + +static void +bounding_box_reset (BoundingBox * bbox) +{ + g_return_if_fail (bbox); + + bbox->x = 0; + bbox->y = 0; + bbox->width = 0; + bbox->height = 0; +} + +static void +prediction_reset (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + self->prediction_id = get_new_id (); + self->enabled = TRUE; + + bounding_box_reset (&self->bbox); + + /* Free al children */ + prediction_free (self); + + self->predictions = g_node_new (self); +} + +static void +prediction_free (GstInferencePrediction * self) +{ + GSList *children = prediction_get_children_unlocked (self); + + /* Free all children recursively */ + g_slist_free_full (children, (GDestroyNotify) gst_inference_prediction_unref); + + /* Now free our classifications */ + g_list_free_full (self->classifications, + (GDestroyNotify) gst_inference_classification_unref); + self->classifications = NULL; + + if (self->predictions) { + g_node_destroy (self->predictions); + self->predictions = NULL; + } +} + +static void +gst_inference_prediction_free (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + prediction_free (self); + + g_mutex_clear (&self->mutex); +} + +void +gst_inference_prediction_append_classification (GstInferencePrediction * self, + GstInferenceClassification * c) +{ + g_return_if_fail (self); + g_return_if_fail (c); + + GST_INFERENCE_PREDICTION_LOCK (self); + self->classifications = g_list_append (self->classifications, c); + GST_INFERENCE_PREDICTION_UNLOCK (self); +} + +static void +compute_factors (GstVideoInfo * from, GstVideoInfo * to, gdouble * hfactor, + gdouble * vfactor) +{ + gint fw, fh, tw, th; + + g_return_if_fail (from); + g_return_if_fail (to); + g_return_if_fail (hfactor); + g_return_if_fail (vfactor); + + fw = GST_VIDEO_INFO_WIDTH (from); + tw = GST_VIDEO_INFO_WIDTH (to); + fh = GST_VIDEO_INFO_HEIGHT (from); + th = GST_VIDEO_INFO_HEIGHT (to); + + g_return_if_fail (fw); + g_return_if_fail (fh); + + *hfactor = tw * 1.0 / fw; + *vfactor = th * 1.0 / fh; +} + +static GstInferencePrediction * +prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, + GstVideoInfo * from) +{ + GstInferencePrediction *dest = NULL; + + gdouble hfactor, vfactor; + + g_return_val_if_fail (self, NULL); + g_return_val_if_fail (to, NULL); + g_return_val_if_fail (from, NULL); + + dest = prediction_copy (self); + + compute_factors (from, to, &hfactor, &vfactor); + + dest->bbox.x = self->bbox.x * hfactor; + dest->bbox.y = self->bbox.y * vfactor; + + dest->bbox.width = self->bbox.width * hfactor; + dest->bbox.height = self->bbox.height * vfactor; + + GST_LOG ("scaled bbox: %dx%d@%dx%d -> %dx%d@%dx%d", + self->bbox.x, self->bbox.y, self->bbox.width, + self->bbox.height, dest->bbox.x, dest->bbox.y, + dest->bbox.width, dest->bbox.height); + + return dest; +} + +static void +prediction_scale_ip (GstInferencePrediction * self, GstVideoInfo * to, + GstVideoInfo * from) +{ + gdouble hfactor, vfactor; + + g_return_if_fail (self); + g_return_if_fail (to); + g_return_if_fail (from); + + compute_factors (from, to, &hfactor, &vfactor); + + self->bbox.x = self->bbox.x * hfactor; + self->bbox.y = self->bbox.y * vfactor; + + self->bbox.width = self->bbox.width * hfactor; + self->bbox.height = self->bbox.height * vfactor; +} + +static gboolean +node_scale_ip (GNode * node, gpointer data) +{ + GstInferencePrediction *self = (GstInferencePrediction *) node->data; + PredictionScaleData *sdata = (PredictionScaleData *) data; + + prediction_scale_ip (self, sdata->to, sdata->from); + + return FALSE; +} + +static gpointer +node_scale (gconstpointer node, gpointer data) +{ + const GstInferencePrediction *self = (GstInferencePrediction *) node; + PredictionScaleData *sdata = (PredictionScaleData *) data; + + return prediction_scale (self, sdata->to, sdata->from); +} + +void +gst_inference_prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from) +{ + PredictionScaleData data = {.from = from,.to = to }; + + g_return_if_fail (self); + g_return_if_fail (to); + g_return_if_fail (from); + + GST_INFERENCE_PREDICTION_LOCK (self); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, + node_scale_ip, &data); + + GST_INFERENCE_PREDICTION_UNLOCK (self); +} + +GstInferencePrediction * +gst_inference_prediction_scale (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from) +{ + GNode *other = NULL; + PredictionScaleData data = {.from = from,.to = to }; + + g_return_val_if_fail (self, NULL); + g_return_val_if_fail (to, NULL); + g_return_val_if_fail (from, NULL); + + GST_INFERENCE_PREDICTION_LOCK (self); + + other = g_node_copy_deep (self->predictions, node_scale, &data); + + /* Now finish assigning the nodes to the predictions */ + g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return (GstInferencePrediction *) other->data; +} + +static gboolean +prediction_find (GstInferencePrediction * obj, PredictionFindData * found) +{ + gboolean ret = FALSE; + + g_return_val_if_fail (obj, TRUE); + g_return_val_if_fail (found, TRUE); + + if (obj->prediction_id == found->prediction_id) { + found->prediction = gst_inference_prediction_ref (obj); + ret = TRUE; + } + + return ret; +} + +static gboolean +node_find (GNode * node, gpointer data) +{ + PredictionFindData *found = (PredictionFindData *) data; + GstInferencePrediction *current = (GstInferencePrediction *) node->data; + + g_return_val_if_fail (found, TRUE); + + return prediction_find (current, found); +} + +static GstInferencePrediction * +prediction_find_unlocked (GstInferencePrediction * self, guint64 id) +{ + PredictionFindData found = { 0, id }; + + g_return_val_if_fail (self, NULL); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_find, + &found); + + return found.prediction; +} + +GstInferencePrediction * +gst_inference_prediction_find (GstInferencePrediction * self, guint64 id) +{ + GstInferencePrediction *found = NULL; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_PREDICTION_LOCK (self); + + found = prediction_find_unlocked (self, id); + + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return found; +} + +static gint +classification_compare (gconstpointer a, gconstpointer b) +{ + GstInferenceClassification *ca = (GstInferenceClassification *) a; + GstInferenceClassification *cb = (GstInferenceClassification *) b; + + return ca->classification_id == cb->classification_id ? 0 : 1; +} + +static void +classification_merge (GList * src, GList ** dst) +{ + GList *iter = NULL; + + g_return_if_fail (dst); + + /* For each classification in the src, see if it exists in the dst */ + for (iter = src; iter; iter = g_list_next (iter)) { + GList *exists = + g_list_find_custom (*dst, iter->data, classification_compare); + + /* Copy and append it to the dst if it doesn't exist */ + if (!exists) { + GstInferenceClassification *child = + gst_inference_classification_copy (iter->data); + *dst = g_list_append (*dst, child); + } + } +} + +static gboolean +prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) +{ + GSList *src_children = prediction_get_children_unlocked (src); + GSList *iter = NULL; + GSList *new_children = NULL; + gboolean new_added = FALSE; + + g_return_val_if_fail (src, FALSE); + g_return_val_if_fail (dst, FALSE); + g_return_val_if_fail (src->prediction_id == dst->prediction_id, FALSE); + + /* Two things might've happened: + * 1) A new class was added + * 2) A new subprediction was added + */ + + /* Handle 1) here */ + classification_merge (src->classifications, &dst->classifications); + + /* Handle 2) here */ + for (iter = src_children; iter; iter = g_slist_next (iter)) { + GstInferencePrediction *current = (GstInferencePrediction *) iter->data; + + /* TODO: Optimize this by only searching the immediate children */ + GstInferencePrediction *found = + prediction_find_unlocked (dst, current->prediction_id); + + /* No matching prediction, save it to append it later */ + if (!found) { + new_children = g_slist_append (new_children, current); + continue; + } + + /* Recurse into the children */ + new_added = prediction_merge (current, found); + + gst_inference_prediction_unref (found); + } + + /* Finally append all the new children to dst. Do it after all + children have been processed */ + for (iter = new_children; iter; iter = g_slist_next (iter)) { + GstInferencePrediction *prediction = + gst_inference_prediction_copy ((GstInferencePrediction *) iter->data); + gst_inference_prediction_append (dst, prediction); + } + + new_added |= new_children ? TRUE : FALSE; + + g_slist_free (src_children); + g_slist_free_full (new_children, + (GDestroyNotify) gst_inference_prediction_unref); + + return new_added; +} + +gboolean +gst_inference_prediction_merge (GstInferencePrediction * src, + GstInferencePrediction * dst) +{ + gboolean needs_scale = FALSE; + + g_return_val_if_fail (src, FALSE); + g_return_val_if_fail (dst, FALSE); + + if (src == dst) { + return needs_scale; + } + + GST_INFERENCE_PREDICTION_LOCK (src); + GST_INFERENCE_PREDICTION_LOCK (dst); + + needs_scale = prediction_merge (src, dst); + + GST_INFERENCE_PREDICTION_UNLOCK (dst); + GST_INFERENCE_PREDICTION_UNLOCK (src); + + return needs_scale; +} + +static void +prediction_get_enabled (GstInferencePrediction * self, GList ** found) +{ + g_return_if_fail (self); + g_return_if_fail (found); + + if (self->enabled) { + *found = g_list_append (*found, self); + } +} + +static gboolean +node_get_enabled (GNode * node, gpointer data) +{ + GstInferencePrediction *self = NULL; + GList **found = (GList **) data; + + g_return_val_if_fail (node, TRUE); + g_return_val_if_fail (found, TRUE); + + self = (GstInferencePrediction *) node->data; + + prediction_get_enabled (self, found); + + return FALSE; +} + +GList * +gst_inference_prediction_get_enabled (GstInferencePrediction * self) +{ + GList *found = NULL; + + g_return_val_if_fail (self, NULL); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, + node_get_enabled, &found); + + return found; +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h new file mode 100644 index 00000000..79a66640 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -0,0 +1,274 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_PREDICTION__ +#define __GST_INFERENCE_PREDICTION__ + +#include +#include + +G_BEGIN_DECLS + +typedef struct _BoundingBox BoundingBox; +typedef struct _GstInferencePrediction GstInferencePrediction; + +/** + * BoundingBox: + * @x: horizontal coordinate of the upper left position of the + * bounding box in pixels + * @y: vertical coordinate of the upper left position of the bounding + * box in pixels + * @width: width of the bounding box in pixels + * @height: height of the bounding box in pixels + * + * Size and coordinates of a prediction region + */ +struct _BoundingBox +{ + gint x; + gint y; + guint width; + guint height; +}; + +/** + * GstInferencePrediction: + * @prediction_id: unique id for this specific prediction + * @enabled: flag indicating wether or not this prediction should be + * used for further inference + * @bbox: the BoundingBox for this specific prediction + * @classifications: a linked list of GstInfereferenceClassification + * associated to this prediction + * @predictions: a n-ary tree of child predictions within this + * specific prediction. It is recommended to to access the tree + * directly, but to use this module's API to interact with the + * children. + * + * Abstraction that represents a prediction + */ +struct _GstInferencePrediction +{ + /**/ + GstMiniObject base; + GMutex mutex; + + /**/ + guint64 prediction_id; + gboolean enabled; + BoundingBox bbox; + GList * classifications; + GNode * predictions; +}; + +/** + * gst_inference_prediction_new: + * + * Creates a new GstInferencePrediction. Values can be later assigned + * manually, however these assignments should be done with the + * GST_INFERENCE_PREDICTION_LOCK held. See + * gst_inference_prediction_new_full for a thread safe version. + * + * Returns: A newly allocated and initialized GstInferencePrediction. + */ +GstInferencePrediction * gst_inference_prediction_new (void); + +/** + * gst_inference_prediction_new_full: + * @bbox: The bounding box of this prediction. + * + * Creates a new GstInferencePrediction and initializes its internal + * values. + * + * Returns: A newly allocated and initialized GstInferencePrediction. + */ +GstInferencePrediction * gst_inference_prediction_new_full (BoundingBox *bbox); + +/** + * gst_inference_prediction_reset: + * @self: the prediction to reset + * + * Clears a prediction, effectively removing al children and resetting + * all members. + */ +void gst_inference_prediction_reset (GstInferencePrediction * self); + +/** + * gst_inference_prediction_copy: + * @self: the prediction to copy + * + * Copies a prediction into a newly allocated one. This is a deep + * copy, meaning that all children and classifications are copied as + * well. No references are shared. + * + * Returns: a newly allocated copy of the original prediction + */ +GstInferencePrediction * gst_inference_prediction_copy (const GstInferencePrediction * self); + +/** + * gst_inference_prediction_ref: + * @self: the prediction to ref + * + * Increase the reference counter of the prediction. + * + * Returns: the same prediction, for convenience purposes. + */ +GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self); + +/** + * gst_inference_prediction_unref: + * @self: the prediction to unref + * + * Decreases the reference counter of the prediction. When the + * reference counter hits zero, the prediction is freed. + */ +void gst_inference_prediction_unref (GstInferencePrediction * self); + +/** + * gst_inference_prediction_to_string: + * @self: the prediction to serialize + * + * Serializes the prediction along with it's classifications and + * children into a JSON-like string. Free this string after usage + * using g_free() + * + * Returns: a string representing the prediction. + */ +gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); + +/** + * gst_inference_prediction_append: + * @self: the parent prediction + * @child: the prediction to append as a child + * + * Append a new prediction as part of the parent prediction + * children. The parent takes ownership, use + * gst_inference_prediction_ref() if you wish to keep a reference. + */ +void gst_inference_prediction_append (GstInferencePrediction * self, GstInferencePrediction * child); + +/** + * gst_inference_prediction_get_children: + * @self: the parent prediction + * + * Gets a list of the immediate children of the current prediction. In + * other words, the children of these childrens are not returned. The + * references of these children are still owned by the parent. + * + * Returns: A linked list of the child predictions. + */ +GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); + +/** + * gst_inference_prediction_append_classification: + * @self: the parent prediction + * @c: the classification to append to the prediction + * + * A new GstInferenceClassification to associate with this + * prediction. The prediction takes ownership of the classification + */ +void gst_inference_prediction_append_classification (GstInferencePrediction * self, + GstInferenceClassification * c); + +/** + * gst_inference_prediction_scale: + * @self: the prediction to scale + * @to: the resulting image size + * @from: the original image size + * + * Modifies the BoundingBox associated with this prediction (and all + * its children) to scale to the new image size. This is typically + * used by the GstMeta subsystem automatically and not for public + * usage. + * + * Returns: a newly allocated and scaled prediction. + */ +GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); + +/** + * gst_inference_prediction_scale_ip: + * @self: the prediction to scale in place + * @to: the resulting image size + * @from: the original image size + * + * Modifies the BoundingBox associated with this prediction (and all + * its children) to scale to the new image size. This is typically + * used by the GstMeta subsystem automatically and not for public + * usage. + */ +void gst_inference_prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); + +/** + * gst_inference_prediction_find: + * @self: the root prediction + * @id: the prediction_id of the prediction to return + * + * Traverses the prediction tree looking for a child with the given + * id. + * + * Returns: a reference to the prediction with id or NULL if not + * found. Unref after usage. + */ +GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, + guint64 id); + +/** + * gst_inference_prediction_get_enabled: + * @self: the root prediction + * + * Traverse the prediction three saving the predictions that are enabled. + * + * Returns: a GList of predictions that are enabled. + */ +GList * gst_inference_prediction_get_enabled (GstInferencePrediction * self); + +/** + * gst_inference_prediction_merge: + * @src: the source prediction + * @dst: the destination prediction + * + * Copies the extra information from src to dst. + * + * Returns: TRUE if new sub-predictions were added, FALSE otherwise. + */ +gboolean gst_inference_prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); + +/** + * GST_INFERENCE_PREDICTION_LOCK: + * @p: The GstInferencePrediction to lock + * + * Locks the prediction to avoid concurrent access from different + * threads. + */ +#define GST_INFERENCE_PREDICTION_LOCK(p) g_mutex_lock (&((p)->mutex)) + +/** + * GST_INFERENCE_PREDICTION_UNLOCK: + * @p: The GstInferencePrediction to unlock + * + * Unlocks the prediction to yield the access to other threads. + */ +#define GST_INFERENCE_PREDICTION_UNLOCK(p) g_mutex_unlock (&((p)->mutex)) + +G_END_DECLS + +#endif // __GST_INFERENCE_PREDICTION__ diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.c b/gst-libs/gst/r2inference/gstinferencepreprocess.c index 2f78bf08..5e52feb4 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,10 +18,11 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencepreprocess.h" #include -static gboolean gst_check_format_RGB (GstVideoFrame * inframe, +static gboolean gst_configure_format_values (GstVideoFrame * inframe, gint * first_index, gint * last_index, gint * offset, gint * channels); static void gst_apply_means_std (GstVideoFrame * inframe, GstVideoFrame * outframe, gint first_index, gint last_index, @@ -64,7 +65,7 @@ gst_apply_means_std (GstVideoFrame * inframe, GstVideoFrame * outframe, } static gboolean -gst_check_format_RGB (GstVideoFrame * inframe, gint * first_index, +gst_configure_format_values (GstVideoFrame * inframe, gint * first_index, gint * last_index, gint * offset, gint * channels) { g_return_val_if_fail (inframe != NULL, FALSE); @@ -101,6 +102,11 @@ gst_check_format_RGB (GstVideoFrame * inframe, gint * first_index, *last_index = 0; *offset = 1; break; + case GST_VIDEO_FORMAT_GRAY8: + *first_index = 0; + *last_index = 0; + *offset = 0; + break; default: return FALSE; break; @@ -115,7 +121,7 @@ gst_normalize (GstVideoFrame * inframe, GstVideoFrame * outframe, gdouble mean, gint first_index = 0, last_index = 0, offset = 0, channels = 0; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } @@ -192,7 +198,7 @@ gst_subtract_mean (GstVideoFrame * inframe, GstVideoFrame * outframe, const gdouble std = 1; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } @@ -210,7 +216,7 @@ gst_pixel_to_float (GstVideoFrame * inframe, GstVideoFrame * outframe, const gdouble std = 1, mean = 0; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.h b/gst-libs/gst/r2inference/gstinferencepreprocess.h index 6337e611..a186f5f8 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc index ec797872..cc2eff89 100644 --- a/gst-libs/gst/r2inference/gstncsdk.cc +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstncsdk.h b/gst-libs/gst/r2inference/gstncsdk.h index 08689b68..17b0ff02 100644 --- a/gst-libs/gst/r2inference/gstncsdk.h +++ b/gst-libs/gst/r2inference/gstncsdk.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gsttensorflow.cc b/gst-libs/gst/r2inference/gsttensorflow.cc index 889cedf7..f00eeeb6 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.cc +++ b/gst-libs/gst/r2inference/gsttensorflow.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gsttensorflow.h b/gst-libs/gst/r2inference/gsttensorflow.h index 793eb6dc..a8594457 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.h +++ b/gst-libs/gst/r2inference/gsttensorflow.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gsttflite.cc b/gst-libs/gst/r2inference/gsttflite.cc new file mode 100644 index 00000000..d2286ac0 --- /dev/null +++ b/gst-libs/gst/r2inference/gsttflite.cc @@ -0,0 +1,54 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gsttflite.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tflite_debug_category); +#define GST_CAT_DEFAULT gst_tflite_debug_category + +struct _GstTflite +{ + GstBackend parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstTflite, gst_tflite, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_tflite_debug_category, "tflite", 0, + "debug category for tflite parameters")); + +static void +gst_tflite_class_init (GstTfliteClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + gst_backend_install_properties (bclass, r2i::FrameworkCode::TFLITE); +} + +static void +gst_tflite_init (GstTflite * self) +{ + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::TFLITE); +} diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.h b/gst-libs/gst/r2inference/gsttflite.h similarity index 55% rename from gst-libs/gst/opencv/gstinferenceoverlay.h rename to gst-libs/gst/r2inference/gsttflite.h index b586bc6c..0260114f 100644 --- a/gst-libs/gst/opencv/gstinferenceoverlay.h +++ b/gst-libs/gst/r2inference/gsttflite.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -19,29 +19,17 @@ * */ -#ifndef __GST_INFERENCE_OVERLAY_H__ -#define __GST_INFERENCE_OVERLAY_H__ +#ifndef __GST_TFLITE_H__ +#define __GST_TFLITE_H__ #include -#include +#include G_BEGIN_DECLS -#define GST_TYPE_INFERENCE_OVERLAY gst_inference_overlay_get_type () -G_DECLARE_DERIVABLE_TYPE (GstInferenceOverlay, gst_inference_overlay, GST, - INFERENCE_OVERLAY, GstVideoFilter); - -struct _GstInferenceOverlayClass -{ - GstVideoFilterClass parent_class; - - GstFlowReturn (* process_meta) (GstInferenceOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels); - - GType meta_type; -}; +#define GST_TYPE_TFLITE gst_tflite_get_type () +G_DECLARE_FINAL_TYPE(GstTflite, gst_tflite, GST, TFLITE, GstBackend); G_END_DECLS -#endif //__GST_INFERENCE_OVERLAY_H__ +#endif //__GST_TFLITE_H__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 01ea6863..5b21d2da 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -21,6 +21,7 @@ #include "gstvideoinference.h" #include "gstinferencebackends.h" +#include "gstinferencemeta.h" #include "gstbackend.h" #include @@ -44,10 +45,12 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_MODEL_LOCATION NULL - +#define DEFAULT_LABELS NULL +#define DEFAULT_NUM_LABELS 0 enum { NEW_PREDICTION_SIGNAL, + NEW_INFERENCE_SIGNAL, LAST_SIGNAL }; @@ -55,9 +58,14 @@ enum { PROP_0, PROP_BACKEND, - PROP_MODEL_LOCATION + PROP_MODEL_LOCATION, + PROP_LABELS, }; +GQuark _size_quark; +GQuark _orientation_quark; +GQuark _scale_quark; +GQuark _copy_quark; typedef struct _GstVideoInferencePad GstVideoInferencePad; struct _GstVideoInferencePad @@ -73,6 +81,7 @@ struct _GstVideoInferencePrivate GstCollectPads *cpads; GstVideoInferencePad *sink_bypass_data; GstVideoInferencePad *sink_model_data; + const GstMetaInfo *inference_meta_info; GstPad *sink_bypass; GstPad *src_bypass; @@ -82,6 +91,16 @@ struct _GstVideoInferencePrivate GstBackend *backend; gchar *model_location; + + GMutex mtx_model_queue; + GMutex mtx_bypass_queue; + + GQueue *model_queue; + GQueue *bypass_queue; + + gchar *labels; + gchar **labels_list; + gint num_labels; }; /* GObject methods */ @@ -113,13 +132,15 @@ static gboolean gst_video_inference_start (GstVideoInference * self); static gboolean gst_video_inference_stop (GstVideoInference * self); static GstPad *gst_video_inference_create_pad (GstVideoInference * self, GstPadTemplate * templ, const gchar * name, GstVideoInferencePad ** data); -static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, - gpointer user_data); -static GstFlowReturn gst_video_inference_pop_buffer (GstVideoInference * self, - GstCollectPads * cpads, GstCollectData * data, GstBuffer ** buffer); +static GstFlowReturn gst_video_inference_process_bypass (GstVideoInference * + self, GstBuffer * buffer, GstVideoInferencePad * pad); +static GstFlowReturn gst_video_inference_process_model (GstVideoInference * + self, GstBuffer * buffer, GstVideoInferencePad * pad); +static GstFlowReturn gst_video_inference_buffer_function (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buffer, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, GstBuffer * buffer, GstPad * pad); -static gboolean gst_video_inference_model_buffer_process (GstVideoInference * +static gboolean gst_video_inference_model_run_prediction (GstVideoInference * self, GstVideoInferenceClass * klass, GstVideoInferencePrivate * priv, GstBuffer * buffer, gpointer * prediction_data, gsize * prediction_size); @@ -130,12 +151,6 @@ static gboolean gst_video_inference_predict (GstVideoInference * self, GstVideoInferencePrivate * priv, GstVideoFrame * frame, gpointer * pred, gsize * pred_size); -static gboolean gst_video_inference_postprocess (GstVideoInference * self, - GstVideoInferenceClass * klass, const gpointer prediction_data, - gsize prediction_size, GstBuffer * buffer_model, - GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, - GstVideoInferencePad * pad_bypass); - static GstIterator *gst_video_inference_iterate_internal_links (GstPad * pad, GstObject * parent); static gboolean gst_video_inference_sink_event (GstCollectPads * pads, @@ -156,14 +171,11 @@ static void video_inference_map_buffers (GstVideoInferencePad * data, GstBuffer * inbuf, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, GstBuffer * buffer, GstVideoInfo * video_info, - GstVideoFrame * out_frame, GstMeta ** out_meta); -static void video_inference_buffer_unref (GstBuffer * buffer); -static void video_inference_frame_unmap (GstBuffer * buffer, - GstVideoFrame * frame); -static void video_inference_remove_meta (GstBuffer * buffer, GstMeta * meta); + GstMeta ** out_meta); static GstMeta *video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, GstVideoInfo * info_bypass); +static void video_inference_flush_queue (GQueue * queue, GMutex * mutex); static guint gst_video_inference_signals[LAST_SIGNAL] = { 0 }; @@ -216,16 +228,31 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) g_param_spec_string ("model-location", "Model Location", "Path to the model to use", DEFAULT_MODEL_LOCATION, G_PARAM_READWRITE)); + g_object_class_install_property (oclass, PROP_LABELS, + g_param_spec_string ("labels", "labels", + "Semicolon separated string containing inference labels", + DEFAULT_LABELS, G_PARAM_READWRITE)); gst_video_inference_signals[NEW_PREDICTION_SIGNAL] = g_signal_new ("new-prediction", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER); + gst_video_inference_signals[NEW_INFERENCE_SIGNAL] = + g_signal_new ("new-inference", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, + G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER); + klass->start = NULL; klass->stop = NULL; klass->preprocess = NULL; klass->postprocess = NULL; + + _size_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_SIZE_STR); + _orientation_quark = + g_quark_from_static_string (GST_META_TAG_VIDEO_ORIENTATION_STR); + _scale_quark = gst_video_meta_transform_scale_get_quark (); + _copy_quark = g_quark_from_static_string ("gst-copy"); } static void @@ -233,6 +260,10 @@ gst_video_inference_init (GstVideoInference * self) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + priv->labels = DEFAULT_LABELS; + priv->labels_list = DEFAULT_LABELS; + priv->num_labels = DEFAULT_NUM_LABELS; + priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; @@ -240,10 +271,20 @@ gst_video_inference_init (GstVideoInference * self) priv->src_bypass = NULL; priv->sink_model = NULL; priv->src_model = NULL; + priv->inference_meta_info = gst_inference_meta_get_info (); priv->cpads = gst_collect_pads_new (); - gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, - (gpointer) (self)); + + g_mutex_init (&priv->mtx_model_queue); + g_mutex_init (&priv->mtx_bypass_queue); + + priv->model_queue = g_queue_new (); + priv->bypass_queue = g_queue_new (); + + /* Use buffer function to handle each pad buffer independently */ + gst_collect_pads_set_buffer_function (priv->cpads, + gst_video_inference_buffer_function, (gpointer) self); + gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, (gpointer) (self)); @@ -282,6 +323,18 @@ gst_video_inference_set_property (GObject * object, } GST_OBJECT_UNLOCK (self); break; + case PROP_LABELS: + if (priv->labels != NULL) { + g_free (priv->labels); + } + if (priv->labels_list != NULL) { + g_strfreev (priv->labels_list); + } + priv->labels = g_value_dup_string (value); + priv->labels_list = g_strsplit (g_value_get_string (value), ";", 0); + priv->num_labels = g_strv_length (priv->labels_list); + GST_DEBUG_OBJECT (self, "Changed inference labels %s", priv->labels); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -304,6 +357,9 @@ gst_video_inference_get_property (GObject * object, case PROP_MODEL_LOCATION: g_value_set_string (value, priv->model_location); break; + case PROP_LABELS: + g_value_set_string (value, priv->labels); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -399,6 +455,9 @@ gst_video_inference_stop (GstVideoInference * self) GST_INFO_OBJECT (self, "Stopping video inference"); + video_inference_flush_queue (priv->model_queue, &priv->mtx_model_queue); + video_inference_flush_queue (priv->bypass_queue, &priv->mtx_bypass_queue); + if (!gst_backend_stop (priv->backend, &err)) { GST_ELEMENT_ERROR (self, LIBRARY, INIT, ("Could not stop the selected backend: (%s)", err->message), (NULL)); @@ -477,7 +536,7 @@ gst_video_inference_create_pad (GstVideoInference * self, *data = (GstVideoInferencePad *) gst_collect_pads_add_pad (priv->cpads, pad, - sizeof (GstVideoInferencePad), NULL, TRUE); + sizeof (GstVideoInferencePad), NULL, FALSE); if (NULL == *data) { GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); goto free_pad; @@ -586,6 +645,7 @@ gst_video_inference_forward_buffer (GstVideoInference * self, { GstFlowReturn ret = GST_FLOW_OK; GstDebugLevel level = GST_LEVEL_LOG; + GstInferenceMeta *imeta = NULL; g_return_val_if_fail (self, GST_FLOW_ERROR); @@ -601,6 +661,27 @@ gst_video_inference_forward_buffer (GstVideoInference * self, return ret; } + /* Get Stream ID */ + imeta = + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + gst_inference_meta_api_get_type ()); + if (imeta) { + if (imeta->stream_id) { + /* Check current stream ID in pad */ + gchar *stream_id = gst_pad_get_stream_id (pad); + if (0 != g_strcmp0 (stream_id, imeta->stream_id)) { + /* Different Stream IDs, forward new stream-start event */ + GstEvent *stream_start = gst_event_new_stream_start (imeta->stream_id); + GST_LOG_OBJECT (self, + "stream-start mismatch, resetting: Current Stream ID: %s, Buffer Stream ID: %s", + stream_id, imeta->stream_id); + if (!gst_pad_push_event (pad, stream_start)) + GST_WARNING_OBJECT (self, "Failed to push stream start event"); + } + g_free (stream_id); + } + } + GST_LOG_OBJECT (self, "Forwarding buffer %" GST_PTR_FORMAT " to %" GST_PTR_FORMAT, buffer, pad); ret = gst_pad_push (pad, buffer); @@ -704,7 +785,7 @@ gst_video_inference_predict (GstVideoInference * self, } static gboolean -gst_video_inference_model_buffer_process (GstVideoInference * self, +gst_video_inference_model_run_prediction (GstVideoInference * self, GstVideoInferenceClass * klass, GstVideoInferencePrivate * priv, GstBuffer * buffer, gpointer * prediction_data, gsize * prediction_size) { @@ -746,47 +827,29 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, - GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame, - GstMeta ** out_meta) + GstBuffer * buffer, GstVideoInfo * video_info, GstMeta ** out_meta) { - GstMapFlags flags; + GstInferenceMeta *imeta = NULL; g_return_val_if_fail (meta_info, FALSE); - g_return_val_if_fail (out_frame, FALSE); - - /* No pad requested, continue without meta */ - if (NULL == buffer || NULL == video_info) { - return TRUE; - } + g_return_val_if_fail (buffer, FALSE); + g_return_val_if_fail (video_info, FALSE); + g_return_val_if_fail (out_meta, FALSE); - if (out_meta) { - g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); - *out_meta = gst_buffer_add_meta (buffer, meta_info, NULL); - } + g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); + out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); - flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); - gst_video_frame_map (out_frame, video_info, buffer, flags); - - return TRUE; -} - -static void -video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame) -{ - g_return_if_fail (frame); + /* Create new meta only if the buffer didn't have one */ + if (!out_meta[1]) { + out_meta[1] = + gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); - if (NULL != buffer) { - gst_video_frame_unmap (frame); + imeta = (GstInferenceMeta *) out_meta[1]; + imeta->prediction->bbox.width = video_info->width; + imeta->prediction->bbox.height = video_info->height; } -} -static void -video_inference_remove_meta (GstBuffer * buffer, GstMeta * meta) -{ - if (NULL != buffer && NULL != meta) { - g_return_if_fail (gst_buffer_is_writable (buffer)); - gst_buffer_remove_meta (buffer, meta); - } + return TRUE; } static GstMeta * @@ -796,11 +859,7 @@ video_inference_transform_meta (GstBuffer * buffer_model, { GstMeta *meta_bypass = NULL; const GstMetaInfo *info; - GQuark size_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_SIZE_STR); - GQuark orientation_quark = - g_quark_from_static_string (GST_META_TAG_VIDEO_ORIENTATION_STR); - GQuark scale_quark = gst_video_meta_transform_scale_get_quark (); - GQuark copy_quark = g_quark_from_static_string ("gst-copy"); + GstVideoMetaTransform trans = { info_model, info_bypass }; g_return_val_if_fail (buffer_model, NULL); g_return_val_if_fail (info_model, NULL); @@ -812,193 +871,320 @@ video_inference_transform_meta (GstBuffer * buffer_model, info = meta_model->info; - if (gst_meta_api_type_has_tag (info->api, size_quark) || - gst_meta_api_type_has_tag (info->api, orientation_quark)) { - GstVideoMetaTransform trans = { info_model, info_bypass }; - - info->transform_func (buffer_bypass, meta_model, buffer_model, - scale_quark, &trans); - } else { - info->transform_func (buffer_bypass, meta_model, buffer_model, - copy_quark, NULL); - } + /* TODO: elements such as videoscale will drop metas that have tags + such as video-size. Until this is fixed, tread all + transformations as scaling transformations + */ + info->transform_func (buffer_bypass, meta_model, buffer_model, + _scale_quark, &trans); meta_bypass = gst_buffer_get_meta (buffer_bypass, info->api); return meta_bypass; } -static gboolean -gst_video_inference_postprocess (GstVideoInference * self, - GstVideoInferenceClass * klass, const gpointer prediction_data, - gsize prediction_size, GstBuffer * buffer_model, - GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, - GstVideoInferencePad * pad_bypass) +static GstFlowReturn +gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, + GstVideoInferencePad * pad) { - GstMeta *meta_model = NULL; - GstMeta *meta_bypass = NULL; - GstVideoFrame frame_model; - GstVideoFrame frame_bypass; + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstFlowReturn ret = GST_FLOW_OK; + GstMeta *current_meta = NULL; + GstMeta *meta_model[2] = { NULL }; GstVideoInfo *info_model = NULL; - GstVideoInfo *info_bypass = NULL; + GstBuffer *buffer_model = NULL; + gpointer prediction_data = NULL; + gsize prediction_size; gboolean pred_valid = FALSE; - g_return_val_if_fail (self, FALSE); - g_return_val_if_fail (klass, FALSE); - g_return_val_if_fail (prediction_data, FALSE); - g_return_val_if_fail (prediction_size, FALSE); - g_return_val_if_fail (buffer_model, FALSE); - g_return_val_if_fail (pad_model, FALSE); + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (pad != NULL, GST_FLOW_ERROR); + + GST_LOG_OBJECT (self, "Processing model buffer"); - /* Subclass didn't implement a post-process, dont fail, just ignore */ if (NULL == klass->postprocess) { - return TRUE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass didn't implement post-process"), (NULL)); + ret = GST_FLOW_ERROR; + goto buffer_free; } - info_model = &(pad_model->info); - info_bypass = pad_bypass ? &(pad_bypass->info) : NULL; + buffer_model = gst_buffer_make_writable (buffer); + current_meta = + gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); + if (current_meta) { + /* Check if root is enabled to be processed, if not, just forward buffer */ + GstInferenceMeta *inference_meta = (GstInferenceMeta *) current_meta; + GstInferencePrediction *root = inference_meta->prediction; + if (!root->enabled) { + GST_INFO_OBJECT (self, + "Current Prediction is not enabled, bypassing processing..."); + goto forward_buffer; + } + } - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model, &meta_model)) { - return FALSE; + /* Run preprocess and inference on the model and generate prediction */ + if (!gst_video_inference_model_run_prediction (self, klass, priv, + buffer_model, &prediction_data, &prediction_size)) { + ret = GST_FLOW_ERROR; + goto buffer_free; } + /* Assign already created inferencemeta, no need to create a new one */ + meta_model[1] = current_meta; + + /* Prepare postprocess */ + info_model = &(pad->info); if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass, NULL)) { - return FALSE; + buffer_model, info_model, meta_model)) { + ret = GST_FLOW_ERROR; + goto prediction_free; } - GST_LOG_OBJECT (self, "Calling frame postprocess"); - if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, - info_model, &pred_valid)) { + /* Subclass Processing */ + if (!klass->postprocess (self, prediction_data, prediction_size, + meta_model, info_model, &pred_valid, priv->labels_list, + priv->num_labels)) { GST_ELEMENT_ERROR (self, STREAM, FAILED, ("Subclass failed at preprocess"), (NULL)); - return FALSE; + ret = GST_FLOW_ERROR; + goto prediction_free; } - if (pred_valid) { - GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - - meta_bypass = - video_inference_transform_meta (buffer_model, info_model, meta_model, - buffer_bypass, info_bypass); - g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, - meta_model, &frame_bypass, meta_bypass, pbpass); + /* Check if bypass pad was requested, if not, forward buffer */ + if (NULL == priv->sink_bypass) { + GST_LOG_OBJECT (self, + "There is no sinkpad for bypass, forwarding model buffer..."); + goto forward_buffer; } else { - video_inference_remove_meta (buffer_model, meta_model); - video_inference_remove_meta (buffer_bypass, meta_bypass); + GstInferenceMeta *imeta = (GstInferenceMeta *) meta_model[1]; + /* Queue buffer */ + GST_LOG_OBJECT (self, "Queue model buffer"); + g_mutex_lock (&priv->mtx_model_queue); + g_queue_push_head (priv->model_queue, (gpointer) buffer_model); + g_mutex_unlock (&priv->mtx_model_queue); + /* Keep current Stream ID */ + if (imeta) { + g_free (imeta->stream_id); + imeta->stream_id = gst_pad_get_stream_id (pad->data.pad); + } + goto out; } - video_inference_frame_unmap (buffer_model, &frame_model); - video_inference_frame_unmap (buffer_bypass, &frame_bypass); +forward_buffer: + ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), + priv->src_model); - return TRUE; +prediction_free: + g_free (prediction_data); + +buffer_free: + gst_buffer_unref (buffer_model); + +out: + return ret; } -static GstFlowReturn -gst_video_inference_pop_buffer (GstVideoInference * self, - GstCollectPads * cpads, GstCollectData * data, GstBuffer ** buffer) +static void +video_inference_notify (GstVideoInference * self, GstBuffer * model_buffer, + GstMeta * meta_model[2], GstBuffer * bypass_buffer, + GstMeta * meta_bypass[2]) { - g_return_val_if_fail (self, GST_FLOW_ERROR); - g_return_val_if_fail (buffer, GST_FLOW_ERROR); - - *buffer = NULL; + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoFrame frame_model; + GstVideoFrame frame_bypass; + GstVideoInfo *info_model = NULL; + GstVideoInfo *info_bypass = NULL; + GstMapFlags flags; - if (NULL == data) { - return GST_FLOW_OK; - } + g_return_if_fail (model_buffer); + g_return_if_fail (meta_model); - *buffer = gst_collect_pads_pop (cpads, data); - if (NULL == *buffer) { - GST_INFO_OBJECT (self, "EOS requested on %" GST_PTR_FORMAT, data->pad); - return GST_FLOW_EOS; - } + info_model = &(priv->sink_model_data->info); + info_bypass = &(priv->sink_bypass_data->info); - *buffer = gst_buffer_make_writable (*buffer); + flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); + gst_video_frame_map (&frame_model, info_model, model_buffer, flags); + gst_video_frame_map (&frame_bypass, info_bypass, bypass_buffer, flags); - GST_LOG_OBJECT (self, "Popped %" GST_PTR_FORMAT " from %" GST_PTR_FORMAT, - *buffer, data->pad); - return GST_FLOW_OK; -} + /* Emit inference signal */ + g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, + meta_model[0], &frame_model, meta_bypass[0], &frame_bypass); + g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, + meta_model[1], &frame_model, meta_bypass[1], &frame_bypass); -static void -video_inference_buffer_unref (GstBuffer * buffer) -{ - if (NULL != buffer) { - gst_buffer_unref (buffer); - } + gst_video_frame_unmap (&frame_model); + gst_video_frame_unmap (&frame_bypass); } static GstFlowReturn -gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) +gst_video_inference_process_bypass (GstVideoInference * self, + GstBuffer * buffer, GstVideoInferencePad * pad) { - GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstFlowReturn ret = GST_FLOW_OK; - GstBuffer *buffer_model = NULL; - GstBuffer *buffer_bypass = NULL; - gpointer prediction_data = NULL; - gsize prediction_size; - - ret = - gst_video_inference_pop_buffer (self, pads, - (GstCollectData *) priv->sink_model_data, &buffer_model); - if (GST_FLOW_OK != ret) { - goto out; + GstMeta *current_meta = NULL; + GstBuffer *bypass_buffer = NULL; + GstBuffer *model_buffer = NULL; + gboolean model_empty = FALSE; + + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (pad != NULL, GST_FLOW_ERROR); + + GST_LOG_OBJECT (self, "Processing bypass buffer"); + + /* Check if model pad was requested, if not, forward buffer */ + if (NULL == priv->sink_model) { + GST_LOG_OBJECT (self, + "There is no sinkpad for model, forwarding bypass buffer..."); + bypass_buffer = buffer; + goto forward_buffer; } - ret = - gst_video_inference_pop_buffer (self, pads, - (GstCollectData *) priv->sink_bypass_data, &buffer_bypass); - if (GST_FLOW_OK != ret) { - goto model_free; + bypass_buffer = gst_buffer_make_writable (buffer); + current_meta = gst_buffer_get_meta (bypass_buffer, + gst_inference_meta_api_get_type ()); + if (current_meta) { + GstInferenceMeta *imeta = (GstInferenceMeta *) current_meta; + GList *found = gst_inference_prediction_get_enabled (imeta->prediction); + + if (!found) { + GST_INFO_OBJECT (self, + "There is no predictions enabled, bypassing processing..."); + goto forward_buffer; + } + + g_list_free (found); } - if (buffer_model) { - /* Run preprocess and inference on the model and generate prediction */ - if (!gst_video_inference_model_buffer_process (self, klass, priv, - buffer_model, &prediction_data, &prediction_size)) { - ret = GST_FLOW_ERROR; - goto bypass_free; - } + /* Queue this new buffer at the head and dequeue the older one */ + GST_LOG_OBJECT (self, "Queue bypass buffer and get older one"); + g_mutex_lock (&priv->mtx_bypass_queue); + g_queue_push_head (priv->bypass_queue, (gpointer) bypass_buffer); + bypass_buffer = GST_BUFFER_CAST (g_queue_pop_tail (priv->bypass_queue)); + g_mutex_unlock (&priv->mtx_bypass_queue); + + while (!model_empty) { + /* Dequeue oldest model buffer from tail */ + g_mutex_lock (&priv->mtx_model_queue); + GST_LOG_OBJECT (self, "Dequeue model buffer"); + model_buffer = GST_BUFFER_CAST (g_queue_pop_tail (priv->model_queue)); + g_mutex_unlock (&priv->mtx_model_queue); + + if (NULL == model_buffer) { + /* Model queue is empty */ + model_empty = TRUE; + } else { + GstInferencePrediction *root_model = NULL; + GstInferencePrediction *root_bypass = NULL; + GstMeta *meta_model[2] = { NULL }; + GstMeta *meta_bypass[2] = { NULL }; + GstVideoInfo *info_model = &(priv->sink_model_data->info); + GstVideoInfo *info_bypass = &(priv->sink_bypass_data->info); + + /* Get both metas from model buffer */ + meta_model[0] = gst_buffer_get_meta (model_buffer, + klass->inference_meta_info->api); + meta_model[1] = gst_buffer_get_meta (model_buffer, + gst_inference_meta_api_get_type ()); + + root_model = ((GstInferenceMeta *) meta_model[1])->prediction; + + /* If bypass doesn't have meta, just transfer the model meta */ + current_meta = gst_buffer_get_meta (bypass_buffer, + gst_inference_meta_api_get_type ()); + + if (current_meta) { + /* Check if model and bypass IDs match */ + GST_LOG_OBJECT (self, "Checking if model and bypass IDs match"); + root_bypass = gst_inference_prediction_find (((GstInferenceMeta *) + current_meta)->prediction, root_model->prediction_id); + if (NULL == root_bypass) { + /* Queue model buffer to tail again */ + g_mutex_lock (&priv->mtx_model_queue); + GST_LOG_OBJECT (self, "Queue model buffer"); + g_queue_push_tail (priv->model_queue, (gpointer) model_buffer); + g_mutex_unlock (&priv->mtx_model_queue); + goto forward_buffer; + } else { + gst_inference_prediction_unref (root_bypass); + } + } - /* Have the subclass analyze the prediction and generate model and bypass metas */ - if (!gst_video_inference_postprocess (self, klass, prediction_data, - prediction_size, buffer_model, priv->sink_model_data, buffer_bypass, - priv->sink_bypass_data)) { - ret = GST_FLOW_ERROR; - goto bypass_free; + /* Transfer meta from model to bypass */ + GST_LOG_OBJECT (self, "Transfering meta from model to bypass"); + + meta_bypass[0] = + video_inference_transform_meta (model_buffer, info_model, + (GstMeta *) meta_model[0], bypass_buffer, info_bypass); + meta_bypass[1] = + video_inference_transform_meta (model_buffer, info_model, + (GstMeta *) meta_model[1], bypass_buffer, info_bypass); + + /* Notify prediction */ + video_inference_notify (self, model_buffer, meta_model, bypass_buffer, + meta_bypass); + + /* Forward buffer to model src pad */ + GST_LOG_OBJECT (self, "Forward model buffer"); + ret = gst_video_inference_forward_buffer (self, model_buffer, + priv->src_model); + model_buffer = NULL; } } - /* Forward buffer to model src pad */ - ret = gst_video_inference_forward_buffer (self, buffer_model, - priv->src_model); + /* Queue old bypass buffer at the tail */ + GST_LOG_OBJECT (self, "Queue bypass buffer"); + g_mutex_lock (&priv->mtx_bypass_queue); + g_queue_push_tail (priv->bypass_queue, (gpointer) bypass_buffer); + g_mutex_unlock (&priv->mtx_bypass_queue); - /* We don't own this buffer anymore, don't free it */ - buffer_model = NULL; - if (GST_FLOW_OK != ret) { - goto bypass_free; - } + return ret; +forward_buffer: /* Forward buffer to bypass src pad */ - ret = gst_video_inference_forward_buffer (self, - buffer_bypass, priv->src_bypass); + GST_LOG_OBJECT (self, "Forward bypass buffer"); + ret = + gst_video_inference_forward_buffer (self, bypass_buffer, + priv->src_bypass); - /* We don't own this buffer anymore, don't free it */ - buffer_bypass = NULL; + return ret; +} - goto out; +static GstFlowReturn +gst_video_inference_buffer_function (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buffer, gpointer user_data) +{ + GstVideoInference *self = (GstVideoInference *) user_data; + GstVideoInferencePrivate *priv = NULL; + GstVideoInferencePad *pad = (GstVideoInferencePad *) data; + GstFlowReturn ret = GST_FLOW_OK; -bypass_free: - video_inference_buffer_unref (buffer_bypass); + if (!buffer) { + /* EOS reached the pad */ + ret = GST_FLOW_EOS; + goto out; + } -model_free: - video_inference_buffer_unref (buffer_model); + g_return_val_if_fail (pads != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); -out: - g_free (prediction_data); + priv = GST_VIDEO_INFERENCE_PRIVATE (self); + if (data->pad == priv->sink_model) { + GST_LOG_OBJECT (self, "Model buffer arrived, processing it..."); + ret = gst_video_inference_process_model (self, buffer, pad); + goto out; + } else { + GST_LOG_OBJECT (self, "Bypass buffer arrived, processing it..."); + ret = gst_video_inference_process_bypass (self, buffer, pad); + goto out; + } + +out: return ret; } @@ -1152,6 +1338,21 @@ gst_video_inference_src_event (GstPad * pad, GstObject * parent, return gst_collect_pads_src_event_default (priv->cpads, pad, event); } +static void +video_inference_flush_queue (GQueue * queue, GMutex * mutex) +{ + GstBuffer *buf = NULL; + + g_return_if_fail (queue); + g_return_if_fail (mutex); + + g_mutex_lock (mutex); + while ((buf = GST_BUFFER_CAST (g_queue_pop_tail (queue)))) { + gst_buffer_unref (buf); + } + g_mutex_unlock (mutex); +} + static void gst_video_inference_finalize (GObject * object) { @@ -1168,6 +1369,16 @@ gst_video_inference_finalize (GObject * object) priv->sink_model_data = NULL; g_free (priv->model_location); priv->model_location = NULL; + g_free (priv->labels); + priv->labels = NULL; + g_free (priv->labels_list); + priv->labels_list = NULL; + + g_mutex_clear (&priv->mtx_model_queue); + g_mutex_clear (&priv->mtx_bypass_queue); + + g_queue_free (priv->model_queue); + g_queue_free (priv->bypass_queue); g_clear_object (&priv->backend); diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 444ec824..1dc610d8 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -26,25 +26,25 @@ #include G_BEGIN_DECLS - #define GST_TYPE_VIDEO_INFERENCE gst_video_inference_get_type () -G_DECLARE_DERIVABLE_TYPE(GstVideoInference, gst_video_inference, GST, - VIDEO_INFERENCE, GstElement); +G_DECLARE_DERIVABLE_TYPE (GstVideoInference, gst_video_inference, GST, + VIDEO_INFERENCE, GstElement); struct _GstVideoInferenceClass { GstElementClass parent_class; - gboolean (* start) (GstVideoInference *self); - gboolean (* stop) (GstVideoInference *self); - gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, - GstVideoFrame *outframe); - gboolean (* postprocess) (GstVideoInference *self, const gpointer prediction, - gsize size, GstMeta *meta_model, GstVideoInfo *info_model, gboolean *valid_prediction); + gboolean (*start) (GstVideoInference * self); + gboolean (*stop) (GstVideoInference * self); + gboolean (*preprocess) (GstVideoInference * self, GstVideoFrame * inframe, + GstVideoFrame * outframe); + gboolean (*postprocess) (GstVideoInference * self, + const gpointer prediction, gsize size, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction, gchar **labels_list, gint num_labels); const GstMetaInfo *inference_meta_info; + }; G_END_DECLS - #endif //__GST_VIDEO_INFERENCE_H__ diff --git a/gst/Makefile.am b/gst/Makefile.am new file mode 100644 index 00000000..1c7fd945 --- /dev/null +++ b/gst/Makefile.am @@ -0,0 +1,4 @@ +SUBDIRS = \ + inferencecrop \ + inferencefilter \ + inferencedebug diff --git a/gst/inferencecrop/Makefile.am b/gst/inferencecrop/Makefile.am new file mode 100644 index 00000000..0b96c0a0 --- /dev/null +++ b/gst/inferencecrop/Makefile.am @@ -0,0 +1,36 @@ +plugin_LTLIBRARIES = libgstinferencecrop.la + +libgstinferencecrop_la_SOURCES = \ + gstdetectioncrop.cc \ + cropelement.cc \ + videocrop.cc + +libgstinferencecrop_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + +libgstinferencecrop_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + -std=c++11 + +libgstinferencecrop_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencecrop_la_LDFLAGS = \ + $(GST_PLUGIN_LDFLAGS) + +libgstinferencecrop_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstdetectioncrop.h \ + cropelement.h \ + videocrop.h diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc new file mode 100644 index 00000000..1c7e1ce1 --- /dev/null +++ b/gst/inferencecrop/cropelement.cc @@ -0,0 +1,91 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "cropelement.h" + +CropElement::CropElement () { + this->element = nullptr; + this->top = 0; + this->bottom = 0; + this->right = 0; + this->left = 0; +} + +bool +CropElement::Validate () { + GstElement *element; + + this->mutex.lock (); + if (nullptr == this->element) { + const std::string factory = this->GetFactory (); + this->element = gst_element_factory_make (factory.c_str (), nullptr); + } + element = this->element; + this->mutex.unlock (); + + return element != nullptr; +} + +GstElement * +CropElement::GetElement () { + GstElement *element; + element = nullptr; + bool validated = Validate(); + this->mutex.lock (); + if (validated) { + element = this->element; + }else{ + const std::string factory = this->GetFactory (); + GST_ERROR_OBJECT (this->element, "Unable to initialize the element %s", factory.c_str ()); + } + + this->mutex.unlock (); + + return element; +} + +void +CropElement::Reset () { + this->SetCroppingSize(0, 0, 0, 0); +} + +void +CropElement::SetCroppingSize (gint top, gint bottom, gint right, gint left) { + this->mutex.lock (); + this->top = top; + this->bottom = bottom; + this->right = right; + this->left = left; + + this->UpdateElement (this->element, + this->top, + this->bottom, + this->right, + this->left); + this->mutex.unlock (); +} + +CropElement::~CropElement () { + if (nullptr != this->element) { + gst_object_unref (this->element); + this->element = nullptr; + } +} diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h new file mode 100644 index 00000000..9f63906d --- /dev/null +++ b/gst/inferencecrop/cropelement.h @@ -0,0 +1,58 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __CROP_ELEMENT_H__ +#define __CROP_ELEMENT_H__ + +#include +#include +#include + + +class CropElement { + public: + CropElement (); + bool Validate (); + GstElement *GetElement (); + void SetCroppingSize (gint top, gint bottom, gint right, gint left); + virtual ~ CropElement (); + virtual const std::string& GetFactory () const = 0; + virtual GstPad *GetSinkPad () = 0; + virtual GstPad *GetSrcPad () = 0; + void Reset (); + + protected: + virtual void UpdateElement (GstElement *element, + gint top, + gint bottom, + gint right, + gint left) = 0; + + private: + GstElement *element; + gint top; + gint bottom; + gint right; + gint left; + std::mutex mutex; +}; + +#endif //__CROP_ELEMENT_H__ diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc new file mode 100644 index 00000000..3b2a7592 --- /dev/null +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -0,0 +1,483 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstdetectioncrop + * + * The detectioncrop element parses a detectionmeta and crops the incomming + * image to the bounding box. + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! videoconvert ! tee name=t t. ! videoscale ! queue ! + net.sink_model t. ! queue ! net.sink_bypass tinyyolov2 name=net model-location=$MODEL_LOCATION ! \ + backend=tensorflow backend::input-layer=$INPUT_LAYER backend::output-layer=OUTPUT_LAYER net.src_bypass \ + detectioncrop aspect-ratio=1/1 ! videoscale ! ximagesink sync=false + * ]| + * Process video frames from the camera using a detectioncrop model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + + +#include "gstdetectioncrop.h" + +#include "gst/r2inference/gstinferencemeta.h" +#include "videocrop.h" + +#include +/* generic templates */ +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); +#define GST_CAT_DEFAULT gst_detection_crop_debug_category + +static void gst_detection_crop_finalize (GObject *object); +static void gst_detection_crop_set_property (GObject *object, + guint property_id, const GValue *value, GParamSpec *pspec); +static void gst_detection_crop_get_property (GObject *object, + guint property_id, GValue *value, GParamSpec *pspec); +static GstStateChangeReturn gst_detection_crop_change_state (GstElement * + element, GstStateChange transition); +static gboolean gst_detection_crop_start (GstDetectionCrop *self); +static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstDetectionCrop *self); +static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, + gint y, + gint width, gint height, gint width_ratio, gint height_ratio, gint *top, + gint *bottom, gint *right, gint *left); +static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, + GstPadProbeInfo *info, GstDetectionCrop *self); +static void gst_detection_crop_find_predictions (GstDetectionCrop *self, + gint *num_detections, GstInferenceMeta *meta, GList **list, + GstInferencePrediction *pred); + + +#define PROP_CROP_RATIO_DEFAULT_WIDTH 1 +#define PROP_CROP_RATIO_DEFAULT_HEIGHT 1 + +enum { + PROP_0, + PROP_CROP_ASPECT_RATIO, +}; + +struct _GstDetectionCrop { + GstBin parent; + GstPad *sinkpad; + GstPad *srcpad; + CropElement *element; + gint width_ratio; + gint height_ratio; + gint width; + gint height; +}; + +struct _GstDetectionCropClass { + GstBinClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstDetectionCrop, gst_detection_crop, + GST_TYPE_BIN, + GST_DEBUG_CATEGORY_INIT (gst_detection_crop_debug_category, "detectioncrop", + 0, "debug category for detectioncrop element")); + +static void +gst_detection_crop_class_init (GstDetectionCropClass *klass) { + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_template)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_template)); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "detectioncrop", "Filter", + "Crops an incoming image based on an inference prediction bounding box", + "Michael Gruner \n\t\t\t" + " Lenin Torres "); + + element_class->change_state = + GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); + + object_class->finalize = gst_detection_crop_finalize; + object_class->set_property = gst_detection_crop_set_property; + object_class->get_property = gst_detection_crop_get_property; + + g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, + gst_param_spec_fraction ("aspect-ratio", "Aspect Ratio", + "Aspect ratio to crop the detections, width and height separated by '/'. " + "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, + G_MAXINT, 1, + PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, + G_PARAM_READWRITE )); +} + +static void +gst_detection_crop_init (GstDetectionCrop *self) { + GstElement *element; + GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; + + self->sinkpad = NULL; + self->element = new VideoCrop (); + self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; + self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; + if (FALSE == self->element->Validate ()) { + const std::string factory = self->element->GetFactory (); + GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); + return; + } + + element = self->element->GetElement (); + gst_bin_add (GST_BIN (self), GST_ELEMENT (gst_object_ref (element))); + + sinkpad = self->element->GetSinkPad (); + g_return_if_fail (sinkpad); + + self->sinkpad = GST_PAD(gst_object_ref(sinkpad)); + + sinkgpad = gst_ghost_pad_new ("sink", sinkpad); + gst_pad_set_active (sinkgpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), sinkgpad); + + g_signal_connect (sinkgpad, "notify::caps", + G_CALLBACK (gst_detection_crop_set_caps), self); + gst_pad_add_probe (sinkgpad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) gst_detection_crop_new_buffer, self, NULL); + + srcpad = self->element->GetSrcPad (); + g_return_if_fail (srcpad); + + self->srcpad = GST_PAD(gst_object_ref(srcpad)); + + srcgpad = gst_ghost_pad_new ("src", srcpad); + gst_pad_set_active (srcgpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), srcgpad); + + gst_object_unref (sinkpad); + gst_object_unref (srcpad); +} + +static void +gst_detection_crop_finalize (GObject *object) { + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + g_return_if_fail(self); + + delete (self->element); + gst_object_unref (self->sinkpad); + self->sinkpad = NULL; + gst_object_unref (self->srcpad); + self->srcpad = NULL; + + G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); +} + +static void +gst_detection_crop_set_property (GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) { + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + GST_DEBUG_OBJECT (self, "set_property"); + + switch (property_id) { + case PROP_CROP_ASPECT_RATIO: + GST_OBJECT_LOCK (self); + if (GST_VALUE_HOLDS_FRACTION (value)) { + self->width_ratio = gst_value_get_fraction_numerator (value); + self->height_ratio = gst_value_get_fraction_denominator (value); + } + GST_OBJECT_UNLOCK (self); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_detection_crop_get_property (GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) { + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + GST_DEBUG_OBJECT (self, "get_property"); + + switch (property_id) { + case PROP_CROP_ASPECT_RATIO: + GST_OBJECT_LOCK (self); + gst_value_set_fraction (value, self->width_ratio, self->height_ratio); + GST_OBJECT_UNLOCK (self); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static gboolean +gst_detection_crop_start (GstDetectionCrop *self) { + g_return_val_if_fail (self, FALSE); + + return self->element->Validate (); +} + +static GstStateChangeReturn +gst_detection_crop_change_state (GstElement *element, + GstStateChange transition) { + GstStateChangeReturn ret; + GstDetectionCrop *self = GST_DETECTION_CROP (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + if (FALSE == gst_detection_crop_start (self)) { + GST_ERROR_OBJECT (self, "Failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + default: + break; + } + ret = + GST_ELEMENT_CLASS (gst_detection_crop_parent_class)->change_state + (element, transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + +out: + return ret; +} + +static void +gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstDetectionCrop *self) { + GstCaps *caps; + GstStructure *st; + gint width; + gint height; + + g_object_get (pad, "caps", &caps, NULL); + if (NULL == caps) { + return; + } + + st = gst_caps_get_structure (caps, 0); + gst_structure_get_int (st, "width", &width); + gst_structure_get_int (st, "height", &height); + + GST_INFO_OBJECT (self, "Set new caps to %" GST_PTR_FORMAT, caps); + self->width = width; + self->height = height; + gst_caps_unref(caps); +} + +static void +gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, + gint width, gint height, gint width_ratio, gint height_ratio, gint *top, + gint *bottom, gint *right, gint *left) { + + *top = y; + *bottom = self->height - y - height; + *left = x; + *right = self->width - x - width; + + if (width_ratio > 0 && height_ratio > 0) { + gint top_bottom_modify = round(((height_ratio * width) / width_ratio - height) / + 2); + gint left_right_modify = round(((width_ratio * height) / height_ratio - width) / + 2); + if (width_ratio <= height_ratio) { + if (width > height) { + *top = *top - top_bottom_modify; + *bottom = *bottom - top_bottom_modify; + } else { + *left = *left - left_right_modify; + *right = *right - left_right_modify; + } + } else { + if (width < height) { + *left = *left - left_right_modify; + *right = *right - left_right_modify; + } else { + *top = *top - top_bottom_modify; + *bottom = *bottom - top_bottom_modify; + } + } + } + + if (*top < 0) { + *top = 0; + } + + if (*bottom < 0) { + *bottom = 0; + } + + if (*left < 0) { + *left = 0; + } + + if (*right < 0) { + *right = 0; + } + +} + +static void +gst_detection_crop_find_predictions (GstDetectionCrop *self, + gint *num_detections, + GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { + GSList *children_list = NULL; + GSList *iter = NULL; + + g_return_if_fail (self); + g_return_if_fail (num_detections); + g_return_if_fail (meta); + g_return_if_fail (list); + g_return_if_fail (pred); + + children_list = gst_inference_prediction_get_children(pred); + + for (iter = children_list; iter != NULL; iter = g_slist_next(iter)) { + GstInferencePrediction *predict = (GstInferencePrediction *)iter->data; + + gst_detection_crop_find_predictions (self, num_detections, meta, list, + predict ); + } + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { + *list = g_list_append (*list, pred); + *num_detections = *num_detections + 1; + } + +} + +static GstPadProbeReturn +gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, + GstDetectionCrop *self) { + GstBuffer *buffer; + GstInferenceMeta *inference_meta; + gint num_detections; + gint crop_width_ratio; + gint crop_height_ratio; + BoundingBox box; + GstPadProbeReturn ret = GST_PAD_PROBE_DROP; + GList *list = NULL; + GList *iter = NULL; + gboolean gap = TRUE; + + GST_OBJECT_LOCK (self); + crop_width_ratio = self->width_ratio; + crop_height_ratio = self->height_ratio; + GST_OBJECT_UNLOCK (self); + + buffer = gst_pad_probe_info_get_buffer (info); + + inference_meta = + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + GST_INFERENCE_META_API_TYPE); + + if (NULL == inference_meta ) { + GST_LOG_OBJECT (self, "No meta found, dropping buffer"); + goto out; + } + + num_detections = 0; + gst_detection_crop_find_predictions (self, &num_detections, + inference_meta, &list, inference_meta->prediction); + + for (iter = list; iter != NULL; iter = g_list_next(iter)) { + GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; + GstBuffer *croped_buffer; + GstInferenceMeta *dmeta; + gint top, bottom, right, left = 0; + box = pred->bbox; + GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, + box.height); + + gst_detection_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, + crop_width_ratio, crop_height_ratio, + &top, &bottom, &right, &left); + self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, + (gint) left); + + croped_buffer = gst_buffer_copy (buffer); + + dmeta = (GstInferenceMeta *) gst_buffer_get_meta (croped_buffer, + GST_INFERENCE_META_API_TYPE); + + gst_inference_prediction_unref (dmeta->prediction); + + dmeta->prediction = gst_inference_prediction_copy (pred); + + dmeta->prediction->bbox.x = 0; + dmeta->prediction->bbox.y = 0; + dmeta->prediction->bbox.width = self->width - right - left; + dmeta->prediction->bbox.height = self->height - top - bottom; + + if (GST_FLOW_OK != gst_pad_chain(self->sinkpad, croped_buffer)) { + GST_ELEMENT_ERROR(self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); + } + gap = FALSE; + } + + ret = GST_PAD_PROBE_DROP; + +out: + if (gap) { + gst_pad_push_event (self->srcpad, gst_event_new_gap (GST_BUFFER_TIMESTAMP (buffer), + GST_BUFFER_DURATION (buffer))); + } + + return ret; +} + +static gboolean +plugin_init (GstPlugin *plugin) { + gboolean ret = TRUE; + + ret = + gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, + GST_TYPE_DETECTION_CROP); + + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencecrop, + "Crops an incoming image based on an inference prediction bounding box", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencecrop/gstdetectioncrop.h b/gst/inferencecrop/gstdetectioncrop.h new file mode 100644 index 00000000..cb577819 --- /dev/null +++ b/gst/inferencecrop/gstdetectioncrop.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_DETECTIONCROP_H_ +#define _GST_DETECTIONCROP_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_DETECTION_CROP gst_detection_crop_get_type () +G_DECLARE_FINAL_TYPE (GstDetectionCrop, gst_detection_crop, GST, DETECTION_CROP, + GstBin) + +G_END_DECLS + +#endif diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc new file mode 100644 index 00000000..041dfdc6 --- /dev/null +++ b/gst/inferencecrop/videocrop.cc @@ -0,0 +1,65 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "videocrop.h" + +#include +#include + +const std::string& +VideoCrop::GetFactory () const { + return this->factory; +} + +void +VideoCrop::UpdateElement (GstElement *element, + gint top, + gint bottom, + gint right, + gint left) { + + g_return_if_fail (element); + + g_object_set (element, + "top", top, + "bottom", bottom, + "left", left, + "right", right, + "qos",FALSE, NULL); +} + +GstPad * +VideoCrop::GetSinkPad () { + return this->GetPad ("sink"); +} + +GstPad * +VideoCrop::GetSrcPad () { + return this->GetPad ("src"); +} + +GstPad * +VideoCrop::GetPad (const std::string &name) { + GstElement *element = this->GetElement (); + GstPad *pad = gst_element_get_static_pad (element, name.c_str()); + + return pad; +} diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h new file mode 100644 index 00000000..2510d342 --- /dev/null +++ b/gst/inferencecrop/videocrop.h @@ -0,0 +1,46 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __VIDEO_CROP_H__ +#define __VIDEO_CROP_H__ + +#include "cropelement.h" + +#include + +class VideoCrop: public CropElement { + public: + const std::string& GetFactory () const override; + GstPad *GetSrcPad () override; + GstPad *GetSinkPad () override; + + protected: + void UpdateElement (GstElement *element, + gint top, + gint bottom, + gint right, + gint left) override; + private: + GstPad *GetPad (const std::string &name); + const std::string factory = "videocrop"; +}; + +#endif //__VIDEO_CROP_H__ diff --git a/gst/inferencedebug/Makefile.am b/gst/inferencedebug/Makefile.am new file mode 100644 index 00000000..3f9c28a3 --- /dev/null +++ b/gst/inferencedebug/Makefile.am @@ -0,0 +1,27 @@ +plugin_LTLIBRARIES = libgstinferencedebug.la + +libgstinferencedebug_la_SOURCES = \ + gstinferencedebug.c + +libgstinferencedebug_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + $(R2INFERENCE_CFLAGS) + +libgstinferencedebug_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencedebug_la_LDFLAGS = \ +$(GST_PLUGIN_LDFLAGS) + +libgstinferencedebug_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstinferencedebug.h diff --git a/gst/inferencedebug/gstinferencedebug.c b/gst/inferencedebug/gstinferencedebug.c new file mode 100644 index 00000000..60559d5a --- /dev/null +++ b/gst/inferencedebug/gstinferencedebug.c @@ -0,0 +1,160 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstinferencedebug + * + * The inferencedebug element prints the inferencemeta predictions tree + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! "video/x-raw, width=1280, height=720" ! videoconvert ! tee name=t \ + t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass \ + tinyyolov2 name=net model-location=$MODEL_LOCATION backend=tensorflow backend::input-layer=$INPUT_LAYER \ + backend::output-layer=$OUTPUT_LAYER net.src_model ! inferencedebug ! fakesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencedebug.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_inference_debug_debug_category); +#define GST_CAT_DEFAULT gst_inference_debug_debug_category + +/* prototypes */ + +static void gst_inference_debug_print_predictions (GstInferenceDebug * + inferencedebug, GstInferencePrediction * root); +static GstFlowReturn gst_inference_debug_transform_ip (GstBaseTransform * trans, + GstBuffer * buf); + +struct _GstInferenceDebug +{ + GstBaseTransform base_inference_debug; +}; + +/* pad templates */ + +static GstStaticPadTemplate gst_inference_debug_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate gst_inference_debug_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferenceDebug, gst_inference_debug, + GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_inference_debug_debug_category, + "inferencedebug", 0, "debug category for inferencedebug element")); + +static void +gst_inference_debug_class_init (GstInferenceDebugClass * klass) +{ + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + + /* Setting up pads and setting metadata should be moved to + base_class_init if you intend to subclass this class. */ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inference_debug_src_template); + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inference_debug_sink_template); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "Inference Debug", "Generic", + "Prints InferenceMeta Predictions Tree", + ""); + + base_transform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_inference_debug_transform_ip); +} + +static void +gst_inference_debug_init (GstInferenceDebug * inferencedebug) +{ +} + +static void +gst_inference_debug_print_predictions (GstInferenceDebug * inferencedebug, + GstInferencePrediction * root) +{ + gchar *prediction_tree = NULL; + + g_return_if_fail (inferencedebug); + g_return_if_fail (root); + + prediction_tree = gst_inference_prediction_to_string (root); + + GST_DEBUG_OBJECT (inferencedebug, "Prediction Tree: \n %s", prediction_tree); + + g_free (prediction_tree); +} + +static GstFlowReturn +gst_inference_debug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +{ + GstInferenceDebug *inferencedebug = GST_INFERENCE_DEBUG (trans); + GstInferenceMeta *meta; + + GST_DEBUG_OBJECT (inferencedebug, "transform_ip"); + + meta = (GstInferenceMeta *) gst_buffer_get_meta (buf, + GST_INFERENCE_META_API_TYPE); + + if (NULL == meta) { + GST_LOG_OBJECT (inferencedebug, + "No inference meta found. Buffer passthrough."); + return GST_FLOW_OK; + } + + g_return_val_if_fail (meta->prediction, GST_FLOW_ERROR); + + gst_inference_debug_print_predictions (inferencedebug, meta->prediction); + + return GST_FLOW_OK; +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "inferencedebug", GST_RANK_NONE, + GST_TYPE_INFERENCE_DEBUG); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencedebug, + "Print InferenceMeta Predictions", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencedebug/gstinferencedebug.h b/gst/inferencedebug/gstinferencedebug.h new file mode 100644 index 00000000..5899e850 --- /dev/null +++ b/gst/inferencedebug/gstinferencedebug.h @@ -0,0 +1,33 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_INFERENCE_DEBUG_H_ +#define _GST_INFERENCE_DEBUG_H_ + +#include + +G_BEGIN_DECLS +#define GST_TYPE_INFERENCE_DEBUG (gst_inference_debug_get_type()) +G_DECLARE_FINAL_TYPE (GstInferenceDebug, gst_inference_debug, GST, + INFERENCE_DEBUG, GstBaseTransform) + +G_END_DECLS +#endif diff --git a/gst/inferencefilter/Makefile.am b/gst/inferencefilter/Makefile.am new file mode 100644 index 00000000..84b68a81 --- /dev/null +++ b/gst/inferencefilter/Makefile.am @@ -0,0 +1,27 @@ +plugin_LTLIBRARIES = libgstinferencefilter.la + +libgstinferencefilter_la_SOURCES = \ + gstinferencefilter.c + +libgstinferencefilter_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + $(R2INFERENCE_CFLAGS) + +libgstinferencefilter_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencefilter_la_LDFLAGS = \ +$(GST_PLUGIN_LDFLAGS) + +libgstinferencefilter_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstinferencefilter.h diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c new file mode 100644 index 00000000..5b821088 --- /dev/null +++ b/gst/inferencefilter/gstinferencefilter.c @@ -0,0 +1,287 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstinferencefilter + * + * The inferencefilter element modifies metadata on input buffer to select + * which prediction on the inference meta should be processed by modifying + * its enable property. + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! "video/x-raw, width=1280, height=720" ! videoconvert ! tee name=t \ + t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass \ + tinyyolov2 name=net model-location=$MODEL_LOCATION backend=tensorflow backend::input-layer=$INPUT_LAYER \ + backend::output-layer=$OUTPUT_LAYER net.src_model ! inferencefilter filter-class=1 ! fakesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencefilter.h" + +#include +#include + + +GST_DEBUG_CATEGORY_STATIC (gst_inferencefilter_debug_category); +#define GST_CAT_DEFAULT gst_inferencefilter_debug_category + +#define GST_INFERENCEFILTER_PROPERTY_FLAGS (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) +#define PROP_FILTER_CLASS_LABEL_DEFAULT -1 +#define PROP_FILTER_CLASS_LABEL_MIN -1 +#define PROP_RESET_ENABLE_DEFAULT FALSE + +/* prototypes */ + +static void gst_inferencefilter_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inferencefilter_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_inferencefilter_filter_enable (GstInferencefilter * + inferencefilter, GstInferencePrediction * rot, gint class_id, + gboolean reset); +static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, + GstBuffer * buf); + +enum +{ + PROP_0, + PROP_FILTER_CLASS_LABEL, + PROP_RESET_ENABLE, +}; + +struct _GstInferencefilter +{ + GstBaseTransform base_inferencefilter; + gint filter_class; + gboolean reset_enable; +}; + + +/* pad templates */ + +static GstStaticPadTemplate gst_inferencefilter_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate gst_inferencefilter_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferencefilter, gst_inferencefilter, + GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_inferencefilter_debug_category, + "inferencefilter", 0, "debug category for inferencefilter element")); + +static void +gst_inferencefilter_class_init (GstInferencefilterClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + + /* Setting up pads and setting metadata should be moved to + base_class_init if you intend to subclass this class. */ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencefilter_src_template); + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencefilter_sink_template); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "Inference Filter", "Generic", + "Enables/disables specific classes contained on the inference metadata to be processed", + ""); + + gobject_class->set_property = gst_inferencefilter_set_property; + gobject_class->get_property = gst_inferencefilter_get_property; + + g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, + g_param_spec_int ("filter-class", "filter-class", + "Filter class (-1 = disabled)", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, + PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + g_object_class_install_property (gobject_class, PROP_RESET_ENABLE, + g_param_spec_boolean ("reset-enable", "Reset enable", + "Enables all inference meta to be processed", + PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + base_transform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); +} + +static void +gst_inferencefilter_init (GstInferencefilter * inferencefilter) +{ + inferencefilter->filter_class = -1; + inferencefilter->reset_enable = FALSE; +} + +void +gst_inferencefilter_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + + GST_DEBUG_OBJECT (inferencefilter, "set_property"); + + switch (property_id) { + case PROP_FILTER_CLASS_LABEL: + GST_OBJECT_LOCK (inferencefilter); + inferencefilter->filter_class = g_value_get_int (value); + GST_OBJECT_UNLOCK (inferencefilter); + break; + case PROP_RESET_ENABLE: + GST_OBJECT_LOCK (inferencefilter); + inferencefilter->reset_enable = g_value_get_boolean (value); + GST_OBJECT_UNLOCK (inferencefilter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inferencefilter_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + + GST_DEBUG_OBJECT (inferencefilter, "get_property"); + + switch (property_id) { + case PROP_FILTER_CLASS_LABEL: + GST_OBJECT_LOCK (inferencefilter); + g_value_set_int (value, inferencefilter->filter_class); + GST_OBJECT_UNLOCK (inferencefilter); + break; + case PROP_RESET_ENABLE: + GST_OBJECT_LOCK (inferencefilter); + g_value_set_boolean (value, inferencefilter->reset_enable); + GST_OBJECT_UNLOCK (inferencefilter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, + GstInferencePrediction * root, gint class_id, gboolean reset) +{ + GList *iter = NULL; + GSList *iter_child = NULL; + + g_return_if_fail (inferencefilter); + g_return_if_fail (root); + + if (NULL == root->classifications) { + root->enabled = FALSE; + } else { + for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { + GstInferenceClassification *classification = + (GstInferenceClassification *) iter->data; + if (classification->class_id == class_id || reset) { + GST_DEBUG_OBJECT (inferencefilter, "Enabling classification id %d", + classification->class_id); + root->enabled = TRUE; + break; + } else { + GST_DEBUG_OBJECT (inferencefilter, "Disabling classification id %d", + classification->class_id); + root->enabled = FALSE; + } + } + } + + for (iter_child = gst_inference_prediction_get_children (root); + iter_child != NULL; iter_child = g_slist_next (iter_child)) { + GstInferencePrediction *predict = + (GstInferencePrediction *) iter_child->data; + gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, + reset); + } + + if (NULL != iter_child) { + g_slist_free (iter_child); + } +} + +static GstFlowReturn +gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + gboolean reset = FALSE; + gint filter = -1; + GstInferenceMeta *meta; + GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); + + meta = (GstInferenceMeta *) gst_buffer_get_meta (buf, + GST_INFERENCE_META_API_TYPE); + + if (NULL == meta) { + GST_LOG_OBJECT (inferencefilter, + "No inference meta found. Buffer passthrough."); + return GST_FLOW_OK; + } + + g_return_val_if_fail (meta->prediction, GST_FLOW_ERROR); + + GST_OBJECT_LOCK (inferencefilter); + reset = inferencefilter->reset_enable; + filter = inferencefilter->filter_class; + GST_OBJECT_UNLOCK (inferencefilter); + + if (filter < 0 && !reset) { + GST_LOG_OBJECT (inferencefilter, "Filter disabled"); + return GST_FLOW_OK; + } + + gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, + filter, reset); + return GST_FLOW_OK; +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "inferencefilter", GST_RANK_NONE, + GST_TYPE_INFERENCEFILTER); +} + + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencefilter, + "Enables/disables selected classes on inference meta to be processed", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencefilter/gstinferencefilter.h b/gst/inferencefilter/gstinferencefilter.h new file mode 100644 index 00000000..85005487 --- /dev/null +++ b/gst/inferencefilter/gstinferencefilter.h @@ -0,0 +1,33 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_INFERENCEFILTER_H_ +#define _GST_INFERENCEFILTER_H_ + +#include + +G_BEGIN_DECLS +#define GST_TYPE_INFERENCEFILTER (gst_inferencefilter_get_type()) +G_DECLARE_FINAL_TYPE (GstInferencefilter, gst_inferencefilter, GST, + INFERENCEFILTER, GstBaseTransform) + +G_END_DECLS +#endif diff --git a/tests/check/Makefile.am b/tests/check/Makefile.am index e546212c..d0df7cbc 100644 --- a/tests/check/Makefile.am +++ b/tests/check/Makefile.am @@ -45,7 +45,6 @@ AM_CFLAGS = \ LDADD = \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-1.0.la \ - $(top_builddir)/gst-libs/gst/opencv/libgstinferenceoverlay-1.0.la \ $(GST_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ -lgstvideo-1.0 \ diff --git a/tests/check/process/preprocess_functions_utils.c b/tests/check/process/preprocess_functions_utils.c index 99463064..f4cc1245 100644 --- a/tests/check/process/preprocess_functions_utils.c +++ b/tests/check/process/preprocess_functions_utils.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/tests/check/process/preprocess_functions_utils.h b/tests/check/process/preprocess_functions_utils.h index 2f257e5d..5e99ff1c 100644 --- a/tests/check/process/preprocess_functions_utils.h +++ b/tests/check/process/preprocess_functions_utils.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef __PREPROCESS_FUNCTIONS_UTILS_H__ #define __PREPROCESS_FUNCTIONS_UTILS_H__ diff --git a/tests/check/process/test_gst_fill_classification_meta_function.c b/tests/check/process/test_gst_fill_classification_meta_function.c index f3428aa1..8c221803 100644 --- a/tests/check/process/test_gst_fill_classification_meta_function.c +++ b/tests/check/process/test_gst_fill_classification_meta_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "gst/r2inference/gstinferencepostprocess.h" #include "gst/r2inference/gstinferencemeta.h" @@ -35,9 +36,7 @@ GST_START_TEST (test_gst_fill_classification_meta) (GstClassificationMeta *) gst_classification_meta_api_get_type (); gst_fill_classification_meta (class_meta, prediction, predsize); - - fail_if (class_meta->num_labels != predsize / sizeof (gfloat)); - + fail_if (class_meta->num_labels != predsize); for (gint i = 0; i < class_meta->num_labels; i++) { fail_if (class_meta->label_probs[i] != values[i]); } diff --git a/tests/check/process/test_gst_normalize_function.c b/tests/check/process/test_gst_normalize_function.c index 83b6248d..bb51b7a8 100644 --- a/tests/check/process/test_gst_normalize_function.c +++ b/tests/check/process/test_gst_normalize_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/check/process/test_gst_pixel_to_float_function.c b/tests/check/process/test_gst_pixel_to_float_function.c index fe5cefac..002c8820 100644 --- a/tests/check/process/test_gst_pixel_to_float_function.c +++ b/tests/check/process/test_gst_pixel_to_float_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/check/process/test_gst_subtract_mean_function.c b/tests/check/process/test_gst_subtract_mean_function.c index f55a9fb7..dc5e8156 100644 --- a/tests/check/process/test_gst_subtract_mean_function.c +++ b/tests/check/process/test_gst_subtract_mean_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/examples/classification/customlogic.cc b/tests/examples/classification/customlogic.cc index 7ac6217b..c19343e0 100644 --- a/tests/examples/classification/customlogic.cc +++ b/tests/examples/classification/customlogic.cc @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" diff --git a/tests/examples/classification/customlogic.h b/tests/examples/classification/customlogic.h index a6a3fcb1..45f3f1f0 100644 --- a/tests/examples/classification/customlogic.h +++ b/tests/examples/classification/customlogic.h @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __CLASSIFICATION_CUSTOM_LOGIC__ diff --git a/tests/examples/classification/gstclassification.c b/tests/examples/classification/gstclassification.c index 211731b6..4b5ffbb0 100644 --- a/tests/examples/classification/gstclassification.c +++ b/tests/examples/classification/gstclassification.c @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H diff --git a/tests/examples/detection/customlogic.cc b/tests/examples/detection/customlogic.cc index ce457e63..fa554cff 100644 --- a/tests/examples/detection/customlogic.cc +++ b/tests/examples/detection/customlogic.cc @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" @@ -16,7 +25,8 @@ void handle_prediction (unsigned char *image, - int width, int height, unsigned int size, BoundingBox * boxes, int num_boxes) + int width, int height, unsigned int size, PredictionBox * boxes, + int num_boxes) { /* FILLME: * Put here your custom logic, you may use C++ here. diff --git a/tests/examples/detection/customlogic.h b/tests/examples/detection/customlogic.h index a28bb42a..afeedbf4 100644 --- a/tests/examples/detection/customlogic.h +++ b/tests/examples/detection/customlogic.h @@ -1,13 +1,22 @@ -/* GstDetection - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __DETECTION_CUSTOM_LOGIC__ @@ -17,8 +26,8 @@ extern "C" { #endif -typedef struct _BoundingBox BoundingBox; -struct _BoundingBox { +typedef struct _PredictionBox PredictionBox; +struct _PredictionBox { int category; double probability; int x; @@ -32,7 +41,7 @@ handle_prediction (unsigned char *image, int width, int height, unsigned int size, - BoundingBox *boxes, + PredictionBox *boxes, int num_boxes); #ifdef __cplusplus diff --git a/tests/examples/detection/gstdetection.c b/tests/examples/detection/gstdetection.c index d0c3b5b4..793aaac7 100644 --- a/tests/examples/detection/gstdetection.c +++ b/tests/examples/detection/gstdetection.c @@ -1,13 +1,22 @@ -/* GstDetection - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H @@ -172,7 +181,7 @@ gst_detection_process_inference (GstElement * element, gpointer user_data) { gint index; - BoundingBox *boxes; + PredictionBox *boxes; g_return_if_fail (element); g_return_if_fail (model_meta); @@ -182,7 +191,8 @@ gst_detection_process_inference (GstElement * element, g_return_if_fail (user_data); boxes = - (BoundingBox *) g_malloc (bypass_meta->num_boxes * sizeof (BoundingBox)); + (PredictionBox *) g_malloc (bypass_meta->num_boxes * + sizeof (PredictionBox)); for (index = 0; index < bypass_meta->num_boxes; index++) { boxes[index].category = bypass_meta->boxes[index].label; diff --git a/tests/examples/embedding/customlogic.cc b/tests/examples/embedding/customlogic.cc index 4088cf5f..a26f7302 100644 --- a/tests/examples/embedding/customlogic.cc +++ b/tests/examples/embedding/customlogic.cc @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" diff --git a/tests/examples/embedding/customlogic.h b/tests/examples/embedding/customlogic.h index 2cbdf048..17b4a189 100644 --- a/tests/examples/embedding/customlogic.h +++ b/tests/examples/embedding/customlogic.h @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __EMBEDDING_CUSTOM_LOGIC__ diff --git a/tests/examples/embedding/gstembedding.c b/tests/examples/embedding/gstembedding.c index e17b7335..dbb9ff8c 100644 --- a/tests/examples/embedding/gstembedding.c +++ b/tests/examples/embedding/gstembedding.c @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H