From 12350841ae47bfdaa3bfaf604b0fc08d3f035325 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 00:37:03 -0600 Subject: [PATCH 01/49] Start development --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 75d50a07..9c9b612a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ # GstInference -A GStreamer deep learning inference framework. +> A GStreamer deep learning inference framework. + +Stay tuned for the official release! \ No newline at end of file From 9daf342fa77a34d97a48c74ff1d1ec43cc50f956 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 00:41:29 -0600 Subject: [PATCH 02/49] Import GStreamer common submodule --- .gitmodules | 3 +++ common | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 common diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..3ff9d95e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common"] + path = common + url = https://anongit.freedesktop.org/git/gstreamer/common.git diff --git a/common b/common new file mode 160000 index 00000000..cd1dee06 --- /dev/null +++ b/common @@ -0,0 +1 @@ +Subproject commit cd1dee06bf07f094677d0cf3eea4a2e8c2636b24 From d94154acfc02523cbe5ea3ecce777b6ed105c9d4 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 01:19:16 -0600 Subject: [PATCH 03/49] Add boilerplate GStreamer Autotools project --- .gitignore | 23 + AUTHORS | 3 + ChangeLog | 1 + MAINTAINERS | 11 + Makefile.am | 55 ++ NEWS | 1 + REQUIREMENTS | 54 ++ autogen.sh | 113 ++++ configure.ac | 360 +++++++++++++ docs/Makefile.am | 18 + docs/plugins/Makefile.am | 80 +++ .../plugins/gst-inference-plugin-sections.txt | 15 + docs/plugins/gst-inference-plugin.types | 0 docs/version.entities.in | 2 + ext/Makefile.am | 11 + ext/r2inference/Makefile.am | 10 + ext/r2inference/gstgooglenet.c | 489 ++++++++++++++++++ ext/r2inference/gstgooglenet.h | 44 ++ ext/r2inference/gstinference.c | 33 ++ gst-inference.doap | 45 ++ m4/Makefile.am | 1 + tests/Makefile.am | 16 + tests/check/Makefile.am | 46 ++ tests/examples/Makefile.am | 11 + tests/files/Makefile.am | 4 + 25 files changed, 1446 insertions(+) create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 ChangeLog create mode 100644 MAINTAINERS create mode 100644 Makefile.am create mode 100644 NEWS create mode 100644 REQUIREMENTS create mode 100755 autogen.sh create mode 100644 configure.ac create mode 100644 docs/Makefile.am create mode 100644 docs/plugins/Makefile.am create mode 100644 docs/plugins/gst-inference-plugin-sections.txt create mode 100644 docs/plugins/gst-inference-plugin.types create mode 100644 docs/version.entities.in create mode 100644 ext/Makefile.am create mode 100644 ext/r2inference/Makefile.am create mode 100644 ext/r2inference/gstgooglenet.c create mode 100644 ext/r2inference/gstgooglenet.h create mode 100644 ext/r2inference/gstinference.c create mode 100644 gst-inference.doap create mode 100644 m4/Makefile.am create mode 100644 tests/Makefile.am create mode 100644 tests/check/Makefile.am create mode 100644 tests/examples/Makefile.am create mode 100644 tests/files/Makefile.am diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..deca46ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Autotools ignore files +INSTALL +Makefile +Makefile.in +*.m4 +autom4te.cache/ +autoregen.sh +compile +config.* +configure +install-sh +libtool +ltmain.sh +missing +stamp-h1 + +# Test framework +test-driver +tests/check/test-suite.log + +# Editor backup +*~ + diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..9e7f24e8 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,3 @@ +Carlos Rodriguez +Jose Jimenez +Michael Gruner diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 00000000..642f772d --- /dev/null +++ b/ChangeLog @@ -0,0 +1 @@ +=== release 0.1.0 === diff --git a/MAINTAINERS b/MAINTAINERS new file mode 100644 index 00000000..3fa45235 --- /dev/null +++ b/MAINTAINERS @@ -0,0 +1,11 @@ +GStreamer is currently maintained by the consensus of a number +of people, including, but not limited to: + +Carlos Rodriguez +Jose Jimenez +Michael Gruner + + +Maintainer-related issues should be addressed to: + + support@ridgerun.com diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 00000000..3481caa7 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,55 @@ +DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc + +ALWAYS_SUBDIRS = \ + ext \ + tests \ + common \ + docs \ + m4 + +SUBDIRS = $(ALWAYS_SUBDIRS) + +DIST_SUBDIRS = $(ALWAYS_SUBDIRS) + +EXTRA_DIST = \ + depcomp \ + AUTHORS NEWS README REQUIREMENTS \ + ChangeLog gst-inference.doap autogen.sh + +DISTCLEANFILES = _stdint.h + +noinst_HEADERS = + +ACLOCAL_AMFLAGS = -I m4 -I common/m4 + +include $(top_srcdir)/common/release.mak + +check-valgrind: + $(MAKE) -C tests/check check-valgrind + +if HAVE_GST_CHECK +check-torture: + $(MAKE) -C tests/check torture + +build-checks: + $(MAKE) -C tests/check build-checks +else +check-torture: + true +build-checks: + true +endif + +include $(top_srcdir)/common/coverage/lcov.mak + +# cruft: plugins that have been merged or moved or renamed + +CRUFT_FILES = \ + $(top_builddir)/common/shave \ + $(top_builddir)/common/shave-libtool + +CRUFT_DIRS = + +include $(top_srcdir)/common/cruft.mak + +all-local: check-cruft diff --git a/NEWS b/NEWS new file mode 100644 index 00000000..2ae77312 --- /dev/null +++ b/NEWS @@ -0,0 +1 @@ +# GstInference 0.1 Release Notes diff --git a/REQUIREMENTS b/REQUIREMENTS new file mode 100644 index 00000000..bd7e6748 --- /dev/null +++ b/REQUIREMENTS @@ -0,0 +1,54 @@ +GStreamer uses a *large* array of tools and libraries, most of which are +optional. We have attempted to make sure that any code that depends on +optional libraries doesn't get built unless you have those libraries. If +you find this not to be the case, please, let us know by filing a bug +report at http://bugzilla.gnome.org/. + +Required tools: +=============== + +An extra set of tools is required if you wish to build GStreamer +from git (using autogen.sh): + +autoconf >= 2.68 https://www.gnu.org/software/autoconf/ +automake >= 1.11 https://www.gnu.org/software/automake/ +libtool >= 2.2.6 https://www.gnu.org/software/libtool/ +pkgconfig >= 0.9.0 https://www.freedesktop.org/software/pkgconfig/ + +Required libraries: +=================== + +Package: GStreamer +Version: 1.x (same 1.x version as this package) +Recommended: Latest 1.x +URL: http://gstreamer.freedesktop.org/ +DebianPackage: libgstreamer1.0-dev +Notes: The required version is updated frequently, so the version + listed in this file is often out of date. If you are compiling + from git master, you will usually need GStreamer core and + gst-plugins-base from git master as well. + +Package: GStreamer Base Plugins +Version: 1.x (same 1.x version as this package) +Recommended: Latest 1.x +URL: http://gstreamer.freedesktop.org/ +DebianPackage: libgstreamer-plugins-base1.0-dev +Notes: The required version is updated frequently, so the version + listed in this file is often out of date. If you are compiling + from git master, you will usually need GStreamer core and + gst-plugins-base from git master as well. + + +Optional libraries: +=================== + +This file lists supporting libraries for which gst-plugins-good contains +plugins, as well as their minimum required version. You can find the +corresponding plugins in ext/(library) + + +Optional (debian) packages: +=========================== + +gtk-doc-tools >= 1.12 -- needed to build documentation +python-xml -- needed to build plugin documentation diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..a6237b6e --- /dev/null +++ b/autogen.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# gst-inference autogen.sh +# +# Run this to generate all the initial makefiles, etc. +# +# This file has been generated from common/autogen.sh.in via common/update-autogen + + +test -n "$srcdir" || srcdir=`dirname "$0"` +test -n "$srcdir" || srcdir=. + +olddir=`pwd` +cd "$srcdir" + +package=gst-inference +srcfile=gst-inference.doap + +# Make sure we have common +if test ! -f common/gst-autogen.sh; +then + echo "+ Setting up common submodule" + git submodule init +fi +git submodule update + +# source helper functions +if test ! -f common/gst-autogen.sh; +then + echo There is something wrong with your source tree. + echo You are missing common/gst-autogen.sh + exit 1 +fi +. common/gst-autogen.sh + +# install pre-commit hook for doing clean commits +if test ! \( -x .git/hooks/pre-commit -a -L .git/hooks/pre-commit \); +then + rm -f .git/hooks/pre-commit + if ! ln -s ../../common/hooks/pre-commit.hook .git/hooks/pre-commit 2> /dev/null + then + echo "Failed to create commit hook symlink, copying instead ..." + cp common/hooks/pre-commit.hook .git/hooks/pre-commit + fi +fi + +CONFIGURE_DEF_OPT='--enable-maintainer-mode --enable-gtk-doc' + +if test "x$package" = "xgstreamer"; then + CONFIGURE_DEF_OPT="$CONFIGURE_DEF_OPT --enable-failing-tests --enable-poisoning" +elif test "x$package" = "xgst-plugins-bad"; then + CONFIGURE_DEF_OPT="$CONFIGURE_DEF_OPT --with-player-tests" +fi + +autogen_options $@ + +printf "+ check for build tools" +if test -z "$NOCHECK"; then + echo + + printf " checking for autoreconf ... " + echo + which "autoreconf" 2>/dev/null || { + echo "not found! Please install the autoconf package." + exit 1 + } + + printf " checking for pkg-config ... " + echo + which "pkg-config" 2>/dev/null || { + echo "not found! Please install pkg-config." + exit 1 + } +else + echo ": skipped version checks" +fi + +# if no arguments specified then this will be printed +if test -z "$*" && test -z "$NOCONFIGURE"; then + echo "+ checking for autogen.sh options" + echo " This autogen script will automatically run ./configure as:" + echo " ./configure $CONFIGURE_DEF_OPT" + echo " To pass any additional options, please specify them on the $0" + echo " command line." +fi + +toplevel_check $srcfile + +# aclocal +if test -f acinclude.m4; then rm acinclude.m4; fi + +autoreconf --force --install || exit 1 + +test -n "$NOCONFIGURE" && { + echo "+ skipping configure stage for package $package, as requested." + echo "+ autogen.sh done." + exit 0 +} + +cd "$olddir" + +echo "+ running configure ... " +test ! -z "$CONFIGURE_DEF_OPT" && echo " default flags: $CONFIGURE_DEF_OPT" +test ! -z "$CONFIGURE_EXT_OPT" && echo " external flags: $CONFIGURE_EXT_OPT" +echo + +echo "$srcdir/configure" $CONFIGURE_DEF_OPT $CONFIGURE_EXT_OPT +"$srcdir/configure" $CONFIGURE_DEF_OPT $CONFIGURE_EXT_OPT || { + echo " configure failed" + exit 1 +} + +echo "Now type 'make' to compile $package." diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..809c6b72 --- /dev/null +++ b/configure.ac @@ -0,0 +1,360 @@ +AC_PREREQ([2.69]) + +dnl please read gstreamer/docs/random/autotools before changing this file + +dnl initialize autoconf +dnl releases only do -Wall, git and prerelease does -Werror too +dnl use a three digit version number for releases, and four for git/pre +AC_INIT([GStreamer Inference],[0.1.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) + +AG_GST_INIT + +dnl initialize automake +AM_INIT_AUTOMAKE([-Wno-portability 1.14 no-dist-gzip dist-xz tar-ustar subdir-objects foreign]) + +dnl define PACKAGE_VERSION_* variables +AS_VERSION + +dnl check if this is a release version +AS_NANO(GST_GIT="no", GST_GIT="yes") + +dnl can autoconf find the source ? +AC_CONFIG_SRCDIR([ext/r2inference/gstinference.c]) + +dnl define the output header for config +AC_CONFIG_HEADERS([config.h]) + +dnl AM_MAINTAINER_MODE only provides the option to configure to enable it +AM_MAINTAINER_MODE([enable]) + +dnl sets host_* variables +AC_CANONICAL_HOST + +dnl use pretty build output with automake >= 1.11 +m4_ifdef([AM_SILENT_RULES],[AM_SILENT_RULES([yes])], + [AM_DEFAULT_VERBOSITY=1 + AC_SUBST(AM_DEFAULT_VERBOSITY)]) + +dnl our libraries and install dirs use GST_API_VERSION in the filename +dnl to allow side-by-side installation of different API versions +GST_API_VERSION=1.0 +AC_SUBST(GST_API_VERSION) +AC_DEFINE_UNQUOTED(GST_API_VERSION, "$GST_API_VERSION", + [GStreamer API Version]) + +AG_GST_LIBTOOL_PREPARE +AS_LIBTOOL(GST, 100, 0, 100) + +dnl *** required versions of GStreamer stuff *** +GST_REQ=1.8.0.1 +GSTPB_REQ=1.8.0.1 + +dnl *** autotools stuff **** + +dnl allow for different autotools +AS_AUTOTOOLS_ALTERNATE + +dnl Add parameters for aclocal +AC_SUBST(ACLOCAL_AMFLAGS, "-I m4 -I common/m4") + +dnl *** check for arguments to configure *** + +AG_GST_ARG_DISABLE_FATAL_WARNINGS +AG_GST_ARG_ENABLE_EXTRA_CHECKS + +AG_GST_ARG_DEBUG +AG_GST_ARG_PROFILING +AG_GST_ARG_VALGRIND +AG_GST_ARG_GCOV + +AG_GST_ARG_EXAMPLES + +AG_GST_ARG_WITH_PKG_CONFIG_PATH +AG_GST_ARG_WITH_PACKAGE_NAME +AG_GST_ARG_WITH_PACKAGE_ORIGIN + +AG_GST_ARG_WITH_PLUGINS + +AG_GST_ARG_ENABLE_EXTERNAL + +AG_GST_ARG_ENABLE_EXPERIMENTAL + +AG_GST_PKG_CONFIG_PATH + +dnl *** checks for platform *** + +dnl * hardware/architecture * + +dnl common/m4/gst-arch.m4 +dnl check CPU type +AG_GST_ARCH + +dnl Determine endianness +AC_C_BIGENDIAN + +dnl *** checks for programs *** + +dnl find a compiler +AC_PROG_CC +AC_PROG_CC_STDC + +dnl determine c++ compiler +AC_PROG_CXX +dnl determine if c++ is available on this system +AC_CHECK_PROG(HAVE_CXX, $CXX, yes, no) + +dnl determine c++ preprocessor +dnl FIXME: do we need this ? +AC_PROG_CXXCPP + +dnl check if the compiler supports '-c' and '-o' options +AM_PROG_CC_C_O + +dnl find an assembler +AM_PROG_AS + +dnl check if the compiler supports do while(0) macros +AG_GST_CHECK_DOWHILE_MACROS + +AC_PATH_PROG(VALGRIND_PATH, valgrind, no) +AM_CONDITIONAL(HAVE_VALGRIND, test ! "x$VALGRIND_PATH" = "xno") + +dnl check for documentation tools +GTK_DOC_CHECK([1.12]) +AG_GST_PLUGIN_DOCS([1.12]) + +dnl *** checks for libraries *** + +dnl check for pthreads +AX_PTHREAD + +dnl *** checks for header files *** + +dnl check if we have ANSI C header files +AC_HEADER_STDC + +dnl *** checks for types/defines *** + +dnl *** checks for structures *** + +dnl *** checks for compiler characteristics *** + +dnl *** checks for library functions *** + +dnl *** checks for dependency libraries *** + + +dnl GLib +GLIB_REQ=2.40.0 +AG_GST_GLIB_CHECK([$GLIB_REQ]) + +dnl checks for gstreamer +dnl uninstalled is selected preferentially -- see pkg-config(1) +AG_GST_CHECK_GST($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_BASE($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_CONTROLLER($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_CHECK($GST_API_VERSION, [$GST_REQ], no) +AG_GST_CHECK_GST_PLUGINS_BASE($GST_API_VERSION, [$GSTPB_REQ], yes) + +GST_TOOLS_DIR=`$PKG_CONFIG --variable=toolsdir gstreamer-$GST_API_VERSION` +if test -z $GST_TOOLS_DIR; then + AC_MSG_ERROR([no tools dir defined in GStreamer pkg-config file; core upgrade needed.]) +fi +AC_SUBST(GST_TOOLS_DIR) + +AC_MSG_NOTICE(Using GStreamer Core Plugins in $GST_PLUGINS_DIR) +AC_MSG_NOTICE(Using GStreamer Base Plugins in $GSTPB_PLUGINS_DIR) + +AM_CONDITIONAL(HAVE_GST_CHECK, test "x$HAVE_GST_CHECK" = "xyes") + +dnl Check for documentation xrefs +GLIB_PREFIX="`$PKG_CONFIG --variable=prefix glib-2.0`" +GST_PREFIX="`$PKG_CONFIG --variable=prefix gstreamer-$GST_API_VERSION`" +GSTPB_PREFIX="`$PKG_CONFIG --variable=prefix gstreamer-plugins-base-$GST_API_VERSION`" +AC_SUBST(GLIB_PREFIX) +AC_SUBST(GST_PREFIX) +AC_SUBST(GSTPB_PREFIX) + +dnl Check for -Bsymbolic-functions linker flag used to avoid +dnl intra-library PLT jumps, if available. +AC_ARG_ENABLE(Bsymbolic, + [AS_HELP_STRING([--disable-Bsymbolic],[avoid linking with -Bsymbolic])],, + [SAVED_LDFLAGS="${LDFLAGS}" SAVED_LIBS="${LIBS}" + AC_MSG_CHECKING([for -Bsymbolic-functions linker flag]) + LDFLAGS=-Wl,-Bsymbolic-functions + LIBS= + AC_TRY_LINK([], [return 0], + AC_MSG_RESULT(yes) + enable_Bsymbolic=yes, + AC_MSG_RESULT(no) + enable_Bsymbolic=no) + LDFLAGS="${SAVED_LDFLAGS}" LIBS="${SAVED_LIBS}"]) + +dnl *** set variables based on configure arguments *** + +dnl set license and copyright notice +GST_LICENSE="Proprietary" +AC_DEFINE_UNQUOTED(GST_LICENSE, "$GST_LICENSE", [GStreamer Inference license]) +AC_SUBST(GST_LICENSE) + +dnl set location of plugin directory +AG_GST_SET_PLUGINDIR + +dnl set release date/time +AG_GST_SET_PACKAGE_RELEASE_DATETIME_WITH_NANO([$PACKAGE_VERSION_NANO], + ["${srcdir}/gst-inference.doap"], + [$PACKAGE_VERSION_MAJOR.$PACKAGE_VERSION_MINOR.$PACKAGE_VERSION_MICRO]) + +# set by AG_GST_PARSE_SUBSYSTEM_DISABLES above +dnl make sure it does not complain about unused variables if debugging is disabled +NO_WARNINGS="" +AG_GST_CHECK_GST_DEBUG_DISABLED([NO_WARNINGS="-Wno-unused"], [NO_WARNINGS=""]) + +dnl define an ERROR_CFLAGS Makefile variable +dnl -Wundef: too many broken headers +AG_GST_SET_ERROR_CFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wmissing-prototypes -Wredundant-decls + -Wwrite-strings -Wold-style-definition -Waggregate-return + -Winit-self -Wmissing-include-dirs -Waddress -Wno-multichar + -Wnested-externs $NO_WARNINGS]) + +dnl define an ERROR_CXXFLAGS Makefile variable +AG_GST_SET_ERROR_CXXFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wredundant-decls + -Wwrite-strings + -Winit-self -Wmissing-include-dirs -Waddress -Wno-multichar + $NO_WARNINGS]) + +dnl define an ERROR_OBJCFLAGS Makefile variable +AG_GST_SET_ERROR_OBJCFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wmissing-prototypes -Wredundant-decls + -Wwrite-strings -Wold-style-definition + -Winit-self -Wmissing-include-dirs -Wno-multichar + -Wnested-externs $NO_WARNINGS]) + +dnl define correct level for debugging messages +AG_GST_SET_LEVEL_DEFAULT($GST_GIT) + +dnl used in examples +AG_GST_DEFAULT_ELEMENTS + +dnl *** plug-ins to include *** + +dnl these are all the gst plug-ins, compilable without additional libs +dnl videofilter is at the top because others depend on it +dnl AG_GST_CHECK_PLUGIN(example) + +dnl disable experimental plug-ins +dnl if test "x$BUILD_EXPERIMENTAL" != "xyes"; then + dnl AG_GST_DISABLE_PLUGIN(example) +dnl fi + +dnl *** sys plug-ins *** + +echo +AC_MSG_NOTICE([Checking libraries for plugins in sys/]) +echo + +dnl *** ext plug-ins *** +dnl keep this list sorted alphabetically ! + +if test "x$BUILD_EXTERNAL" = "xyes"; then + +echo +AC_MSG_NOTICE([Checking libraries for plugins in ext/]) +echo + +AM_CONDITIONAL(USE_R2INFERENCE, true) + +else + +dnl not building plugins with external dependencies, +dnl but we still need to set the conditionals +AM_CONDITIONAL(USE_R2INFERENCE, false) + +fi dnl of EXT plugins + +dnl *** finalize CFLAGS, LDFLAGS, LIBS + +dnl Overview: +dnl GST_OPTION_CFLAGS: common flags for profiling, debugging, errors, ... +dnl GST_*: flags shared by all built objects +dnl GST_ALL_LDFLAGS: linker flags shared by all +dnl GST_LIB_LDFLAGS: not needed, we do not install libraries +dnl GST_LT_LDFLAGS: library versioning of our libraries +dnl GST_PLUGIN_LDFLAGS: flags to be used for all plugins + +dnl GST_OPTION_CFLAGS +if test "x$USE_DEBUG" = xyes; then + PROFILE_CFLAGS="-g" +fi +AC_SUBST(PROFILE_CFLAGS) + +if test "x$PACKAGE_VERSION_NANO" = "x1"; then + dnl Define _only_ for git (not pre-releases or releases) + DEPRECATED_CFLAGS="-DGST_DISABLE_DEPRECATED" +else + DEPRECATED_CFLAGS="" +fi +AC_SUBST(DEPRECATED_CFLAGS) + +dnl every flag in GST_OPTION_CFLAGS, GST_OPTION_CXXFLAGS and GST_OPTION_OBJCFLAGS can be overridden +dnl at make time with e.g. make ERROR_CFLAGS="" +GST_OPTION_CFLAGS="\$(WARNING_CFLAGS) \$(ERROR_CFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +GST_OPTION_CXXFLAGS="\$(WARNING_CXXFLAGS) \$(ERROR_CXXFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +GST_OPTION_OBJCFLAGS="\$(WARNING_OBJCFLAGS) \$(ERROR_OBJCFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +AC_SUBST(GST_OPTION_CFLAGS) +AC_SUBST(GST_OPTION_CXXFLAGS) +AC_SUBST(GST_OPTION_OBJCFLAGS) + +dnl our libraries need to be versioned correctly +AC_SUBST(GST_LT_LDFLAGS) + +dnl FIXME: do we want to rename to GST_ALL_* ? +dnl prefer internal headers to already installed ones +dnl also add builddir include for enumtypes and marshal +dnl add ERROR_CFLAGS, but overridable +GST_CFLAGS="$GST_CFLAGS -DGST_USE_UNSTABLE_API -Wno-error=missing-include-dirs" +GST_CXXFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_CXXFLAGS)" +GST_OBJCFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_OBJCFLAGS)" +GST_CFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_CFLAGS)" +AC_SUBST(GST_CFLAGS) +AC_SUBST(GST_CXXFLAGS) +AC_SUBST(GST_OBJCFLAGS) +AC_SUBST(GST_LIBS) + +dnl LDFLAGS really should only contain flags, not libs - they get added before +dnl whatevertarget_LIBS and -L flags here affect the rest of the linking +GST_ALL_LDFLAGS="-no-undefined" +if test "x${enable_Bsymbolic}" = "xyes"; then + GST_ALL_LDFLAGS="$GST_ALL_LDFLAGS -Wl,-Bsymbolic-functions" +fi +AC_SUBST(GST_ALL_LDFLAGS) + +dnl this really should only contain flags, not libs - they get added before +dnl whatevertarget_LIBS and -L flags here affect the rest of the linking +GST_PLUGIN_LDFLAGS="-module -avoid-version -export-symbols-regex '^[_]*gst_plugin_.*' $GST_ALL_LDFLAGS" +AC_SUBST(GST_PLUGIN_LDFLAGS) + +dnl *** output files *** + +dnl keep this alphabetic per directory, please +AC_CONFIG_FILES( +Makefile +common/Makefile +common/m4/Makefile +docs/Makefile +docs/version.entities +docs/plugins/Makefile +m4/Makefile +ext/Makefile +ext/r2inference/Makefile +tests/Makefile +tests/check/Makefile +tests/examples/Makefile +tests/files/Makefile +) +AC_OUTPUT + +AG_GST_OUTPUT_PLUGINS + diff --git a/docs/Makefile.am b/docs/Makefile.am new file mode 100644 index 00000000..b71b89a4 --- /dev/null +++ b/docs/Makefile.am @@ -0,0 +1,18 @@ +if ENABLE_GTK_DOC +if ENABLE_PLUGIN_DOCS +PLUGIN_DOCS_DIRS = plugins +else +PLUGIN_DOCS_DIRS = +endif +else +PLUGIN_DOCS_DIRS = plugins +endif + +SUBDIRS = $(PLUGIN_DOCS_DIRS) +DIST_SUBDIRS = plugins + +EXTRA_DIST = \ + version.entities.in + +upload: + @if test "x$(SUBDIRS)" != x; then for a in $(SUBDIRS); do cd $$a; make upload; cd ..; done; fi diff --git a/docs/plugins/Makefile.am b/docs/plugins/Makefile.am new file mode 100644 index 00000000..07687303 --- /dev/null +++ b/docs/plugins/Makefile.am @@ -0,0 +1,80 @@ +GST_DOC_SCANOBJ = $(top_srcdir)/common/gstdoc-scangobj + +## Process this file with automake to produce Makefile.in + +# The name of the module, e.g. 'glib'. +MODULE=gst-inference +DOC_MODULE=$(MODULE)-plugin + +# for upload-doc.mak +DOC=$(MODULE)-plugin +FORMATS=html +html: html-build.stamp +include $(top_srcdir)/common/upload-doc.mak + +# The top-level SGML file. Change it if you want. +DOC_MAIN_SGML_FILE=$(DOC_MODULE)-docs.sgml + +# The directory containing the source code. +# gtk-doc will search all .c & .h files beneath here for inline comments +# documenting functions and macros. +DOC_SOURCE_DIR = $(top_srcdir)/ext + +# Extra options to supply to gtkdoc-scan. +SCAN_OPTIONS= + +# Extra options to supply to gtkdoc-mkdb. +MKDB_OPTIONS=--sgml-mode --source-suffixes=c,h,cc + +# Extra options to supply to gtkdoc-fixref. +FIXXREF_OPTIONS=--extra-dir=$(GLIB_PREFIX)/share/gtk-doc/html \ + --extra-dir=$(GST_PREFIX)/share/gtk-doc/html \ + --extra-dir=$(GSTPB_PREFIX)/share/gtk-doc/html + +# Used for dependencies. +HFILE_GLOB= \ + $(top_srcdir)/ext/*/*.h +CFILE_GLOB= \ + $(top_srcdir)/ext/*/*.c + +# Header files to ignore when scanning. +IGNORE_HFILES = +IGNORE_CFILES = + +# we add all .h files of elements that have signals/args we want +# sadly this also pulls in the private methods - maybe we should +# move those around in the source ? +# also, we should add some stuff here conditionally based on whether +# or not the plugin will actually build +# but I'm not sure about that - it might be this Just Works given that +# the registry won't have the element + +EXTRA_HFILES = +# $(top_srcdir)/ext/example/example.h# +# $(top_srcdir)/gst/example/example.h# +# $(top_srcdir)/sys/example/example.h + +# example code that needs to be converted to xml and placed in xml/ +EXAMPLE_CFILES = +# $(top_srcdir)/tests/examples/example/example.c + +# Images to copy into HTML directory. +HTML_IMAGES = + +# Extra SGML files that are included by $(DOC_MAIN_SGML_FILE). +content_files = + +# Other files to distribute. +extra_files = + +# CFLAGS and LDFLAGS for compiling scan program. Only needed if your app/lib +# contains GtkObjects/GObjects and you want to document signals and properties. +GTKDOC_CFLAGS = $(GST_BASE_CFLAGS) -I$(top_builddir) +GTKDOC_LIBS = $(GST_BASE_LIBS) + +# If you need to override some of the declarations, place them in this file +# and uncomment this line. +#DOC_OVERRIDES = $(DOC_MODULE)-overrides.txt +DOC_OVERRIDES = + +include $(top_srcdir)/common/gtk-doc-plugins.mak diff --git a/docs/plugins/gst-inference-plugin-sections.txt b/docs/plugins/gst-inference-plugin-sections.txt new file mode 100644 index 00000000..a0c620d9 --- /dev/null +++ b/docs/plugins/gst-inference-plugin-sections.txt @@ -0,0 +1,15 @@ +
+element-example +example +GstExample + +GstExampleClass +GST_EXAMPLE +GST_EXAMPLE_CAST +GST_IS_EXAMPLE +GST_EXAMPLE_CLASS +GST_IS_EXAMPLE_CLASS +GST_TYPE_EXAMPLE + +gst_example_get_type +
diff --git a/docs/plugins/gst-inference-plugin.types b/docs/plugins/gst-inference-plugin.types new file mode 100644 index 00000000..e69de29b diff --git a/docs/version.entities.in b/docs/version.entities.in new file mode 100644 index 00000000..286989f5 --- /dev/null +++ b/docs/version.entities.in @@ -0,0 +1,2 @@ + + diff --git a/ext/Makefile.am b/ext/Makefile.am new file mode 100644 index 00000000..30d735f3 --- /dev/null +++ b/ext/Makefile.am @@ -0,0 +1,11 @@ +if USE_R2INFERENCE +R2INFERENCE_DIR=r2inference +else +R2INFERENCE_DIR= +endif + +SUBDIRS=$(R2INFERENCE_DIR) + +DIST_SUBDIRS=r2inference + +include $(top_srcdir)/common/parallel-subdirs.mak diff --git a/ext/r2inference/Makefile.am b/ext/r2inference/Makefile.am new file mode 100644 index 00000000..08d914cc --- /dev/null +++ b/ext/r2inference/Makefile.am @@ -0,0 +1,10 @@ +plugin_LTLIBRARIES = libgstinference.la + +# sources used to compile this plug-in +libgstinference_la_SOURCES = gstgooglenet.h gstgooglenet.c gstinference.c + +# compiler and linker flags used to compile this plugin, set in configure.ac +libgstinference_la_CFLAGS = $(GST_CFLAGS) $(GST_BASE_CFLAGS) +libgstinference_la_LIBADD = $(GST_LIBS) $(GST_BASE_LIBS) +libgstinference_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstinference_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c new file mode 100644 index 00000000..1de1758f --- /dev/null +++ b/ext/r2inference/gstgooglenet.c @@ -0,0 +1,489 @@ +/* + * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +/** + * SECTION:element-gstgooglenet + * + * The googlenet element allows the user to infer/execute a pretrained model + * based on the GoogLeNet architecture on incoming image frames. + * + * + * Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! googlenet ! xvimagesink + * ]| + * Process video frames from the camera using a GoogLeNet model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "gstgooglenet.h" + +GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); +#define GST_CAT_DEFAULT gst_googlenet_debug_category + +/* prototypes */ + + +static void gst_googlenet_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_googlenet_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_googlenet_dispose (GObject * object); +static void gst_googlenet_finalize (GObject * object); + +#if 0 +static GstCaps *gst_googlenet_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter); +static GstCaps *gst_googlenet_fixate_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); +static gboolean gst_googlenet_accept_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps); +static gboolean gst_googlenet_set_caps (GstBaseTransform * trans, + GstCaps * incaps, GstCaps * outcaps); +static gboolean gst_googlenet_query (GstBaseTransform * trans, + GstPadDirection direction, GstQuery * query); +static gboolean gst_googlenet_decide_allocation (GstBaseTransform * trans, + GstQuery * query); +static gboolean gst_googlenet_filter_meta (GstBaseTransform * trans, + GstQuery * query, GType api, const GstStructure * params); +static gboolean gst_googlenet_propose_allocation (GstBaseTransform * trans, + GstQuery * decide_query, GstQuery * query); +static gboolean gst_googlenet_transform_size (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps, + gsize * othersize); +static gboolean gst_googlenet_get_unit_size (GstBaseTransform * trans, + GstCaps * caps, gsize * size); +static gboolean gst_googlenet_start (GstBaseTransform * trans); +static gboolean gst_googlenet_stop (GstBaseTransform * trans); +static gboolean gst_googlenet_sink_event (GstBaseTransform * trans, + GstEvent * event); +static gboolean gst_googlenet_src_event (GstBaseTransform * trans, + GstEvent * event); +static GstFlowReturn gst_googlenet_prepare_output_buffer (GstBaseTransform * + trans, GstBuffer * input, GstBuffer ** outbuf); +static gboolean gst_googlenet_copy_metadata (GstBaseTransform * trans, + GstBuffer * input, GstBuffer * outbuf); +static gboolean gst_googlenet_transform_meta (GstBaseTransform * trans, + GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf); +static void gst_googlenet_before_transform (GstBaseTransform * trans, + GstBuffer * buffer); +static GstFlowReturn gst_googlenet_transform (GstBaseTransform * trans, + GstBuffer * inbuf, GstBuffer * outbuf); +#endif +static GstFlowReturn gst_googlenet_transform_ip (GstBaseTransform * trans, + GstBuffer * buf); + +enum +{ + PROP_0 +}; + +/* pad templates */ + +static GstStaticPadTemplate gst_googlenet_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + +static GstStaticPadTemplate gst_googlenet_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstGooglenet, gst_googlenet, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_googlenet_debug_category, "googlenet", 0, + "debug category for googlenet element")); + +static void +gst_googlenet_class_init (GstGooglenetClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + + /* Setting up pads and setting metadata should be moved to + base_class_init if you intend to subclass this class. */ + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_static_pad_template_get (&gst_googlenet_src_template)); + gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), + gst_static_pad_template_get (&gst_googlenet_sink_template)); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "googlenet", "Filter", "Infers incoming image frames using a pretrained GoogLeNet model", + "Carlos Rodriguez \n\t\t\t" + " Jose Jimenez \n\t\t\t" + " Michael Gruner "); + + gobject_class->set_property = gst_googlenet_set_property; + gobject_class->get_property = gst_googlenet_get_property; + gobject_class->dispose = gst_googlenet_dispose; + gobject_class->finalize = gst_googlenet_finalize; +#if 0 + base_transform_class->transform_caps = + GST_DEBUG_FUNCPTR (gst_googlenet_transform_caps); + base_transform_class->fixate_caps = + GST_DEBUG_FUNCPTR (gst_googlenet_fixate_caps); + base_transform_class->accept_caps = + GST_DEBUG_FUNCPTR (gst_googlenet_accept_caps); + base_transform_class->set_caps = GST_DEBUG_FUNCPTR (gst_googlenet_set_caps); + base_transform_class->query = GST_DEBUG_FUNCPTR (gst_googlenet_query); + base_transform_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_googlenet_decide_allocation); + base_transform_class->filter_meta = + GST_DEBUG_FUNCPTR (gst_googlenet_filter_meta); + base_transform_class->propose_allocation = + GST_DEBUG_FUNCPTR (gst_googlenet_propose_allocation); + base_transform_class->transform_size = + GST_DEBUG_FUNCPTR (gst_googlenet_transform_size); + base_transform_class->get_unit_size = + GST_DEBUG_FUNCPTR (gst_googlenet_get_unit_size); + base_transform_class->start = GST_DEBUG_FUNCPTR (gst_googlenet_start); + base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_googlenet_stop); + base_transform_class->sink_event = + GST_DEBUG_FUNCPTR (gst_googlenet_sink_event); + base_transform_class->src_event = + GST_DEBUG_FUNCPTR (gst_googlenet_src_event); + base_transform_class->prepare_output_buffer = + GST_DEBUG_FUNCPTR (gst_googlenet_prepare_output_buffer); + base_transform_class->copy_metadata = + GST_DEBUG_FUNCPTR (gst_googlenet_copy_metadata); + base_transform_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_googlenet_transform_meta); + base_transform_class->before_transform = + GST_DEBUG_FUNCPTR (gst_googlenet_before_transform); + base_transform_class->transform = + GST_DEBUG_FUNCPTR (gst_googlenet_transform); +#endif + base_transform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_googlenet_transform_ip); + +} + +static void +gst_googlenet_init (GstGooglenet * googlenet) +{ +} + +void +gst_googlenet_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "set_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_googlenet_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "get_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_googlenet_dispose (GObject * object) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "dispose"); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS (gst_googlenet_parent_class)->dispose (object); +} + +void +gst_googlenet_finalize (GObject * object) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS (gst_googlenet_parent_class)->finalize (object); +} + +#if 0 +static GstCaps * +gst_googlenet_transform_caps (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + GstCaps *othercaps; + + GST_DEBUG_OBJECT (googlenet, "transform_caps"); + + othercaps = gst_caps_copy (caps); + + /* Copy other caps and modify as appropriate */ + /* This works for the simplest cases, where the transform modifies one + * or more fields in the caps structure. It does not work correctly + * if passthrough caps are preferred. */ + if (direction == GST_PAD_SRC) { + /* transform caps going upstream */ + } else { + /* transform caps going downstream */ + } + + if (filter) { + GstCaps *intersect; + + intersect = gst_caps_intersect (othercaps, filter); + gst_caps_unref (othercaps); + + return intersect; + } else { + return othercaps; + } +} + +static GstCaps * +gst_googlenet_fixate_caps (GstBaseTransform * trans, GstPadDirection direction, + GstCaps * caps, GstCaps * othercaps) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "fixate_caps"); + + return NULL; +} + +static gboolean +gst_googlenet_accept_caps (GstBaseTransform * trans, GstPadDirection direction, + GstCaps * caps) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "accept_caps"); + + return TRUE; +} + +static gboolean +gst_googlenet_set_caps (GstBaseTransform * trans, GstCaps * incaps, + GstCaps * outcaps) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "set_caps"); + + return TRUE; +} + +static gboolean +gst_googlenet_query (GstBaseTransform * trans, GstPadDirection direction, + GstQuery * query) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "query"); + + return TRUE; +} + +/* decide allocation query for output buffers */ +static gboolean +gst_googlenet_decide_allocation (GstBaseTransform * trans, GstQuery * query) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "decide_allocation"); + + return TRUE; +} + +static gboolean +gst_googlenet_filter_meta (GstBaseTransform * trans, GstQuery * query, + GType api, const GstStructure * params) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "filter_meta"); + + return TRUE; +} + +/* propose allocation query parameters for input buffers */ +static gboolean +gst_googlenet_propose_allocation (GstBaseTransform * trans, + GstQuery * decide_query, GstQuery * query) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "propose_allocation"); + + return TRUE; +} + +/* transform size */ +static gboolean +gst_googlenet_transform_size (GstBaseTransform * trans, + GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps, + gsize * othersize) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "transform_size"); + + return TRUE; +} + +static gboolean +gst_googlenet_get_unit_size (GstBaseTransform * trans, GstCaps * caps, + gsize * size) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "get_unit_size"); + + return TRUE; +} + +/* states */ +static gboolean +gst_googlenet_start (GstBaseTransform * trans) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "start"); + + return TRUE; +} + +static gboolean +gst_googlenet_stop (GstBaseTransform * trans) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "stop"); + + return TRUE; +} + +/* sink and src pad event handlers */ +static gboolean +gst_googlenet_sink_event (GstBaseTransform * trans, GstEvent * event) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "sink_event"); + + return + GST_BASE_TRANSFORM_CLASS (gst_googlenet_parent_class)->sink_event (trans, + event); +} + +static gboolean +gst_googlenet_src_event (GstBaseTransform * trans, GstEvent * event) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "src_event"); + + return + GST_BASE_TRANSFORM_CLASS (gst_googlenet_parent_class)->src_event (trans, + event); +} + +static GstFlowReturn +gst_googlenet_prepare_output_buffer (GstBaseTransform * trans, + GstBuffer * input, GstBuffer ** outbuf) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "prepare_output_buffer"); + + return GST_FLOW_OK; +} + +/* metadata */ +static gboolean +gst_googlenet_copy_metadata (GstBaseTransform * trans, GstBuffer * input, + GstBuffer * outbuf) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "copy_metadata"); + + return TRUE; +} + +static gboolean +gst_googlenet_transform_meta (GstBaseTransform * trans, GstBuffer * outbuf, + GstMeta * meta, GstBuffer * inbuf) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "transform_meta"); + + return TRUE; +} + +static void +gst_googlenet_before_transform (GstBaseTransform * trans, GstBuffer * buffer) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "before_transform"); + +} + +/* transform */ +static GstFlowReturn +gst_googlenet_transform (GstBaseTransform * trans, GstBuffer * inbuf, + GstBuffer * outbuf) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "transform"); + + return GST_FLOW_OK; +} +#endif + +static GstFlowReturn +gst_googlenet_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +{ + GstGooglenet *googlenet = GST_GOOGLENET (trans); + + GST_DEBUG_OBJECT (googlenet, "transform_ip"); + + return GST_FLOW_OK; +} diff --git a/ext/r2inference/gstgooglenet.h b/ext/r2inference/gstgooglenet.h new file mode 100644 index 00000000..e08cd1c3 --- /dev/null +++ b/ext/r2inference/gstgooglenet.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#ifndef _GST_GOOGLENET_H_ +#define _GST_GOOGLENET_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_GOOGLENET (gst_googlenet_get_type()) +#define GST_GOOGLENET(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GOOGLENET,GstGooglenet)) +#define GST_GOOGLENET_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GOOGLENET,GstGooglenetClass)) +#define GST_IS_GOOGLENET(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GOOGLENET)) +#define GST_IS_GOOGLENET_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GOOGLENET)) + +typedef struct _GstGooglenet GstGooglenet; +typedef struct _GstGooglenetClass GstGooglenetClass; + +struct _GstGooglenet +{ + GstBaseTransform base_googlenet; + +}; + +struct _GstGooglenetClass +{ + GstBaseTransformClass base_googlenet_class; +}; + +GType gst_googlenet_get_type (void); + +G_END_DECLS + +#endif diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c new file mode 100644 index 00000000..f90210ab --- /dev/null +++ b/ext/r2inference/gstinference.c @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstgooglenet.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + + /* FIXME Remember to set the rank if it's an element that is meant + to be autoplugged by decodebin. */ + return gst_element_register (plugin, "googlenet", GST_RANK_NONE, + GST_TYPE_GOOGLENET); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + gstinference, + "Infers a GoogLeNet trained model on incomming image frames", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst-inference.doap b/gst-inference.doap new file mode 100644 index 00000000..a50cff64 --- /dev/null +++ b/gst-inference.doap @@ -0,0 +1,45 @@ + + + GStreamer Deep Learning Inference Framework + gst-inference + + 2018-10-22 + +GStreamer Inference Framework + + +GstInference is a GStreamer based deep learning inference framework. It provides +base clases, elements and utilities to easlily and efficiently execute deep-learning +models to detect or classify objects in a scene. + + + + + + C + + + + + + + + + + + Carlos Rodriguez + + + Jose Jimenez + + + Michael Gruner + + + + diff --git a/m4/Makefile.am b/m4/Makefile.am new file mode 100644 index 00000000..d539a6ec --- /dev/null +++ b/m4/Makefile.am @@ -0,0 +1 @@ +EXTRA_DIST = diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 00000000..c6991a63 --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,16 @@ +if HAVE_GST_CHECK +SUBDIRS_CHECK = check files +else +SUBDIRS_CHECK = +endif + +#if BUILD_EXAMPLES +#SUBDIR_EXAMPLES = examples +#else +#SUBDIR_EXAMPLES = +#endif + +SUBDIRS = $(SUBDIRS_CHECK) $(SUBDIR_EXAMPLES) + +DIST_SUBDIRS = check files + diff --git a/tests/check/Makefile.am b/tests/check/Makefile.am new file mode 100644 index 00000000..0e9271ce --- /dev/null +++ b/tests/check/Makefile.am @@ -0,0 +1,46 @@ +include $(top_srcdir)/common/check.mak + +CHECK_REGISTRY = $(top_builddir)/tests/check/test-registry.reg +TEST_FILES_DIRECTORY = $(top_srcdir)/tests/files + +REGISTRY_ENVIRONMENT = \ + GST_REGISTRY_1_0=$(CHECK_REGISTRY) + +AM_TESTS_ENVIRONMENT += \ + $(REGISTRY_ENVIRONMENT) \ + GST_PLUGIN_SYSTEM_PATH_1_0= \ + GST_PLUGIN_PATH_1_0=$(top_builddir)/gst:$(top_builddir)/ext:$(top_builddir)/sys:$(GSTPB_PLUGINS_DIR):$(GST_PLUGINS_DIR) \ + GST_PLUGIN_LOADING_WHITELIST="gstreamer@$(GST_PLUGINS_DIR):gst-plugins-base@$(GSTPB_PLUGINS_DIR):gst-inference@$(top_builddir)" \ + GST_STATE_IGNORE_ELEMENTS= \ + GSETTINGS_BACKEND="memory" + +# the core dumps of some machines have PIDs appended +CLEANFILES = core.* test-registry.* + +clean-local: clean-local-check + +#if USE_PLUGIN_EXAMPLE +#check_example = \ +# elements/example \ +# elements/example +#else +#check_example = +#endif + +check_PROGRAMS = + +VALGRIND_TO_FIX = + +TESTS = $(check_PROGRAMS) + +AM_CFLAGS = $(GST_OBJ_CFLAGS) $(GST_CHECK_CFLAGS) $(CHECK_CFLAGS) \ + $(GST_OPTION_CFLAGS) $(GST_CFLAGS) -DGST_TEST_FILES_PATH="\"$(TEST_FILES_DIRECTORY)\"" \ + -DGST_CHECK_TEST_ENVIRONMENT_BEACON="\"GST_PLUGIN_LOADING_WHITELIST\"" \ + -UG_DISABLE_ASSERT -UG_DISABLE_CAST_CHECKS $(PTHREAD_CFLAGS) +LDADD = $(GST_OBJ_LIBS) $(GST_CHECK_LIBS) $(CHECK_LIBS) + +# valgrind testing +VALGRIND_TESTS_DISABLE = \ + $(VALGRIND_TO_FIX) + +SUPPRESSIONS = $(top_srcdir)/common/gst.supp diff --git a/tests/examples/Makefile.am b/tests/examples/Makefile.am new file mode 100644 index 00000000..f75716da --- /dev/null +++ b/tests/examples/Makefile.am @@ -0,0 +1,11 @@ +#if USE_EXAMPLE +#EXAMPLE_DIR=example +#else +#EXAMPLE_DIR= +#endif + +SUBDIRS = $(EXAMPLE_DIR) + +DIST_SUBDIRS = + +include $(top_srcdir)/common/parallel-subdirs.mak diff --git a/tests/files/Makefile.am b/tests/files/Makefile.am new file mode 100644 index 00000000..1f04c07b --- /dev/null +++ b/tests/files/Makefile.am @@ -0,0 +1,4 @@ + +EXTRA_DIST = + + From 3b8083d2703f82d7ab75f0d2c1621d6ae9565e01 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 01:47:00 -0600 Subject: [PATCH 04/49] Ask for r2inference during configuration --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 809c6b72..aa8de876 100644 --- a/configure.ac +++ b/configure.ac @@ -264,7 +264,9 @@ echo AC_MSG_NOTICE([Checking libraries for plugins in ext/]) echo -AM_CONDITIONAL(USE_R2INFERENCE, true) +AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun's Inference Framework], gstinference, [ + AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0) +]) else From 340f9924f8c5817d0d72428559d84b40b981f0bd Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 11:02:20 -0600 Subject: [PATCH 05/49] Extend project to build a base library as well --- Makefile.am | 11 +++++----- configure.ac | 3 +++ gst-libs/Makefile.am | 3 +++ gst-libs/gst/Makefile.am | 8 ++++++++ gst-libs/gst/r2inference/Makefile.am | 21 +++++++++++++++++++ gst-libs/gst/r2inference/gstinference.c | 27 +++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 gst-libs/Makefile.am create mode 100644 gst-libs/gst/Makefile.am create mode 100644 gst-libs/gst/r2inference/Makefile.am create mode 100644 gst-libs/gst/r2inference/gstinference.c diff --git a/Makefile.am b/Makefile.am index 3481caa7..a33a6965 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,10 +1,11 @@ DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc -ALWAYS_SUBDIRS = \ - ext \ - tests \ - common \ - docs \ +ALWAYS_SUBDIRS = \ + gst-libs \ + ext \ + tests \ + common \ + docs \ m4 SUBDIRS = $(ALWAYS_SUBDIRS) diff --git a/configure.ac b/configure.ac index aa8de876..bfa7cd94 100644 --- a/configure.ac +++ b/configure.ac @@ -348,6 +348,9 @@ common/m4/Makefile docs/Makefile docs/version.entities docs/plugins/Makefile +gst-libs/Makefile +gst-libs/gst/Makefile +gst-libs/gst/r2inference/Makefile m4/Makefile ext/Makefile ext/r2inference/Makefile diff --git a/gst-libs/Makefile.am b/gst-libs/Makefile.am new file mode 100644 index 00000000..99c71898 --- /dev/null +++ b/gst-libs/Makefile.am @@ -0,0 +1,3 @@ + +SUBDIRS=gst + diff --git a/gst-libs/gst/Makefile.am b/gst-libs/gst/Makefile.am new file mode 100644 index 00000000..a2d5c66e --- /dev/null +++ b/gst-libs/gst/Makefile.am @@ -0,0 +1,8 @@ + +SUBDIRS= + +if USE_R2INFERENCE +SUBDIRS += r2inference +endif #USE_R2INFERENCE + +DIST_SUBDIRS = r2inference diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am new file mode 100644 index 00000000..b8d7344e --- /dev/null +++ b/gst-libs/gst/r2inference/Makefile.am @@ -0,0 +1,21 @@ + +lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la + +libgstinference_@GST_API_VERSION@_la_SOURCES= \ + gstinference.c + +libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ + $(GST_CXXFLAGS) \ + $(R2INFERENCE_CFLAGS) + +libgstinference_@GST_API_VERSION@_la_CFLAGS= \ + $(GST_CFLAGS) \ + $(R2INFERENCE_CFLAGS) + +libgstinference_@GST_API_VERSION@_la_LIBADD= \ + $(GST_LIBS) \ + $(R2INFERENCE_LIBS) + +gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ + +gstinferenceinclude_HEADERS= diff --git a/gst-libs/gst/r2inference/gstinference.c b/gst-libs/gst/r2inference/gstinference.c new file mode 100644 index 00000000..787e31e7 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinference.c @@ -0,0 +1,27 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +void init (void); + +void +init () +{ +} From e8d2b41d362bf7b6e90436c15dfeb5a4997d1fd3 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 11:03:55 -0600 Subject: [PATCH 06/49] Update gitignore --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index deca46ca..cba47b79 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,17 @@ libtool ltmain.sh missing stamp-h1 +depcomp + +# Build outputs +.deps/ +.libs/ +*.la +*.lo +*.o + +# GtkDoc +docs/version.entities # Test framework test-driver From 9a3a0c256c638346158bf59c409470f8a6ab6032 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 11:20:35 -0600 Subject: [PATCH 07/49] Add placeholder for video inference base class --- gst-libs/gst/r2inference/Makefile.am | 5 ++-- gst-libs/gst/r2inference/gstvideoinference.cc | 27 +++++++++++++++++++ .../{gstinference.c => gstvideoinference.h} | 5 ---- 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 gst-libs/gst/r2inference/gstvideoinference.cc rename gst-libs/gst/r2inference/{gstinference.c => gstvideoinference.h} (97%) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index b8d7344e..22e49779 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -2,7 +2,7 @@ lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la libgstinference_@GST_API_VERSION@_la_SOURCES= \ - gstinference.c + gstvideoinference.cc libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ @@ -18,4 +18,5 @@ libgstinference_@GST_API_VERSION@_la_LIBADD= \ gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ -gstinferenceinclude_HEADERS= +gstinferenceinclude_HEADERS= \ + gstvideoinference.h diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc new file mode 100644 index 00000000..e51ab154 --- /dev/null +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -0,0 +1,27 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstvideoinference.h" + +void +init () +{ +} diff --git a/gst-libs/gst/r2inference/gstinference.c b/gst-libs/gst/r2inference/gstvideoinference.h similarity index 97% rename from gst-libs/gst/r2inference/gstinference.c rename to gst-libs/gst/r2inference/gstvideoinference.h index 787e31e7..8800a428 100644 --- a/gst-libs/gst/r2inference/gstinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -20,8 +20,3 @@ */ void init (void); - -void -init () -{ -} From 076027f2ad91af959bb440f839543dac135e07ed Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 24 Oct 2018 18:36:10 -0600 Subject: [PATCH 08/49] Add barebones videoinference base class --- .gitignore | 50 ++- ext/r2inference/Makefile.am | 32 +- ext/r2inference/gstgooglenet.c | 379 ++---------------- ext/r2inference/gstgooglenet.h | 25 +- ext/r2inference/gstinference.c | 4 +- gst-libs/gst/r2inference/Makefile.am | 7 + gst-libs/gst/r2inference/gstvideoinference.cc | 70 +++- gst-libs/gst/r2inference/gstvideoinference.h | 19 +- 8 files changed, 186 insertions(+), 400 deletions(-) diff --git a/.gitignore b/.gitignore index cba47b79..c3beb265 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,32 @@ -# Autotools ignore files -INSTALL -Makefile -Makefile.in -*.m4 +# Editor backup +*~ +*.bak +\#* +*.orig + +# Autotools cruft +aclocal.m4 autom4te.cache/ -autoregen.sh compile -config.* +config.guess +config.h +config.h.in +config.status +config.sub configure install-sh -libtool -ltmain.sh missing -stamp-h1 +Makefile +Makefile.in depcomp +libtool +ltmain.sh +m4/ +.deps/ +.libs/ +stamp-h* +autoregen.sh +gstinference-*.pc # Build outputs .deps/ @@ -22,13 +35,22 @@ depcomp *.lo *.o +# Git conflict files +.merge_file_* + # GtkDoc docs/version.entities -# Test framework +# Tests test-driver -tests/check/test-suite.log +*.xml +*.log +*.trs -# Editor backup -*~ +# 3rd party tools +.clang_complete +cscope.files +cscope.out + +# Examples diff --git a/ext/r2inference/Makefile.am b/ext/r2inference/Makefile.am index 08d914cc..0ebfbc1c 100644 --- a/ext/r2inference/Makefile.am +++ b/ext/r2inference/Makefile.am @@ -1,10 +1,26 @@ plugin_LTLIBRARIES = libgstinference.la -# sources used to compile this plug-in -libgstinference_la_SOURCES = gstgooglenet.h gstgooglenet.c gstinference.c - -# compiler and linker flags used to compile this plugin, set in configure.ac -libgstinference_la_CFLAGS = $(GST_CFLAGS) $(GST_BASE_CFLAGS) -libgstinference_la_LIBADD = $(GST_LIBS) $(GST_BASE_LIBS) -libgstinference_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) -libgstinference_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) +libgstinference_la_SOURCES = \ + gstgooglenet.c \ + gstinference.c + +libgstinference_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + +libgstinference_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinference_la_LDFLAGS = \ + $(GST_PLUGIN_LDFLAGS) + +libgstinference_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstgooglenet.h diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index 1de1758f..ad9f037d 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -29,8 +29,6 @@ #include "config.h" #endif -#include -#include #include "gstgooglenet.h" GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); @@ -46,48 +44,6 @@ static void gst_googlenet_get_property (GObject * object, static void gst_googlenet_dispose (GObject * object); static void gst_googlenet_finalize (GObject * object); -#if 0 -static GstCaps *gst_googlenet_transform_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, GstCaps * filter); -static GstCaps *gst_googlenet_fixate_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); -static gboolean gst_googlenet_accept_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps); -static gboolean gst_googlenet_set_caps (GstBaseTransform * trans, - GstCaps * incaps, GstCaps * outcaps); -static gboolean gst_googlenet_query (GstBaseTransform * trans, - GstPadDirection direction, GstQuery * query); -static gboolean gst_googlenet_decide_allocation (GstBaseTransform * trans, - GstQuery * query); -static gboolean gst_googlenet_filter_meta (GstBaseTransform * trans, - GstQuery * query, GType api, const GstStructure * params); -static gboolean gst_googlenet_propose_allocation (GstBaseTransform * trans, - GstQuery * decide_query, GstQuery * query); -static gboolean gst_googlenet_transform_size (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps, - gsize * othersize); -static gboolean gst_googlenet_get_unit_size (GstBaseTransform * trans, - GstCaps * caps, gsize * size); -static gboolean gst_googlenet_start (GstBaseTransform * trans); -static gboolean gst_googlenet_stop (GstBaseTransform * trans); -static gboolean gst_googlenet_sink_event (GstBaseTransform * trans, - GstEvent * event); -static gboolean gst_googlenet_src_event (GstBaseTransform * trans, - GstEvent * event); -static GstFlowReturn gst_googlenet_prepare_output_buffer (GstBaseTransform * - trans, GstBuffer * input, GstBuffer ** outbuf); -static gboolean gst_googlenet_copy_metadata (GstBaseTransform * trans, - GstBuffer * input, GstBuffer * outbuf); -static gboolean gst_googlenet_transform_meta (GstBaseTransform * trans, - GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf); -static void gst_googlenet_before_transform (GstBaseTransform * trans, - GstBuffer * buffer); -static GstFlowReturn gst_googlenet_transform (GstBaseTransform * trans, - GstBuffer * inbuf, GstBuffer * outbuf); -#endif -static GstFlowReturn gst_googlenet_transform_ip (GstBaseTransform * trans, - GstBuffer * buf); - enum { PROP_0 @@ -95,24 +51,35 @@ enum /* pad templates */ -static GstStaticPadTemplate gst_googlenet_src_template = -GST_STATIC_PAD_TEMPLATE ("src", - GST_PAD_SRC, - GST_PAD_ALWAYS, - GST_STATIC_CAPS ("ANY") - ); +#define CAPS "video/x-raw,format=BGR,width=224,height=224" -static GstStaticPadTemplate gst_googlenet_sink_template = -GST_STATIC_PAD_TEMPLATE ("sink", +static GstStaticPadTemplate sink_model_factory = +GST_STATIC_PAD_TEMPLATE ("sink_model", GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS ("ANY") + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) ); +static GstStaticPadTemplate src_model_factory = +GST_STATIC_PAD_TEMPLATE ("src_model", + GST_PAD_SRC, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +struct _GstGooglenet +{ + GstVideoInference parent; +}; + +struct _GstGooglenetClass +{ + GstVideoInferenceClass parent; +}; /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstGooglenet, gst_googlenet, GST_TYPE_BASE_TRANSFORM, +G_DEFINE_TYPE_WITH_CODE (GstGooglenet, gst_googlenet, GST_TYPE_VIDEO_INFERENCE, GST_DEBUG_CATEGORY_INIT (gst_googlenet_debug_category, "googlenet", 0, "debug category for googlenet element")); @@ -120,18 +87,15 @@ static void gst_googlenet_class_init (GstGooglenetClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstBaseTransformClass *base_transform_class = - GST_BASE_TRANSFORM_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); - /* Setting up pads and setting metadata should be moved to - base_class_init if you intend to subclass this class. */ - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_static_pad_template_get (&gst_googlenet_src_template)); - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_static_pad_template_get (&gst_googlenet_sink_template)); + gst_element_class_add_static_pad_template (element_class, + &sink_model_factory); + gst_element_class_add_static_pad_template (element_class, &src_model_factory); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "googlenet", "Filter", "Infers incoming image frames using a pretrained GoogLeNet model", + "googlenet", "Filter", + "Infers incoming image frames using a pretrained GoogLeNet model", "Carlos Rodriguez \n\t\t\t" " Jose Jimenez \n\t\t\t" " Michael Gruner "); @@ -140,45 +104,6 @@ gst_googlenet_class_init (GstGooglenetClass * klass) gobject_class->get_property = gst_googlenet_get_property; gobject_class->dispose = gst_googlenet_dispose; gobject_class->finalize = gst_googlenet_finalize; -#if 0 - base_transform_class->transform_caps = - GST_DEBUG_FUNCPTR (gst_googlenet_transform_caps); - base_transform_class->fixate_caps = - GST_DEBUG_FUNCPTR (gst_googlenet_fixate_caps); - base_transform_class->accept_caps = - GST_DEBUG_FUNCPTR (gst_googlenet_accept_caps); - base_transform_class->set_caps = GST_DEBUG_FUNCPTR (gst_googlenet_set_caps); - base_transform_class->query = GST_DEBUG_FUNCPTR (gst_googlenet_query); - base_transform_class->decide_allocation = - GST_DEBUG_FUNCPTR (gst_googlenet_decide_allocation); - base_transform_class->filter_meta = - GST_DEBUG_FUNCPTR (gst_googlenet_filter_meta); - base_transform_class->propose_allocation = - GST_DEBUG_FUNCPTR (gst_googlenet_propose_allocation); - base_transform_class->transform_size = - GST_DEBUG_FUNCPTR (gst_googlenet_transform_size); - base_transform_class->get_unit_size = - GST_DEBUG_FUNCPTR (gst_googlenet_get_unit_size); - base_transform_class->start = GST_DEBUG_FUNCPTR (gst_googlenet_start); - base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_googlenet_stop); - base_transform_class->sink_event = - GST_DEBUG_FUNCPTR (gst_googlenet_sink_event); - base_transform_class->src_event = - GST_DEBUG_FUNCPTR (gst_googlenet_src_event); - base_transform_class->prepare_output_buffer = - GST_DEBUG_FUNCPTR (gst_googlenet_prepare_output_buffer); - base_transform_class->copy_metadata = - GST_DEBUG_FUNCPTR (gst_googlenet_copy_metadata); - base_transform_class->transform_meta = - GST_DEBUG_FUNCPTR (gst_googlenet_transform_meta); - base_transform_class->before_transform = - GST_DEBUG_FUNCPTR (gst_googlenet_before_transform); - base_transform_class->transform = - GST_DEBUG_FUNCPTR (gst_googlenet_transform); -#endif - base_transform_class->transform_ip = - GST_DEBUG_FUNCPTR (gst_googlenet_transform_ip); - } static void @@ -239,251 +164,3 @@ gst_googlenet_finalize (GObject * object) G_OBJECT_CLASS (gst_googlenet_parent_class)->finalize (object); } - -#if 0 -static GstCaps * -gst_googlenet_transform_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, GstCaps * filter) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - GstCaps *othercaps; - - GST_DEBUG_OBJECT (googlenet, "transform_caps"); - - othercaps = gst_caps_copy (caps); - - /* Copy other caps and modify as appropriate */ - /* This works for the simplest cases, where the transform modifies one - * or more fields in the caps structure. It does not work correctly - * if passthrough caps are preferred. */ - if (direction == GST_PAD_SRC) { - /* transform caps going upstream */ - } else { - /* transform caps going downstream */ - } - - if (filter) { - GstCaps *intersect; - - intersect = gst_caps_intersect (othercaps, filter); - gst_caps_unref (othercaps); - - return intersect; - } else { - return othercaps; - } -} - -static GstCaps * -gst_googlenet_fixate_caps (GstBaseTransform * trans, GstPadDirection direction, - GstCaps * caps, GstCaps * othercaps) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "fixate_caps"); - - return NULL; -} - -static gboolean -gst_googlenet_accept_caps (GstBaseTransform * trans, GstPadDirection direction, - GstCaps * caps) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "accept_caps"); - - return TRUE; -} - -static gboolean -gst_googlenet_set_caps (GstBaseTransform * trans, GstCaps * incaps, - GstCaps * outcaps) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "set_caps"); - - return TRUE; -} - -static gboolean -gst_googlenet_query (GstBaseTransform * trans, GstPadDirection direction, - GstQuery * query) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "query"); - - return TRUE; -} - -/* decide allocation query for output buffers */ -static gboolean -gst_googlenet_decide_allocation (GstBaseTransform * trans, GstQuery * query) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "decide_allocation"); - - return TRUE; -} - -static gboolean -gst_googlenet_filter_meta (GstBaseTransform * trans, GstQuery * query, - GType api, const GstStructure * params) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "filter_meta"); - - return TRUE; -} - -/* propose allocation query parameters for input buffers */ -static gboolean -gst_googlenet_propose_allocation (GstBaseTransform * trans, - GstQuery * decide_query, GstQuery * query) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "propose_allocation"); - - return TRUE; -} - -/* transform size */ -static gboolean -gst_googlenet_transform_size (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps, - gsize * othersize) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "transform_size"); - - return TRUE; -} - -static gboolean -gst_googlenet_get_unit_size (GstBaseTransform * trans, GstCaps * caps, - gsize * size) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "get_unit_size"); - - return TRUE; -} - -/* states */ -static gboolean -gst_googlenet_start (GstBaseTransform * trans) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "start"); - - return TRUE; -} - -static gboolean -gst_googlenet_stop (GstBaseTransform * trans) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "stop"); - - return TRUE; -} - -/* sink and src pad event handlers */ -static gboolean -gst_googlenet_sink_event (GstBaseTransform * trans, GstEvent * event) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "sink_event"); - - return - GST_BASE_TRANSFORM_CLASS (gst_googlenet_parent_class)->sink_event (trans, - event); -} - -static gboolean -gst_googlenet_src_event (GstBaseTransform * trans, GstEvent * event) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "src_event"); - - return - GST_BASE_TRANSFORM_CLASS (gst_googlenet_parent_class)->src_event (trans, - event); -} - -static GstFlowReturn -gst_googlenet_prepare_output_buffer (GstBaseTransform * trans, - GstBuffer * input, GstBuffer ** outbuf) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "prepare_output_buffer"); - - return GST_FLOW_OK; -} - -/* metadata */ -static gboolean -gst_googlenet_copy_metadata (GstBaseTransform * trans, GstBuffer * input, - GstBuffer * outbuf) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "copy_metadata"); - - return TRUE; -} - -static gboolean -gst_googlenet_transform_meta (GstBaseTransform * trans, GstBuffer * outbuf, - GstMeta * meta, GstBuffer * inbuf) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "transform_meta"); - - return TRUE; -} - -static void -gst_googlenet_before_transform (GstBaseTransform * trans, GstBuffer * buffer) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "before_transform"); - -} - -/* transform */ -static GstFlowReturn -gst_googlenet_transform (GstBaseTransform * trans, GstBuffer * inbuf, - GstBuffer * outbuf) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "transform"); - - return GST_FLOW_OK; -} -#endif - -static GstFlowReturn -gst_googlenet_transform_ip (GstBaseTransform * trans, GstBuffer * buf) -{ - GstGooglenet *googlenet = GST_GOOGLENET (trans); - - GST_DEBUG_OBJECT (googlenet, "transform_ip"); - - return GST_FLOW_OK; -} diff --git a/ext/r2inference/gstgooglenet.h b/ext/r2inference/gstgooglenet.h index e08cd1c3..c0ded587 100644 --- a/ext/r2inference/gstgooglenet.h +++ b/ext/r2inference/gstgooglenet.h @@ -13,31 +13,12 @@ #ifndef _GST_GOOGLENET_H_ #define _GST_GOOGLENET_H_ -#include +#include G_BEGIN_DECLS -#define GST_TYPE_GOOGLENET (gst_googlenet_get_type()) -#define GST_GOOGLENET(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GOOGLENET,GstGooglenet)) -#define GST_GOOGLENET_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GOOGLENET,GstGooglenetClass)) -#define GST_IS_GOOGLENET(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GOOGLENET)) -#define GST_IS_GOOGLENET_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GOOGLENET)) - -typedef struct _GstGooglenet GstGooglenet; -typedef struct _GstGooglenetClass GstGooglenetClass; - -struct _GstGooglenet -{ - GstBaseTransform base_googlenet; - -}; - -struct _GstGooglenetClass -{ - GstBaseTransformClass base_googlenet_class; -}; - -GType gst_googlenet_get_type (void); +#define GST_TYPE_GOOGLENET gst_googlenet_get_type () +G_DECLARE_FINAL_TYPE (GstGooglenet, gst_googlenet, GST, GOOGLENET, GstVideoInference) G_END_DECLS diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index f90210ab..b66a71d7 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -28,6 +28,6 @@ plugin_init (GstPlugin * plugin) GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, - gstinference, - "Infers a GoogLeNet trained model on incomming image frames", + inference, + "Infer pre-trained model on incomming image frames on a variety of architectures and different backends", plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 22e49779..d1e1d2ee 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -6,14 +6,21 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ $(R2INFERENCE_CFLAGS) libgstinference_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -lgstvideo-@GST_API_VERSION@ \ $(R2INFERENCE_CFLAGS) libgstinference_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index e51ab154..d4803091 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -21,7 +21,73 @@ #include "gstvideoinference.h" -void -init () +#include + +static GstStaticPadTemplate sink_bypass_factory = GST_STATIC_PAD_TEMPLATE ("sink_bypass", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS ("ANY") + ); + +static GstStaticPadTemplate src_bypass_factory = GST_STATIC_PAD_TEMPLATE ("src_bypass", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS ("ANY") + ); + +GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); +#define GST_CAT_DEFAULT gst_video_inference_debug_category + +typedef struct _GstVideoInferencePrivate GstVideoInferencePrivate; +struct _GstVideoInferencePrivate +{ + GstCollectPads * cpads; + GstPad * sink_bypass; + GstPad * src_bypass; + GstPad * sink_model; + GstPad * src_model; +}; + +G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, GST_TYPE_ELEMENT, + GST_DEBUG_CATEGORY_INIT (gst_video_inference_debug_category, "videoinference", 0, + "debug category for videoinference base class"); + G_ADD_PRIVATE (GstVideoInference)); + +#define GST_VIDEO_INFERENCE_PRIVATE(self) \ + (GstVideoInferencePrivate *)(gst_video_inference_get_instance_private (self)) + + +static void gst_video_inference_dispose (GObject * object); + +static void +gst_video_inference_class_init (GstVideoInferenceClass * klass) { + GObjectClass *oclass = G_OBJECT_CLASS (klass); + GstElementClass *eclass = GST_ELEMENT_CLASS (klass); + + oclass->dispose = gst_video_inference_dispose; + + gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); + gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); +} + +static void +gst_video_inference_init (GstVideoInference * self) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + priv->cpads = gst_collect_pads_new (); + priv->sink_bypass = NULL; + priv->src_bypass = NULL; + priv->sink_model = NULL; + priv->src_model = NULL; +} + +static void +gst_video_inference_dispose (GObject * object) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_clear_object (&(priv->cpads)); } diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 8800a428..9b7ae325 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -19,4 +19,21 @@ * */ -void init (void); +#ifndef __GST_VIDEO_INFERENCE_H__ +#define __GST_VIDEO_INFERENCE_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_VIDEO_INFERENCE gst_video_inference_get_type () +G_DECLARE_DERIVABLE_TYPE(GstVideoInference, gst_video_inference, GST, VIDEO_INFERENCE, GstElement); + +struct _GstVideoInferenceClass +{ + GstElementClass parent_class; +}; + +G_END_DECLS + +#endif //__GST_VIDEO_INFERENCE_H__ From fd0c56330027960b78ab20a2b4b42f9ca8f052ca Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 00:52:47 -0600 Subject: [PATCH 09/49] Add methods to start and stop processing --- gst-libs/gst/r2inference/gstvideoinference.cc | 87 ++++++++++++++++++- gst-libs/gst/r2inference/gstvideoinference.h | 3 + 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index d4803091..d72498ce 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -56,9 +56,17 @@ G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, GST_TYPE_ELEMEN #define GST_VIDEO_INFERENCE_PRIVATE(self) \ (GstVideoInferencePrivate *)(gst_video_inference_get_instance_private (self)) - +/* GObject methods */ static void gst_video_inference_dispose (GObject * object); +/* GstElement methods */ +static GstStateChangeReturn gst_video_inference_change_state (GstElement *element, + GstStateChange transition); + +/* GstVideoInference methods */ +static gboolean gst_video_inference_start (GstVideoInference *self); +static gboolean gst_video_inference_stop (GstVideoInference *self); + static void gst_video_inference_class_init (GstVideoInferenceClass * klass) { @@ -67,8 +75,12 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) oclass->dispose = gst_video_inference_dispose; + eclass->change_state = GST_DEBUG_FUNCPTR (gst_video_inference_change_state); gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); + + klass->start = NULL; + klass->stop = NULL; } static void @@ -83,6 +95,79 @@ gst_video_inference_init (GstVideoInference * self) priv->src_model = NULL; } +static gboolean +gst_video_inference_start (GstVideoInference *self) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + gboolean ret = TRUE; + + if (klass->start != NULL) { + ret = klass->start (self); + } + + return ret; +} + +static gboolean +gst_video_inference_stop (GstVideoInference *self) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + gboolean ret = TRUE; + + if (klass->stop != NULL) { + ret = klass->stop (self); + } + + return ret; +} + +static GstStateChangeReturn +gst_video_inference_change_state (GstElement *element, GstStateChange transition) +{ + GstStateChangeReturn ret; + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + gst_collect_pads_start (priv->cpads); + + if (FALSE == gst_video_inference_start (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_collect_pads_stop (priv->cpads); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (gst_video_inference_parent_class)->change_state (element, + transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + if (FALSE == gst_video_inference_stop (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to stop"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + break; + default: + break; + } + + out: + return ret; +} + static void gst_video_inference_dispose (GObject * object) { diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 9b7ae325..9f08655b 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -32,6 +32,9 @@ G_DECLARE_DERIVABLE_TYPE(GstVideoInference, gst_video_inference, GST, VIDEO_INFE struct _GstVideoInferenceClass { GstElementClass parent_class; + + gboolean (* start) (GstVideoInference *self); + gboolean (* stop) (GstVideoInference *self); }; G_END_DECLS From 6f1f970b1c2961f8e101d86da862a28e7c4692e6 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 10:20:59 -0600 Subject: [PATCH 10/49] Install and remove request pads --- gst-libs/gst/r2inference/gstvideoinference.cc | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index d72498ce..6edc11d2 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -42,6 +42,9 @@ typedef struct _GstVideoInferencePrivate GstVideoInferencePrivate; struct _GstVideoInferencePrivate { GstCollectPads * cpads; + GstCollectData * sink_bypass_data; + GstCollectData * sink_model_data; + GstPad * sink_bypass; GstPad * src_bypass; GstPad * sink_model; @@ -62,10 +65,15 @@ static void gst_video_inference_dispose (GObject * object); /* GstElement methods */ static GstStateChangeReturn gst_video_inference_change_state (GstElement *element, GstStateChange transition); +static GstPad * gst_video_inference_request_new_pad (GstElement *element, + GstPadTemplate *templ, const gchar* name, const GstCaps *caps); +static void gst_video_inference_release_pad (GstElement *element, GstPad *pad); /* GstVideoInference methods */ static gboolean gst_video_inference_start (GstVideoInference *self); static gboolean gst_video_inference_stop (GstVideoInference *self); +static GstPad * gst_video_inference_create_pad (GstVideoInference * self, + GstPadTemplate *templ, const gchar* name, GstCollectData **data); static void gst_video_inference_class_init (GstVideoInferenceClass * klass) @@ -76,6 +84,8 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) oclass->dispose = gst_video_inference_dispose; eclass->change_state = GST_DEBUG_FUNCPTR (gst_video_inference_change_state); + eclass->request_new_pad = GST_DEBUG_FUNCPTR (gst_video_inference_request_new_pad); + eclass->release_pad = GST_DEBUG_FUNCPTR (gst_video_inference_release_pad); gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); @@ -89,6 +99,9 @@ gst_video_inference_init (GstVideoInference * self) GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); priv->cpads = gst_collect_pads_new (); + priv->sink_bypass_data = NULL; + priv->sink_model_data = NULL; + priv->sink_bypass = NULL; priv->src_bypass = NULL; priv->sink_model = NULL; @@ -168,6 +181,115 @@ gst_video_inference_change_state (GstElement *element, GstStateChange transition return ret; } +static GstPad * +gst_video_inference_create_pad (GstVideoInference * self, + GstPadTemplate *templ, const gchar* name, GstCollectData **data) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstElement *element = GST_ELEMENT (self); + GstPad *pad; + + GST_INFO_OBJECT (self, "Requested pad %s", name); + pad = gst_pad_new_from_template (templ, name); + + if (GST_PAD_IS_SINK (pad)) { + g_return_val_if_fail (data, NULL); + + *data = gst_collect_pads_add_pad (priv->cpads, pad, sizeof (GstCollectData), NULL, + TRUE); + if (NULL == *data) { + GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); + goto free_pad; + } + } + + if (FALSE == gst_element_add_pad (element, pad)) { + GST_ERROR_OBJECT (self, "Unable to add pad %s to element", name); + goto remove_pad; + } + + return GST_PAD_CAST(gst_object_ref (pad)); + + remove_pad: + gst_collect_pads_remove_pad (priv->cpads, pad); + + free_pad: + gst_object_unref (pad); + return NULL; +} + +static GstPad * +gst_video_inference_request_new_pad (GstElement *element, + GstPadTemplate *templ, const gchar* name, const GstCaps *caps) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + const gchar *tname; + GstPad *pad; + GstCollectData **data; + + tname = GST_PAD_TEMPLATE_NAME_TEMPLATE (templ); + + if (0 == g_strcmp0 (tname, "sink_bypass")) { + pad = priv->sink_bypass; + data = &priv->sink_bypass_data; + } else if (0 == g_strcmp0 (tname, "sink_model")) { + pad = priv->sink_model; + data = &priv->sink_model_data; + } else if (0 == g_strcmp0 (tname, "src_bypass")) { + pad = priv->src_bypass; + data = NULL; + } else if (0 == g_strcmp0 (tname, "src_model")) { + pad = priv->src_model; + data = NULL; + } else { + g_return_val_if_reached (NULL); + } + + if (NULL == pad) { + pad = gst_video_inference_create_pad (self, templ, name, data); + } else { + GST_ERROR_OBJECT (self, "Pad %s already exists", name); + pad = NULL; + } + + return pad; +} + +static void +gst_video_inference_release_pad (GstElement *element, GstPad *pad) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstPad **ourpad; + GstCollectData **data; + + if (pad == priv->sink_bypass) { + ourpad = &priv->sink_bypass; + data = &priv->sink_bypass_data; + } else if (pad == priv->src_bypass) { + ourpad = &priv->src_bypass; + data = NULL; + } else if (pad == priv->sink_model) { + ourpad = &priv->sink_model; + data = &priv->sink_model_data; + } else if (pad == priv->src_model) { + ourpad = &priv->src_model; + data = NULL; + } else { + g_return_if_reached (); + } + + GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); + *data = NULL; + + if (GST_PAD_IS_SINK (pad)) { + gst_collect_pads_remove_pad (priv->cpads, pad); + } + + g_clear_object (ourpad); +} + static void gst_video_inference_dispose (GObject * object) { @@ -175,4 +297,11 @@ gst_video_inference_dispose (GObject * object) GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); g_clear_object (&(priv->cpads)); + g_clear_object (&(priv->sink_bypass)); + g_clear_object (&(priv->sink_model)); + g_clear_object (&(priv->src_bypass)); + g_clear_object (&(priv->src_model)); + + priv->sink_bypass_data = NULL; + priv->sink_model_data = NULL; } From 19c75fda2e45d41cf83a208eafbf5ab9230a4e4a Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 12:19:00 -0600 Subject: [PATCH 11/49] Forward sink buffers to src pads accordingly --- gst-libs/gst/r2inference/gstvideoinference.cc | 89 ++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 6edc11d2..e1db3be4 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -30,7 +30,7 @@ static GstStaticPadTemplate sink_bypass_factory = GST_STATIC_PAD_TEMPLATE ("sink ); static GstStaticPadTemplate src_bypass_factory = GST_STATIC_PAD_TEMPLATE ("src_bypass", - GST_PAD_SINK, + GST_PAD_SRC, GST_PAD_REQUEST, GST_STATIC_CAPS ("ANY") ); @@ -74,6 +74,10 @@ static gboolean gst_video_inference_start (GstVideoInference *self); static gboolean gst_video_inference_stop (GstVideoInference *self); static GstPad * gst_video_inference_create_pad (GstVideoInference * self, GstPadTemplate *templ, const gchar* name, GstCollectData **data); +static GstFlowReturn gst_video_inference_collected (GstCollectPads *pads, + gpointer user_data); +static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference *self, + GstVideoInferencePrivate *priv, GstCollectData *data, GstPad *pad); static void gst_video_inference_class_init (GstVideoInferenceClass * klass) @@ -98,7 +102,6 @@ gst_video_inference_init (GstVideoInference * self) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - priv->cpads = gst_collect_pads_new (); priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; @@ -106,6 +109,11 @@ gst_video_inference_init (GstVideoInference * self) priv->src_bypass = NULL; priv->sink_model = NULL; priv->src_model = NULL; + + priv->cpads = gst_collect_pads_new (); + gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, + (gpointer) (self)); + } static gboolean @@ -225,35 +233,35 @@ gst_video_inference_request_new_pad (GstElement *element, GstVideoInference *self = GST_VIDEO_INFERENCE (element); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); const gchar *tname; - GstPad *pad; + GstPad **pad; GstCollectData **data; tname = GST_PAD_TEMPLATE_NAME_TEMPLATE (templ); if (0 == g_strcmp0 (tname, "sink_bypass")) { - pad = priv->sink_bypass; + pad = &priv->sink_bypass; data = &priv->sink_bypass_data; } else if (0 == g_strcmp0 (tname, "sink_model")) { - pad = priv->sink_model; + pad = &priv->sink_model; data = &priv->sink_model_data; } else if (0 == g_strcmp0 (tname, "src_bypass")) { - pad = priv->src_bypass; + pad = &priv->src_bypass; data = NULL; } else if (0 == g_strcmp0 (tname, "src_model")) { - pad = priv->src_model; + pad = &priv->src_model; data = NULL; } else { g_return_val_if_reached (NULL); } - if (NULL == pad) { - pad = gst_video_inference_create_pad (self, templ, name, data); + if (NULL == *pad) { + *pad = gst_video_inference_create_pad (self, templ, name, data); } else { GST_ERROR_OBJECT (self, "Pad %s already exists", name); pad = NULL; } - return pad; + return *pad; } static void @@ -290,6 +298,67 @@ gst_video_inference_release_pad (GstElement *element, GstPad *pad) g_clear_object (ourpad); } +static GstFlowReturn +gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePrivate *priv, + GstCollectData *data, GstPad *pad) +{ + GstBuffer *buffer = NULL; + GstFlowReturn ret = GST_FLOW_OK; + + /* User didn't request this pad */ + if (NULL == data) { + goto out; + } + + buffer = gst_collect_pads_pop (priv->cpads, data); + if (NULL == buffer) { + GST_INFO_OBJECT (self, "EOS requested on %s:%s", GST_DEBUG_PAD_NAME (data->pad)); + ret = GST_FLOW_EOS; + goto out; + } + + if (NULL != pad) { + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); + ret = gst_pad_push (pad, buffer); + } else { + GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", GST_DEBUG_PAD_NAME (data->pad)); + gst_buffer_unref (buffer); + goto out; + } + + if (GST_FLOW_OK != ret) { + GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", GST_DEBUG_PAD_NAME (pad), ret, + gst_flow_get_name (ret)); + } + + out: + return ret; +} + +static GstFlowReturn +gst_video_inference_collected (GstCollectPads *pads, + gpointer user_data) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstFlowReturn ret; + + ret = gst_video_inference_forward_buffer(self, priv, priv->sink_bypass_data, + priv->src_bypass); + if (GST_FLOW_OK != ret) { + goto out; + } + + ret = gst_video_inference_forward_buffer(self, priv, priv->sink_model_data, + priv->src_model); + if (GST_FLOW_OK != ret) { + goto out; + } + + out: + return ret; +} + static void gst_video_inference_dispose (GObject * object) { From 5f6b1805a977c6277e4766e902a971468fcfd245 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 15:46:56 -0600 Subject: [PATCH 12/49] Handle event forwarding between sink and src pads --- gst-libs/gst/r2inference/gstvideoinference.cc | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index e1db3be4..78f68f55 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -78,6 +78,12 @@ static GstFlowReturn gst_video_inference_collected (GstCollectPads *pads, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePrivate *priv, GstCollectData *data, GstPad *pad); +static gboolean gst_video_inference_sink_event (GstCollectPads *pads, GstCollectData *pad, + GstEvent *event, gpointer user_data); +static gboolean gst_video_inference_src_event (GstPad *pad, GstObject *parent, + GstEvent *event); +static GstPad * gst_video_inference_get_src_pad (GstVideoInference *self, + GstVideoInferencePrivate *priv, GstPad *pad); static void gst_video_inference_class_init (GstVideoInferenceClass * klass) @@ -113,7 +119,8 @@ gst_video_inference_init (GstVideoInference * self) priv->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, (gpointer) (self)); - + gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, + (gpointer) (self)); } static gboolean @@ -122,6 +129,8 @@ gst_video_inference_start (GstVideoInference *self) GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); gboolean ret = TRUE; + GST_INFO_OBJECT (self, "Starting video inference"); + if (klass->start != NULL) { ret = klass->start (self); } @@ -135,6 +144,8 @@ gst_video_inference_stop (GstVideoInference *self) GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); gboolean ret = TRUE; + GST_INFO_OBJECT (self, "Stopping video inference"); + if (klass->stop != NULL) { ret = klass->stop (self); } @@ -209,6 +220,8 @@ gst_video_inference_create_pad (GstVideoInference * self, GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); goto free_pad; } + } else { + gst_pad_set_event_function (pad, gst_video_inference_src_event); } if (FALSE == gst_element_add_pad (element, pad)) { @@ -258,7 +271,7 @@ gst_video_inference_request_new_pad (GstElement *element, *pad = gst_video_inference_create_pad (self, templ, name, data); } else { GST_ERROR_OBJECT (self, "Pad %s already exists", name); - pad = NULL; + return NULL; } return *pad; @@ -289,9 +302,9 @@ gst_video_inference_release_pad (GstElement *element, GstPad *pad) } GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); - *data = NULL; if (GST_PAD_IS_SINK (pad)) { + *data = NULL; gst_collect_pads_remove_pad (priv->cpads, pad); } @@ -359,6 +372,58 @@ gst_video_inference_collected (GstCollectPads *pads, return ret; } +static GstPad * +gst_video_inference_get_src_pad (GstVideoInference *self, GstVideoInferencePrivate *priv, GstPad *sinkpad) +{ + GstPad *pad; + + if (sinkpad == priv->sink_model) { + pad = priv->src_model; + } else if (sinkpad == priv->sink_bypass) { + pad = priv->src_bypass; + } else { + g_return_val_if_reached (NULL); + } + + return pad; +} + +static gboolean +gst_video_inference_sink_event (GstCollectPads *pads, GstCollectData *pad, + GstEvent *event, gpointer user_data) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + gboolean ret = FALSE; + GstPad *srcpad; + + srcpad = gst_video_inference_get_src_pad (self, priv, pad->pad); + + /* Collect pads will decrease the refcount of the event when we return */ + gst_event_ref (event); + + if (FALSE == gst_pad_push_event (srcpad, event)) { + GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", GST_EVENT_TYPE_NAME (event), + GST_DEBUG_PAD_NAME (srcpad)); + goto out; + } + + ret = gst_collect_pads_event_default (priv->cpads, pad, event, FALSE); + + out: + return ret; +} + +static gboolean +gst_video_inference_src_event (GstPad *pad, GstObject *parent, + GstEvent *event) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + return gst_collect_pads_src_event_default (priv->cpads, pad, event); +} + static void gst_video_inference_dispose (GObject * object) { From 9a9af0310ac2e6129ad90a27f40b78862917439d Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 18:28:25 -0600 Subject: [PATCH 13/49] Implement preprocess and postprocess virtual methods --- ext/r2inference/gstgooglenet.c | 62 ++++++++++++++++++- gst-libs/gst/r2inference/gstvideoinference.cc | 54 +++++++++++++++- gst-libs/gst/r2inference/gstvideoinference.h | 3 + 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index ad9f037d..ca0711da 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -1,10 +1,10 @@ -/* +/* * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) * All Rights Reserved. * * The contents of this software are proprietary and confidential to RidgeRun, * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of + * into another programming language without prior written consent of * RidgeRun, LLC. The user is free to modify the source code after obtaining * a software license from RidgeRun. All source code changes must be provided * back to RidgeRun without any encumbrance. @@ -44,6 +44,13 @@ static void gst_googlenet_get_property (GObject * object, static void gst_googlenet_dispose (GObject * object); static void gst_googlenet_finalize (GObject * object); +static gboolean gst_googlenet_preprocess (GstVideoInference * vi, + GstBuffer * inbuf, GstBuffer * outbuf); +static gboolean gst_googlenet_postprocess (GstVideoInference * vi, + GstBuffer * buf, const gpointer prediction, gsize predsize); +static gboolean gst_googlenet_start (GstVideoInference * vi); +static gboolean gst_googlenet_stop (GstVideoInference * vi); + enum { PROP_0 @@ -88,6 +95,7 @@ gst_googlenet_class_init (GstGooglenetClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoInferenceClass *vi_class = GST_VIDEO_INFERENCE_CLASS (klass); gst_element_class_add_static_pad_template (element_class, &sink_model_factory); @@ -104,6 +112,11 @@ gst_googlenet_class_init (GstGooglenetClass * klass) gobject_class->get_property = gst_googlenet_get_property; gobject_class->dispose = gst_googlenet_dispose; gobject_class->finalize = gst_googlenet_finalize; + + vi_class->start = GST_DEBUG_FUNCPTR (gst_googlenet_start); + vi_class->stop = GST_DEBUG_FUNCPTR (gst_googlenet_stop); + vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_googlenet_preprocess); + vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_googlenet_postprocess); } static void @@ -164,3 +177,48 @@ gst_googlenet_finalize (GObject * object) G_OBJECT_CLASS (gst_googlenet_parent_class)->finalize (object); } + +static gboolean +gst_googlenet_preprocess (GstVideoInference * vi, + GstBuffer * inbuf, GstBuffer * outbuf) +{ + GstMapInfo ininfo; + GstMapInfo outinfo; + + GST_LOG_OBJECT (vi, "Preprocess"); + + gst_buffer_map (inbuf, &ininfo, GST_MAP_READ); + gst_buffer_map (outbuf, &outinfo, GST_MAP_WRITE); + + memcpy (outinfo.data, ininfo.data, ininfo.size); + + gst_buffer_unmap (inbuf, &ininfo); + gst_buffer_unmap (outbuf, &outinfo); + + return TRUE; +} + +static gboolean +gst_googlenet_postprocess (GstVideoInference * vi, + GstBuffer * buf, const gpointer prediction, gsize predsize) +{ + GST_LOG_OBJECT (vi, "Postprocess"); + + return TRUE; +} + +static gboolean +gst_googlenet_start (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Starting GoogLeNet"); + + return TRUE; +} + +static gboolean +gst_googlenet_stop (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Stopping GoogLeNet"); + + return TRUE; +} diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 78f68f55..7371f75d 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -315,8 +315,14 @@ static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePrivate *priv, GstCollectData *data, GstPad *pad) { + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstBuffer *buffer = NULL; GstFlowReturn ret = GST_FLOW_OK; + gboolean subret = FALSE; + GstBuffer *prepared; + gpointer prediction; + gsize predsize; + /* User didn't request this pad */ if (NULL == data) { @@ -330,12 +336,52 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr goto out; } + if (data->pad == priv->sink_model && NULL != klass->preprocess) { + /* Allocate a new buffer for the preprocessing stage */ + GstAllocationParams params; + gsize size = gst_buffer_get_size (buffer); + GstBufferCopyFlags flags = (GstBufferCopyFlags)(GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS | GST_BUFFER_COPY_META); + + gst_allocation_params_init (¶ms); + prepared = gst_buffer_new_allocate (NULL, size, ¶ms); + gst_buffer_copy_into (prepared, buffer, flags, 0, size); + subret = klass->preprocess (self, buffer, prepared); + } else { + prepared = gst_buffer_ref (buffer); + subret = TRUE; + } + + /* This reference is no longer needed because the pre-process copied + it or now its called prepared */ + gst_buffer_unref (buffer); + + if (FALSE == subret) { + GST_ERROR_OBJECT (self, "Subclass failed preprocess"); + goto suberror; + } + + // TODO: Do processing here + predsize = 1000*sizeof (float); + prediction = g_malloc (predsize); + + if (NULL != klass->postprocess) { + subret = klass->postprocess (self, prepared, prediction, predsize); + } + + /* Prediction is no longer needed */ + g_free (prediction); + + if (FALSE == subret) { + GST_ERROR_OBJECT (self, "Subclass failed postprocess"); + goto suberror; + } + if (NULL != pad) { GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); - ret = gst_pad_push (pad, buffer); + ret = gst_pad_push (pad, prepared); } else { GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", GST_DEBUG_PAD_NAME (data->pad)); - gst_buffer_unref (buffer); + gst_buffer_unref (prepared); goto out; } @@ -346,6 +392,10 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr out: return ret; + + suberror: + gst_buffer_unref (prepared); + return GST_FLOW_ERROR; } static GstFlowReturn diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 9f08655b..459880e2 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -35,6 +35,9 @@ struct _GstVideoInferenceClass gboolean (* start) (GstVideoInference *self); gboolean (* stop) (GstVideoInference *self); + gboolean (* preprocess) (GstVideoInference *self, GstBuffer * inbuf, GstBuffer * outbuf); + gboolean (* postprocess) (GstVideoInference *self, GstBuffer *buf, const gpointer prediction, + gsize size); }; G_END_DECLS From 60bd55e089a3d2ab551c83c401f61a1a97d2ec68 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 21:07:26 -0600 Subject: [PATCH 14/49] Explicitly initialize virtual methods to NULL --- gst-libs/gst/r2inference/gstvideoinference.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 7371f75d..2f741558 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -101,6 +101,8 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) klass->start = NULL; klass->stop = NULL; + klass->preprocess = NULL; + klass->postprocess = NULL; } static void From e88131d443c9f55f0b4879424d0a65f39b58db23 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 25 Oct 2018 23:50:31 -0600 Subject: [PATCH 15/49] Implement child proxy to configure backend parameters --- gst-libs/gst/r2inference/Makefile.am | 11 +- gst-libs/gst/r2inference/gstbackend.cc | 186 ++++++++++ gst-libs/gst/r2inference/gstbackend.h | 46 +++ gst-libs/gst/r2inference/gstncsdk.cc | 50 +++ gst-libs/gst/r2inference/gstncsdk.h | 35 ++ gst-libs/gst/r2inference/gstvideoinference.cc | 341 ++++++++++++------ 6 files changed, 547 insertions(+), 122 deletions(-) create mode 100644 gst-libs/gst/r2inference/gstbackend.cc create mode 100644 gst-libs/gst/r2inference/gstbackend.h create mode 100644 gst-libs/gst/r2inference/gstncsdk.cc create mode 100644 gst-libs/gst/r2inference/gstncsdk.h diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index d1e1d2ee..858219a4 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -2,13 +2,16 @@ lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la libgstinference_@GST_API_VERSION@_la_SOURCES= \ - gstvideoinference.cc + gstvideoinference.cc \ + gstbackend.cc \ + gstncsdk.cc libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ - $(R2INFERENCE_CFLAGS) + $(R2INFERENCE_CFLAGS) \ + -std=c++11 libgstinference_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ @@ -26,4 +29,6 @@ libgstinference_@GST_API_VERSION@_la_LIBADD= \ gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ gstinferenceinclude_HEADERS= \ - gstvideoinference.h + gstvideoinference.h \ + gstbackend.h \ + gstncsdk.h diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc new file mode 100644 index 00000000..38930a1c --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -0,0 +1,186 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstbackend.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); +#define GST_CAT_DEFAULT gst_backend_debug_category + +typedef struct _GstBackendPrivate GstBackendPrivate; +struct _GstBackendPrivate +{ + std::unique_ptr params; +}; + +G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, GST_TYPE_OBJECT, + GST_DEBUG_CATEGORY_INIT (gst_backend_debug_category, "backend", 0, + "debug category for backend parameters"); + G_ADD_PRIVATE (GstBackend)); + +#define GST_BACKEND_PRIVATE(self) \ + (GstBackendPrivate *)(gst_backend_get_instance_private (self)) + +static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta * param); +static int gst_backend_param_flags (int flags); +static void gst_backend_install_properties (GstBackendClass * klass); +static void gst_backend_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec); +static void gst_backend_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec); + +static void +gst_backend_class_init (GstBackendClass * klass) +{ + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + + gst_backend_install_properties (klass); +} + +static void +gst_backend_init (GstBackend * self) +{ +} + +static void +gst_backend_install_properties (GstBackendClass * klass) +{ + GObjectClass *oclass = G_OBJECT_CLASS (klass); + r2i::RuntimeError error; + std::vector < r2i::ParameterMeta > params; + static gint nprop = 1; + + klass->props = g_hash_table_new (g_direct_hash, g_direct_equal); + + auto factory = r2i::IFrameworkFactory::MakeFactory (klass->backend, error); + auto pfactory = factory->MakeParameters (error); + error = pfactory->List (params); + + for (auto & param:params) { + GParamSpec *spec = gst_backend_param_to_spec (¶m); + + g_hash_table_insert (klass->props, GINT_TO_POINTER (nprop), + (gpointer) (param.name.c_str ())); + g_object_class_install_property (oclass, nprop, spec); + nprop++; + } +} + +static int +gst_backend_param_flags (int flags) +{ + int pflags = G_PARAM_STATIC_STRINGS; + + if (r2i::ParameterMeta::Flags::READ & flags) { + pflags += G_PARAM_READABLE; + } + + if (r2i::ParameterMeta::Flags::WRITE & flags) { + pflags += G_PARAM_WRITABLE; + } + + return pflags; +} + +static GParamSpec * +gst_backend_param_to_spec (r2i::ParameterMeta * param) +{ + GParamSpec *spec = NULL; + + switch (param->type) { + case (r2i::ParameterMeta::Type::INTEGER):{ + spec = g_param_spec_int (param->name.c_str (), + param->name.c_str (), + param->description.c_str (), + G_MININT, + G_MAXINT, 0, (GParamFlags) gst_backend_param_flags (param->flags)); + break; + } + case (r2i::ParameterMeta::Type::STRING):{ + spec = g_param_spec_string (param->name.c_str (), + param->name.c_str (), + param->description.c_str (), + NULL, (GParamFlags) gst_backend_param_flags (param->flags)); + break; + } + default: + g_return_val_if_reached (NULL); + } + + return spec; +} + +void +gst_backend_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstBackendClass *klass = GST_BACKEND_GET_CLASS (object); + GstBackend *self = GST_BACKEND (object); + const gchar *param; + + GST_DEBUG_OBJECT (self, "set_property"); + + param = (const gchar *) g_hash_table_lookup (klass->props, + GINT_TO_POINTER (property_id)); + if (NULL == param) { + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + return; + } +} + +void +gst_backend_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstBackendClass *klass = GST_BACKEND_GET_CLASS (object); + GstBackend *self = GST_BACKEND (object); + const gchar *param; + + GST_DEBUG_OBJECT (self, "get_property"); + + param = (const gchar *) g_hash_table_lookup (klass->props, + GINT_TO_POINTER (property_id)); + if (NULL == param) { + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + return; + } +} + +gboolean +gst_backend_configure (GstBackend *self, std::shared_ptr engine, + std::shared_ptr model) +{ + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + r2i::RuntimeError error; + + error = priv->params->Configure (engine, model); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Error configuring backend parameters: (%d): %s", + error.GetCode (), error.GetDescription ().c_str()); + return FALSE; + } + + return TRUE; +} diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h new file mode 100644 index 00000000..8c6158ca --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -0,0 +1,46 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_BACKEND_H__ +#define __GST_BACKEND_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_BACKEND gst_backend_get_type () +G_DECLARE_DERIVABLE_TYPE (GstBackend, gst_backend, GST, BACKEND, GstObject); + +struct _GstBackendClass +{ + GstObjectClass parent_class; + + r2i::FrameworkCode backend; + GHashTable *props; +}; + +gboolean gst_backend_configure (GstBackend *self, + std::shared_ptr engine, std::shared_ptr model); + +G_END_DECLS + +#endif //__GST_BACKEND_H__ diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc new file mode 100644 index 00000000..c079d825 --- /dev/null +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -0,0 +1,50 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstncsdk.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_ncsdk_debug_category); +#define GST_CAT_DEFAULT gst_ncsdk_debug_category + +struct _GstNcsdk +{ + GstObject parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstNcsdk, gst_ncsdk, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_ncsdk_debug_category, "ncsdk", 0, + "debug category for ncsdk parameters")); + +static void +gst_ncsdk_class_init (GstNcsdkClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + + bclass->backend = r2i::FrameworkCode::NCSDK; +} + +static void +gst_ncsdk_init (GstNcsdk * self) +{ + +} diff --git a/gst-libs/gst/r2inference/gstncsdk.h b/gst-libs/gst/r2inference/gstncsdk.h new file mode 100644 index 00000000..f29a2f87 --- /dev/null +++ b/gst-libs/gst/r2inference/gstncsdk.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_NCSDK_H__ +#define __GST_NCSDK_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_NCSDK gst_ncsdk_get_type () +G_DECLARE_FINAL_TYPE(GstNcsdk, gst_ncsdk, GST, NCSDK, GstBackend); + +G_END_DECLS + +#endif //__GST_NCSDK_H__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 2f741558..b2ae39d5 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -22,18 +22,23 @@ #include "gstvideoinference.h" #include +#include -static GstStaticPadTemplate sink_bypass_factory = GST_STATIC_PAD_TEMPLATE ("sink_bypass", +#include "gstncsdk.h" + +static GstStaticPadTemplate sink_bypass_factory = +GST_STATIC_PAD_TEMPLATE ("sink_bypass", GST_PAD_SINK, GST_PAD_REQUEST, GST_STATIC_CAPS ("ANY") - ); + ); -static GstStaticPadTemplate src_bypass_factory = GST_STATIC_PAD_TEMPLATE ("src_bypass", +static GstStaticPadTemplate src_bypass_factory = +GST_STATIC_PAD_TEMPLATE ("src_bypass", GST_PAD_SRC, GST_PAD_REQUEST, GST_STATIC_CAPS ("ANY") - ); + ); GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category @@ -41,49 +46,68 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); typedef struct _GstVideoInferencePrivate GstVideoInferencePrivate; struct _GstVideoInferencePrivate { - GstCollectPads * cpads; - GstCollectData * sink_bypass_data; - GstCollectData * sink_model_data; - - GstPad * sink_bypass; - GstPad * src_bypass; - GstPad * sink_model; - GstPad * src_model; -}; + GstCollectPads *cpads; + GstCollectData *sink_bypass_data; + GstCollectData *sink_model_data; -G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, GST_TYPE_ELEMENT, - GST_DEBUG_CATEGORY_INIT (gst_video_inference_debug_category, "videoinference", 0, - "debug category for videoinference base class"); - G_ADD_PRIVATE (GstVideoInference)); + GstPad *sink_bypass; + GstPad *src_bypass; + GstPad *sink_model; + GstPad *src_model; -#define GST_VIDEO_INFERENCE_PRIVATE(self) \ - (GstVideoInferencePrivate *)(gst_video_inference_get_instance_private (self)) + GstBackend *backend; + + std::shared_ptr engine; + std::shared_ptr loader; + std::shared_ptr model; +}; /* GObject methods */ -static void gst_video_inference_dispose (GObject * object); +static void gst_video_inference_finalize (GObject * object); /* GstElement methods */ -static GstStateChangeReturn gst_video_inference_change_state (GstElement *element, - GstStateChange transition); -static GstPad * gst_video_inference_request_new_pad (GstElement *element, - GstPadTemplate *templ, const gchar* name, const GstCaps *caps); -static void gst_video_inference_release_pad (GstElement *element, GstPad *pad); +static GstStateChangeReturn gst_video_inference_change_state (GstElement * + element, GstStateChange transition); +static GstPad *gst_video_inference_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); +static void gst_video_inference_release_pad (GstElement * element, + GstPad * pad); + +/* GstChildProxy methods */ +static void gst_video_inference_child_proxy_init (GstChildProxyInterface * + iface); +static GObject *gst_video_inference_get_child_by_name (GstChildProxy * parent, + const gchar * name); +static GObject *gst_video_inference_get_child_by_index (GstChildProxy * parent, + guint index); +static guint gst_video_inference_get_children_count (GstChildProxy * parent); /* GstVideoInference methods */ -static gboolean gst_video_inference_start (GstVideoInference *self); -static gboolean gst_video_inference_stop (GstVideoInference *self); -static GstPad * gst_video_inference_create_pad (GstVideoInference * self, - GstPadTemplate *templ, const gchar* name, GstCollectData **data); -static GstFlowReturn gst_video_inference_collected (GstCollectPads *pads, +static gboolean gst_video_inference_start (GstVideoInference * self); +static gboolean gst_video_inference_stop (GstVideoInference * self); +static GstPad *gst_video_inference_create_pad (GstVideoInference * self, + GstPadTemplate * templ, const gchar * name, GstCollectData ** data); +static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, gpointer user_data); -static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference *self, - GstVideoInferencePrivate *priv, GstCollectData *data, GstPad *pad); -static gboolean gst_video_inference_sink_event (GstCollectPads *pads, GstCollectData *pad, - GstEvent *event, gpointer user_data); -static gboolean gst_video_inference_src_event (GstPad *pad, GstObject *parent, - GstEvent *event); -static GstPad * gst_video_inference_get_src_pad (GstVideoInference *self, - GstVideoInferencePrivate *priv, GstPad *pad); +static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * + self, GstVideoInferencePrivate * priv, GstCollectData * data, GstPad * pad); +static gboolean gst_video_inference_sink_event (GstCollectPads * pads, + GstCollectData * pad, GstEvent * event, gpointer user_data); +static gboolean gst_video_inference_src_event (GstPad * pad, GstObject * parent, + GstEvent * event); +static GstPad *gst_video_inference_get_src_pad (GstVideoInference * self, + GstVideoInferencePrivate * priv, GstPad * pad); + +G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, + GST_TYPE_ELEMENT, + GST_DEBUG_CATEGORY_INIT (gst_video_inference_debug_category, + "videoinference", 0, "debug category for videoinference base class"); + G_ADD_PRIVATE (GstVideoInference); + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_video_inference_child_proxy_init)); + +#define GST_VIDEO_INFERENCE_PRIVATE(self) \ + (GstVideoInferencePrivate *)(gst_video_inference_get_instance_private (self)) static void gst_video_inference_class_init (GstVideoInferenceClass * klass) @@ -91,10 +115,11 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) GObjectClass *oclass = G_OBJECT_CLASS (klass); GstElementClass *eclass = GST_ELEMENT_CLASS (klass); - oclass->dispose = gst_video_inference_dispose; + oclass->finalize = gst_video_inference_finalize; eclass->change_state = GST_DEBUG_FUNCPTR (gst_video_inference_change_state); - eclass->request_new_pad = GST_DEBUG_FUNCPTR (gst_video_inference_request_new_pad); + eclass->request_new_pad = + GST_DEBUG_FUNCPTR (gst_video_inference_request_new_pad); eclass->release_pad = GST_DEBUG_FUNCPTR (gst_video_inference_release_pad); gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); @@ -109,6 +134,9 @@ static void gst_video_inference_init (GstVideoInference * self) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + r2i::RuntimeError error; + auto factory = r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, + error); priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; @@ -121,27 +149,91 @@ gst_video_inference_init (GstVideoInference * self) priv->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, (gpointer) (self)); - gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, - (gpointer) (self)); + gst_collect_pads_set_event_function (priv->cpads, + gst_video_inference_sink_event, (gpointer) (self)); + + priv->backend = GST_BACKEND (g_object_new (GST_TYPE_NCSDK, NULL)); + + priv->engine = factory->MakeEngine(error); + priv->loader = factory->MakeLoader(error); + priv->model = nullptr; +} + +static void +gst_video_inference_child_proxy_init (GstChildProxyInterface * iface) +{ + iface->get_child_by_name = gst_video_inference_get_child_by_name; + iface->get_child_by_index = gst_video_inference_get_child_by_index; + iface->get_children_count = gst_video_inference_get_children_count; +} + +static GObject * +gst_video_inference_get_child_by_name (GstChildProxy * parent, + const gchar * name) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_DEBUG_OBJECT (self, "Requested for child %s", name); + + if (0 == g_strcmp0 (name, "backend")) { + return G_OBJECT (g_object_ref (priv->backend)); + } else { + GST_ERROR_OBJECT (self, "No such child %s", name); + return NULL; + } +} + +static GObject * +gst_video_inference_get_child_by_index (GstChildProxy * parent, guint index) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_DEBUG_OBJECT (self, "Requested for child %d", index); + + if (0 == index) { + return G_OBJECT (g_object_ref (priv->backend)); + } else { + GST_DEBUG_OBJECT (self, "No such child %d", index); + return NULL; + } +} + +static guint +gst_video_inference_get_children_count (GstChildProxy * parent) +{ + return 1; } static gboolean -gst_video_inference_start (GstVideoInference *self) +gst_video_inference_start (GstVideoInference * self) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + //GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + r2i::RuntimeError error; gboolean ret = TRUE; GST_INFO_OBJECT (self, "Starting video inference"); + //priv->model = priv->loader->Load ("/path/to/model", error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Unable to load model: (%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + ret = FALSE; + goto out; + } + if (klass->start != NULL) { ret = klass->start (self); } + out: return ret; } static gboolean -gst_video_inference_stop (GstVideoInference *self) +gst_video_inference_stop (GstVideoInference * self) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); gboolean ret = TRUE; @@ -156,55 +248,57 @@ gst_video_inference_stop (GstVideoInference *self) } static GstStateChangeReturn -gst_video_inference_change_state (GstElement *element, GstStateChange transition) +gst_video_inference_change_state (GstElement * element, + GstStateChange transition) { GstStateChangeReturn ret; GstVideoInference *self = GST_VIDEO_INFERENCE (element); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - switch (transition) { - case GST_STATE_CHANGE_READY_TO_PAUSED: - gst_collect_pads_start (priv->cpads); - - if (FALSE == gst_video_inference_start (self)) { - GST_ERROR_OBJECT (self, "Subclass failed to start"); - ret = GST_STATE_CHANGE_FAILURE; - goto out; - } - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_collect_pads_stop (priv->cpads); - break; - default: - break; - } - - ret = GST_ELEMENT_CLASS (gst_video_inference_parent_class)->change_state (element, - transition); - if (GST_STATE_CHANGE_FAILURE == ret) { - GST_ERROR_OBJECT (self, "Parent failed to change state"); - goto out; - } - - switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: - if (FALSE == gst_video_inference_stop (self)) { - GST_ERROR_OBJECT (self, "Subclass failed to stop"); - ret = GST_STATE_CHANGE_FAILURE; - goto out; - } - break; - default: - break; - } + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + if (FALSE == gst_video_inference_start (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + + gst_collect_pads_start (priv->cpads); + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_collect_pads_stop (priv->cpads); + break; + default: + break; + } - out: - return ret; + ret = + GST_ELEMENT_CLASS (gst_video_inference_parent_class)-> + change_state (element, transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + if (FALSE == gst_video_inference_stop (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to stop"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + break; + default: + break; + } + +out: + return ret; } static GstPad * gst_video_inference_create_pad (GstVideoInference * self, - GstPadTemplate *templ, const gchar* name, GstCollectData **data) + GstPadTemplate * templ, const gchar * name, GstCollectData ** data) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstElement *element = GST_ELEMENT (self); @@ -216,8 +310,9 @@ gst_video_inference_create_pad (GstVideoInference * self, if (GST_PAD_IS_SINK (pad)) { g_return_val_if_fail (data, NULL); - *data = gst_collect_pads_add_pad (priv->cpads, pad, sizeof (GstCollectData), NULL, - TRUE); + *data = + gst_collect_pads_add_pad (priv->cpads, pad, sizeof (GstCollectData), + NULL, TRUE); if (NULL == *data) { GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); goto free_pad; @@ -231,19 +326,19 @@ gst_video_inference_create_pad (GstVideoInference * self, goto remove_pad; } - return GST_PAD_CAST(gst_object_ref (pad)); + return GST_PAD_CAST (gst_object_ref (pad)); - remove_pad: +remove_pad: gst_collect_pads_remove_pad (priv->cpads, pad); - free_pad: +free_pad: gst_object_unref (pad); return NULL; } static GstPad * -gst_video_inference_request_new_pad (GstElement *element, - GstPadTemplate *templ, const gchar* name, const GstCaps *caps) +gst_video_inference_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps) { GstVideoInference *self = GST_VIDEO_INFERENCE (element); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -280,13 +375,15 @@ gst_video_inference_request_new_pad (GstElement *element, } static void -gst_video_inference_release_pad (GstElement *element, GstPad *pad) +gst_video_inference_release_pad (GstElement * element, GstPad * pad) { GstVideoInference *self = GST_VIDEO_INFERENCE (element); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstPad **ourpad; GstCollectData **data; + GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); + if (pad == priv->sink_bypass) { ourpad = &priv->sink_bypass; data = &priv->sink_bypass_data; @@ -303,8 +400,6 @@ gst_video_inference_release_pad (GstElement *element, GstPad *pad) g_return_if_reached (); } - GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); - if (GST_PAD_IS_SINK (pad)) { *data = NULL; gst_collect_pads_remove_pad (priv->cpads, pad); @@ -314,8 +409,8 @@ gst_video_inference_release_pad (GstElement *element, GstPad *pad) } static GstFlowReturn -gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePrivate *priv, - GstCollectData *data, GstPad *pad) +gst_video_inference_forward_buffer (GstVideoInference * self, + GstVideoInferencePrivate * priv, GstCollectData * data, GstPad * pad) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstBuffer *buffer = NULL; @@ -333,7 +428,8 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr buffer = gst_collect_pads_pop (priv->cpads, data); if (NULL == buffer) { - GST_INFO_OBJECT (self, "EOS requested on %s:%s", GST_DEBUG_PAD_NAME (data->pad)); + GST_INFO_OBJECT (self, "EOS requested on %s:%s", + GST_DEBUG_PAD_NAME (data->pad)); ret = GST_FLOW_EOS; goto out; } @@ -342,7 +438,9 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr /* Allocate a new buffer for the preprocessing stage */ GstAllocationParams params; gsize size = gst_buffer_get_size (buffer); - GstBufferCopyFlags flags = (GstBufferCopyFlags)(GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS | GST_BUFFER_COPY_META); + GstBufferCopyFlags flags = + (GstBufferCopyFlags) (GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS + | GST_BUFFER_COPY_META); gst_allocation_params_init (¶ms); prepared = gst_buffer_new_allocate (NULL, size, ¶ms); @@ -361,9 +459,8 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr GST_ERROR_OBJECT (self, "Subclass failed preprocess"); goto suberror; } - // TODO: Do processing here - predsize = 1000*sizeof (float); + predsize = 1000 * sizeof (float); prediction = g_malloc (predsize); if (NULL != klass->postprocess) { @@ -379,53 +476,55 @@ gst_video_inference_forward_buffer (GstVideoInference *self, GstVideoInferencePr } if (NULL != pad) { - GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", + GST_DEBUG_PAD_NAME (pad)); ret = gst_pad_push (pad, prepared); } else { - GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", GST_DEBUG_PAD_NAME (data->pad)); + GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", + GST_DEBUG_PAD_NAME (data->pad)); gst_buffer_unref (prepared); goto out; } if (GST_FLOW_OK != ret) { - GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", GST_DEBUG_PAD_NAME (pad), ret, - gst_flow_get_name (ret)); + GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", + GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); } - out: +out: return ret; - suberror: +suberror: gst_buffer_unref (prepared); return GST_FLOW_ERROR; } static GstFlowReturn -gst_video_inference_collected (GstCollectPads *pads, - gpointer user_data) +gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) { GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstFlowReturn ret; - ret = gst_video_inference_forward_buffer(self, priv, priv->sink_bypass_data, + ret = gst_video_inference_forward_buffer (self, priv, priv->sink_bypass_data, priv->src_bypass); if (GST_FLOW_OK != ret) { goto out; } - ret = gst_video_inference_forward_buffer(self, priv, priv->sink_model_data, + ret = gst_video_inference_forward_buffer (self, priv, priv->sink_model_data, priv->src_model); if (GST_FLOW_OK != ret) { goto out; } - out: +out: return ret; } static GstPad * -gst_video_inference_get_src_pad (GstVideoInference *self, GstVideoInferencePrivate *priv, GstPad *sinkpad) +gst_video_inference_get_src_pad (GstVideoInference * self, + GstVideoInferencePrivate * priv, GstPad * sinkpad) { GstPad *pad; @@ -441,8 +540,8 @@ gst_video_inference_get_src_pad (GstVideoInference *self, GstVideoInferencePriva } static gboolean -gst_video_inference_sink_event (GstCollectPads *pads, GstCollectData *pad, - GstEvent *event, gpointer user_data) +gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, + GstEvent * event, gpointer user_data) { GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -455,20 +554,20 @@ gst_video_inference_sink_event (GstCollectPads *pads, GstCollectData *pad, gst_event_ref (event); if (FALSE == gst_pad_push_event (srcpad, event)) { - GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", GST_EVENT_TYPE_NAME (event), - GST_DEBUG_PAD_NAME (srcpad)); + GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); goto out; } ret = gst_collect_pads_event_default (priv->cpads, pad, event, FALSE); - out: +out: return ret; } static gboolean -gst_video_inference_src_event (GstPad *pad, GstObject *parent, - GstEvent *event) +gst_video_inference_src_event (GstPad * pad, GstObject * parent, + GstEvent * event) { GstVideoInference *self = GST_VIDEO_INFERENCE (parent); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -477,7 +576,7 @@ gst_video_inference_src_event (GstPad *pad, GstObject *parent, } static void -gst_video_inference_dispose (GObject * object) +gst_video_inference_finalize (GObject * object) { GstVideoInference *self = GST_VIDEO_INFERENCE (object); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -490,4 +589,8 @@ gst_video_inference_dispose (GObject * object) priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; + + g_clear_object (&priv->backend); + + G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); } From 92268bb49546d00d773fc7c0bae8e27ddab21f8b Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Wed, 31 Oct 2018 18:23:23 -0600 Subject: [PATCH 16/49] Add Backend Parameters Inspector --- gst-libs/gst/r2inference/Makefile.am | 4 + gst-libs/gst/r2inference/gstchildinspector.c | 206 ++++++++++++++++++ gst-libs/gst/r2inference/gstchildinspector.h | 27 +++ .../gst/r2inference/gstinferencebackends.cc | 120 ++++++++++ .../gst/r2inference/gstinferencebackends.h | 36 +++ 5 files changed, 393 insertions(+) create mode 100644 gst-libs/gst/r2inference/gstchildinspector.c create mode 100644 gst-libs/gst/r2inference/gstchildinspector.h create mode 100644 gst-libs/gst/r2inference/gstinferencebackends.cc create mode 100644 gst-libs/gst/r2inference/gstinferencebackends.h diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 858219a4..d0e4b564 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -3,6 +3,8 @@ lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstvideoinference.cc \ + gstchildinspector.c \ + gstinferencebackends.cc \ gstbackend.cc \ gstncsdk.cc @@ -30,5 +32,7 @@ gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ gstinferenceinclude_HEADERS= \ gstvideoinference.h \ + gstchildinspector.h \ + gstinferencebackends.h \ gstbackend.h \ gstncsdk.h diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c new file mode 100644 index 00000000..546e87e6 --- /dev/null +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2016 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include "gstchildinspector.h" + +typedef struct _GstChildInspectorFlag GstChildInspectorFlag; +typedef struct _GstChildInspectorType GstChildInspectorType; + +typedef gchar *(*GstChildInspectorTypeToString) (GParamSpec * pspec, + GValue * value); + +static gchar *gst_child_inspector_type_int_to_string (GParamSpec * pspec, + GValue * value); +static gchar *gst_child_inspector_type_string_to_string (GParamSpec * pspec, + GValue * value); + +struct _GstChildInspectorFlag +{ + gint value; + const gchar *to_string; +}; + +struct _GstChildInspectorType +{ + GType value; + GstChildInspectorTypeToString to_string; +}; + +static GstChildInspectorFlag flags[] = { + {G_PARAM_READABLE, "readable"}, + {G_PARAM_WRITABLE, "writable"}, + {} +}; + +static GstChildInspectorType types[] = { + {G_TYPE_INT, gst_child_inspector_type_int_to_string}, + {G_TYPE_STRING, gst_child_inspector_type_string_to_string}, + {} +}; + +static gchar * +gst_child_inspector_type_string_to_string (GParamSpec * pspec, GValue * value) +{ + return g_strdup_printf ("String. Default: \"%s\"", + g_value_get_string (value)); +} + +static gchar * +gst_child_inspector_type_int_to_string (GParamSpec * pspec, GValue * value) +{ + GParamSpecInt *pint = G_PARAM_SPEC_INT (pspec); + + return g_strdup_printf ("Integer. Range: %d - %d Default: %d", + pint->minimum, pint->maximum, g_value_get_int (value)); +} + +static const gchar * +gst_child_inspector_flag_to_string (GParamFlags flag) +{ + GstChildInspectorFlag *current_flag; + const gchar *to_string = NULL; + + for (current_flag = flags; current_flag->to_string; ++current_flag) { + if (current_flag->value == flag) { + to_string = current_flag->to_string; + break; + } + } + + return to_string; +} + +static gchar * +gst_child_inspector_flags_to_string (GParamFlags flags) +{ + guint32 bitflags; + gint i; + gchar *sflags = NULL; + + /* Walk through all the bits in the flags */ + bitflags = flags; + for (i = 31; i >= 0; --i) { + const gchar *sflag = NULL; + guint32 bitflag = 0; + + /* Filter the desired bit */ + bitflag = bitflags & (1 << i); + sflag = gst_child_inspector_flag_to_string (bitflag); + + /* Is this a flag we want to serialize? */ + if (sflag) { + /* No trailing comma needed on the first case */ + if (!sflags) { + sflags = g_strdup (sflag); + } else { + sflags = g_strdup_printf ("%s, %s", sflag, sflags); + } + } + } + + return sflags; +} + +static gchar * +gst_child_inspector_type_to_string (GParamSpec * pspec, GValue * value) +{ + GstChildInspectorType *current_type; + const GType value_type = G_VALUE_TYPE (value); + gchar *to_string = NULL; + + for (current_type = types; current_type->to_string; ++current_type) { + if (current_type->value == value_type) { + to_string = current_type->to_string (pspec, value); + break; + } + } + + return to_string; +} + +gchar * +gst_child_inspector_property_to_string (GObject * object, GParamSpec * param, + guint alignment) +{ + GValue value = { 0, }; + const gchar *name; + const gchar *blurb; + gchar *flags = NULL; + gchar *type; + gchar *prop; + + g_return_val_if_fail (param, NULL); + g_return_val_if_fail (G_IS_OBJECT (object), NULL); + + name = g_param_spec_get_name (param); + blurb = g_param_spec_get_blurb (param); + + flags = gst_child_inspector_flags_to_string (param->flags); + + g_value_init (&value, param->value_type); + if (param->flags & G_PARAM_READABLE) { + g_object_get_property (object, param->name, &value); + } else { + g_param_value_set_default (param, &value); + } + + type = gst_child_inspector_type_to_string (param, &value); + g_value_unset (&value); + + prop = g_strdup_printf ("%*s%-20s: %s\n" + "%*s%-21.21s flags: %s\n" + "%*s%-21.21s %s", + alignment, "", name, blurb, + alignment, "", "", flags, alignment, "", "", type); + + g_free (type); + g_free (flags); + + return prop; +} + + +gchar * +gst_child_inspector_properties_to_string (GObject * object, guint alignment, + gchar * title) +{ + GParamSpec **property_specs; + guint num_properties, i; + gchar *prop, *props = NULL; + + g_return_val_if_fail (G_IS_OBJECT (object), NULL); + + property_specs = g_object_class_list_properties + (G_OBJECT_GET_CLASS (object), &num_properties); + + if (title) + props = title; + + for (i = 0; i < num_properties; i++) { + prop = + gst_child_inspector_property_to_string (object, property_specs[i], + alignment); + if (props) { + gchar *tmp; + tmp = g_strdup_printf ("%s\n%s", props, prop); + g_free (prop); + g_free (props); + props = tmp; + } else { + props = prop; + } + } + + g_free (property_specs); + + return props; +} diff --git a/gst-libs/gst/r2inference/gstchildinspector.h b/gst-libs/gst/r2inference/gstchildinspector.h new file mode 100644 index 00000000..54c4bfd0 --- /dev/null +++ b/gst-libs/gst/r2inference/gstchildinspector.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2016 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#ifndef __GST_CHILD_INSPECTOR_H__ +#define __GST_CHILD_INSPECTOR_H__ + +#include + +G_BEGIN_DECLS + +gchar * gst_child_inspector_property_to_string (GObject * object, + GParamSpec * param, guint alignment); +gchar * gst_child_inspector_properties_to_string (GObject * object, + guint alignment, gchar * title); + +G_END_DECLS + +#endif /* __GST_CHILD_INSPECTOR_H__ */ diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc new file mode 100644 index 00000000..60c0f088 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -0,0 +1,120 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstinferencebackends.h" +#include "gstchildinspector.h" +#include "gstncsdk.h" +#include "gstbackend.h" +#include +#include +#include + +#define DEFAULT_ALIGNMENT 32 + +static std::unordered_map < int, GType > +backend_types ( + { + { + r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, { + r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} + }); + +static void +gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, + gchar ** backends_parameters, r2i::RuntimeError error, + unsigned int alignment); +static GType +gst_inference_backends_search_type (r2i::FrameworkCode id); + +GType +gst_inference_backends_get_type (void) +{ + static GType backend_type = 0; + static const GEnumValue + backend_desc[] = { + {0, "NCSDK Backend", "NCSDK"}, + {0, NULL, NULL} + }; + if (!backend_type) { + backend_type = + g_enum_register_static ("Backends", (GEnumValue *) backend_desc); + } + return backend_type; +} + +static GType +gst_inference_backends_search_type (r2i::FrameworkCode id) +{ + auto search = backend_types.find (id); + if (backend_types.end () == search) { + search = backend_types.find (r2i::FrameworkCode::MAX_FRAMEWORK); + } + return search->second; +} + +static void +gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, + gchar ** backends_parameters, r2i::RuntimeError error, + unsigned int alignment) +{ + GstBackend * backend = NULL; + gchar * parameters, * backend_name; + GType backend_type; + + backend_type = gst_inference_backends_search_type (meta.code); + + if (G_TYPE_INVALID == backend_type) { + GST_ERROR_OBJECT (backend, "Failed to find Backend type: %s", + meta.name.c_str ()); + return; + } + + backend = (GstBackend *) g_object_new (backend_type, NULL); + + backend_name = + g_strdup_printf ("%*s: %s. Version: %s\n", alignment, meta.name.c_str (), + meta.description.c_str (), meta.version.c_str ()); + + parameters = + gst_child_inspector_properties_to_string (G_OBJECT (backend), alignment, + backend_name); + + if (NULL == *backends_parameters) + *backends_parameters = parameters; + else + *backends_parameters = g_strconcat (*backends_parameters, parameters, NULL); + + g_object_unref (backend); +} + +gchar * +gst_inference_backends_get_string_properties (void) +{ + gchar * backends_parameters = NULL; + r2i::RuntimeError error; + + for (auto & meta:r2i::IFrameworkFactory::List (error)) { + gst_inference_backends_add_frameworkmeta (meta, &backends_parameters, error, + DEFAULT_ALIGNMENT); + } + + return backends_parameters; +} diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h new file mode 100644 index 00000000..33e71f61 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -0,0 +1,36 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_BACKENDS_H__ +#define __GST_INFERENCE_BACKENDS_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_INFERENCE_BACKENDS (gst_inference_backends_get_type()) + +GType gst_inference_backends_get_type(void); +gchar * gst_inference_backends_get_string_properties (void); + +G_END_DECLS + +#endif //__GST_INFERENCE_BACKENDS_H__ From 0fe4caf85c7ee57efa5acc40c3e1a9725cdd8ab4 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Wed, 31 Oct 2018 18:28:31 -0600 Subject: [PATCH 17/49] Add backend specific properties to VideoInference --- ext/r2inference/gstgooglenet.c | 1 + gst-libs/gst/r2inference/gstbackend.cc | 16 ++--- gst-libs/gst/r2inference/gstbackend.h | 11 ++- .../gst/r2inference/gstinferencebackends.cc | 20 +++--- gst-libs/gst/r2inference/gstncsdk.cc | 2 +- gst-libs/gst/r2inference/gstvideoinference.cc | 70 ++++++++++++++++--- 6 files changed, 84 insertions(+), 36 deletions(-) diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index ca0711da..f84a0abd 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -30,6 +30,7 @@ #endif #include "gstgooglenet.h" +#include GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); #define GST_CAT_DEFAULT gst_googlenet_debug_category diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 38930a1c..90ee1bd3 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -29,13 +29,12 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); typedef struct _GstBackendPrivate GstBackendPrivate; struct _GstBackendPrivate { - std::unique_ptr params; + std::unique_ptr < r2i::IParameters > params; }; -G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, GST_TYPE_OBJECT, +G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, GST_DEBUG_CATEGORY_INIT (gst_backend_debug_category, "backend", 0, - "debug category for backend parameters"); - G_ADD_PRIVATE (GstBackend)); + "debug category for backend parameters"); G_ADD_PRIVATE (GstBackend)); #define GST_BACKEND_PRIVATE(self) \ (GstBackendPrivate *)(gst_backend_get_instance_private (self)) @@ -69,7 +68,7 @@ gst_backend_install_properties (GstBackendClass * klass) { GObjectClass *oclass = G_OBJECT_CLASS (klass); r2i::RuntimeError error; - std::vector < r2i::ParameterMeta > params; + static std::vector < r2i::ParameterMeta > params; static gint nprop = 1; klass->props = g_hash_table_new (g_direct_hash, g_direct_equal); @@ -169,8 +168,9 @@ gst_backend_get_property (GObject * object, guint property_id, } gboolean -gst_backend_configure (GstBackend *self, std::shared_ptr engine, - std::shared_ptr model) +gst_backend_configure (GstBackend * self, + std::shared_ptr < r2i::IEngine > engine, + std::shared_ptr < r2i::IModel > model) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; @@ -178,7 +178,7 @@ gst_backend_configure (GstBackend *self, std::shared_ptr engine, error = priv->params->Configure (engine, model); if (error.IsError ()) { GST_ERROR_OBJECT (self, "Error configuring backend parameters: (%d): %s", - error.GetCode (), error.GetDescription ().c_str()); + error.GetCode (), error.GetDescription ().c_str ()); return FALSE; } diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 8c6158ca..5a66cc20 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -26,21 +26,20 @@ #include G_BEGIN_DECLS - #define GST_TYPE_BACKEND gst_backend_get_type () -G_DECLARE_DERIVABLE_TYPE (GstBackend, gst_backend, GST, BACKEND, GstObject); +G_DECLARE_DERIVABLE_TYPE (GstBackend, gst_backend, GST, BACKEND, GObject); struct _GstBackendClass { - GstObjectClass parent_class; + GObjectClass parent_class; r2i::FrameworkCode backend; GHashTable *props; }; -gboolean gst_backend_configure (GstBackend *self, - std::shared_ptr engine, std::shared_ptr model); +gboolean gst_backend_configure (GstBackend * self, + std::shared_ptr < r2i::IEngine > engine, + std::shared_ptr < r2i::IModel > model); G_END_DECLS - #endif //__GST_BACKEND_H__ diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index 60c0f088..ed4a7ddc 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -29,18 +29,16 @@ #define DEFAULT_ALIGNMENT 32 -static std::unordered_map < int, GType > -backend_types ( - { - { - r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, { - r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} - }); +static std::unordered_map +backend_types ({ + {r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, + {r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} +}); static void gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, gchar ** backends_parameters, r2i::RuntimeError error, - unsigned int alignment); + guint alignment); static GType gst_inference_backends_search_type (r2i::FrameworkCode id); @@ -50,12 +48,12 @@ gst_inference_backends_get_type (void) static GType backend_type = 0; static const GEnumValue backend_desc[] = { - {0, "NCSDK Backend", "NCSDK"}, + {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, {0, NULL, NULL} }; if (!backend_type) { backend_type = - g_enum_register_static ("Backends", (GEnumValue *) backend_desc); + g_enum_register_static ("GstInferenceBackends", (GEnumValue *) backend_desc); } return backend_type; } @@ -73,7 +71,7 @@ gst_inference_backends_search_type (r2i::FrameworkCode id) static void gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, gchar ** backends_parameters, r2i::RuntimeError error, - unsigned int alignment) + guint alignment) { GstBackend * backend = NULL; gchar * parameters, * backend_name; diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc index c079d825..a2f8948f 100644 --- a/gst-libs/gst/r2inference/gstncsdk.cc +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -28,7 +28,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_ncsdk_debug_category); struct _GstNcsdk { - GstObject parent; + GstBackend parent; }; G_DEFINE_TYPE_WITH_CODE (GstNcsdk, gst_ncsdk, GST_TYPE_BACKEND, diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index b2ae39d5..1e720031 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -20,6 +20,7 @@ */ #include "gstvideoinference.h" +#include "gstinferencebackends.h" #include #include @@ -43,6 +44,15 @@ GST_STATIC_PAD_TEMPLATE ("src_bypass", GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category +#define DEFAULT_BACKEND 0 + +enum +{ + PROP_0, + PROP_BACKEND +}; + + typedef struct _GstVideoInferencePrivate GstVideoInferencePrivate; struct _GstVideoInferencePrivate { @@ -57,13 +67,17 @@ struct _GstVideoInferencePrivate GstBackend *backend; - std::shared_ptr engine; - std::shared_ptr loader; - std::shared_ptr model; + std::shared_ptr < r2i::IEngine > engine; + std::shared_ptr < r2i::ILoader > loader; + std::shared_ptr < r2i::IModel > model; }; /* GObject methods */ static void gst_video_inference_finalize (GObject * object); +static void gst_video_inference_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_video_inference_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); /* GstElement methods */ static GstStateChangeReturn gst_video_inference_change_state (GstElement * @@ -114,8 +128,11 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) { GObjectClass *oclass = G_OBJECT_CLASS (klass); GstElementClass *eclass = GST_ELEMENT_CLASS (klass); + gchar *backend_blurb, *backends_params = NULL; oclass->finalize = gst_video_inference_finalize; + oclass->set_property = gst_video_inference_set_property; + oclass->get_property = gst_video_inference_get_property; eclass->change_state = GST_DEBUG_FUNCPTR (gst_video_inference_change_state); eclass->request_new_pad = @@ -124,6 +141,22 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); + backends_params = gst_inference_backends_get_string_properties (); + backend_blurb = g_strdup_printf ("Type of predefined backend to use.\n" + "\t\t\tAccording to the selected backend " + "different properties will be available.\n " + "\t\t\tThese properties can be accessed using the " + "\"backend::\" syntax.\n" + "\t\t\tThe following list details the properties " + "for each backend\n%s", backends_params); + + g_free (backends_params); + + g_object_class_install_property (oclass, PROP_BACKEND, + g_param_spec_enum ("backend", "Backend", + backend_blurb, GST_TYPE_INFERENCE_BACKENDS, DEFAULT_BACKEND, + G_PARAM_READWRITE)); + klass->start = NULL; klass->stop = NULL; klass->preprocess = NULL; @@ -154,11 +187,28 @@ gst_video_inference_init (GstVideoInference * self) priv->backend = GST_BACKEND (g_object_new (GST_TYPE_NCSDK, NULL)); - priv->engine = factory->MakeEngine(error); - priv->loader = factory->MakeLoader(error); + priv->engine = factory->MakeEngine (error); + priv->loader = factory->MakeLoader (error); priv->model = nullptr; } +static void +gst_video_inference_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (object); + GST_LOG_OBJECT (klass, "Set Property"); +} + +static void +gst_video_inference_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (object); + GST_LOG_OBJECT (klass, "Get Property"); +} + + static void gst_video_inference_child_proxy_init (GstChildProxyInterface * iface) { @@ -223,12 +273,12 @@ gst_video_inference_start (GstVideoInference * self) ret = FALSE; goto out; } - + if (klass->start != NULL) { ret = klass->start (self); } - out: +out: return ret; } @@ -273,8 +323,8 @@ gst_video_inference_change_state (GstElement * element, } ret = - GST_ELEMENT_CLASS (gst_video_inference_parent_class)-> - change_state (element, transition); + GST_ELEMENT_CLASS (gst_video_inference_parent_class)->change_state + (element, transition); if (GST_STATE_CHANGE_FAILURE == ret) { GST_ERROR_OBJECT (self, "Parent failed to change state"); goto out; @@ -383,7 +433,7 @@ gst_video_inference_release_pad (GstElement * element, GstPad * pad) GstCollectData **data; GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); - + if (pad == priv->sink_bypass) { ourpad = &priv->sink_bypass; data = &priv->sink_bypass_data; From cf7970ddaf8f2ae49774b8435de0d0ad3269ec86 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Thu, 8 Nov 2018 11:57:39 -0600 Subject: [PATCH 18/49] Change License to LGPL --- ext/r2inference/gstgooglenet.c | 25 ++++++++++++------ ext/r2inference/gstgooglenet.h | 27 +++++++++++++------- ext/r2inference/gstinference.c | 27 +++++++++++++------- gst-libs/gst/r2inference/gstchildinspector.c | 27 +++++++++++++------- gst-libs/gst/r2inference/gstchildinspector.h | 27 +++++++++++++------- 5 files changed, 89 insertions(+), 44 deletions(-) diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index f84a0abd..dbea8a8b 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -1,13 +1,22 @@ /* - * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. */ /** diff --git a/ext/r2inference/gstgooglenet.h b/ext/r2inference/gstgooglenet.h index c0ded587..000e9cd1 100644 --- a/ext/r2inference/gstgooglenet.h +++ b/ext/r2inference/gstgooglenet.h @@ -1,13 +1,22 @@ -/* - * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. */ #ifndef _GST_GOOGLENET_H_ diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index b66a71d7..511b2b7f 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -1,13 +1,22 @@ -/* - * Copyright (C) 2017 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c index 546e87e6..932c69fd 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.c +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -1,13 +1,22 @@ -/* - * Copyright (C) 2016 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. */ #include "gstchildinspector.h" diff --git a/gst-libs/gst/r2inference/gstchildinspector.h b/gst-libs/gst/r2inference/gstchildinspector.h index 54c4bfd0..28392601 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.h +++ b/gst-libs/gst/r2inference/gstchildinspector.h @@ -1,13 +1,22 @@ -/* - * Copyright (C) 2016 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. */ #ifndef __GST_CHILD_INSPECTOR_H__ From e5993a151c9790d2207683091ad98d5743183478 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Thu, 8 Nov 2018 15:46:29 -0600 Subject: [PATCH 19/49] Add Model Location Property --- gst-libs/gst/r2inference/gstvideoinference.cc | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 1e720031..738d48d1 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -45,11 +45,13 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_BACKEND 0 +#define DEFAULT_MODEL_LOCATION nullptr enum { PROP_0, - PROP_BACKEND + PROP_BACKEND, + PROP_MODEL_LOCATION }; @@ -70,6 +72,8 @@ struct _GstVideoInferencePrivate std::shared_ptr < r2i::IEngine > engine; std::shared_ptr < r2i::ILoader > loader; std::shared_ptr < r2i::IModel > model; + + gchar *model_location; }; /* GObject methods */ @@ -157,6 +161,11 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) backend_blurb, GST_TYPE_INFERENCE_BACKENDS, DEFAULT_BACKEND, G_PARAM_READWRITE)); + g_object_class_install_property (oclass, PROP_MODEL_LOCATION, + g_param_spec_string ("model-location", "Model Location", + "Path to the model to use", DEFAULT_MODEL_LOCATION, + G_PARAM_READWRITE)); + klass->start = NULL; klass->stop = NULL; klass->preprocess = NULL; @@ -190,22 +199,53 @@ gst_video_inference_init (GstVideoInference * self) priv->engine = factory->MakeEngine (error); priv->loader = factory->MakeLoader (error); priv->model = nullptr; + + priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); } static void gst_video_inference_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (object); - GST_LOG_OBJECT (klass, "Set Property"); + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_LOG_OBJECT (self, "Set Property"); + + switch (property_id) { + case PROP_BACKEND: + break; + case PROP_MODEL_LOCATION: + if (priv->model_location) { + g_free (priv->model_location); + } + priv->model_location = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } } static void gst_video_inference_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (object); - GST_LOG_OBJECT (klass, "Get Property"); + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_LOG_OBJECT (self, "Get Property"); + + switch (property_id) { + case PROP_BACKEND: + break; + case PROP_MODEL_LOCATION: + g_value_set_string (value, priv->model_location); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } } @@ -260,13 +300,13 @@ static gboolean gst_video_inference_start (GstVideoInference * self) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); - //GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); r2i::RuntimeError error; gboolean ret = TRUE; GST_INFO_OBJECT (self, "Starting video inference"); - //priv->model = priv->loader->Load ("/path/to/model", error); + priv->model = priv->loader->Load (priv->model_location, error); if (error.IsError ()) { GST_ERROR_OBJECT (self, "Unable to load model: (%d) %s", error.GetCode (), error.GetDescription ().c_str ()); @@ -639,6 +679,9 @@ gst_video_inference_finalize (GObject * object) priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; + if (priv->model_location) { + g_free (priv->model_location); + } g_clear_object (&priv->backend); From 0c917d7b128bae818827ee5429c8fc7cf3d88b31 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Thu, 8 Nov 2018 17:10:38 -0600 Subject: [PATCH 20/49] Change Model Location only in NULL state This avoids the model location to be changed when the model was already loaded. --- gst-libs/gst/r2inference/gstvideoinference.cc | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index 738d48d1..f39a4d67 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -45,7 +45,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_BACKEND 0 -#define DEFAULT_MODEL_LOCATION nullptr +#define DEFAULT_MODEL_LOCATION NULL enum { @@ -69,9 +69,9 @@ struct _GstVideoInferencePrivate GstBackend *backend; - std::shared_ptr < r2i::IEngine > engine; - std::shared_ptr < r2i::ILoader > loader; - std::shared_ptr < r2i::IModel > model; + std::shared_ptr < r2i::IEngine > engine; + std::shared_ptr < r2i::ILoader > loader; + std::shared_ptr < r2i::IModel > model; gchar *model_location; }; @@ -216,10 +216,16 @@ gst_video_inference_set_property (GObject * object, case PROP_BACKEND: break; case PROP_MODEL_LOCATION: - if (priv->model_location) { - g_free (priv->model_location); + GstState actual_state; + gst_element_get_state (GST_ELEMENT(self), &actual_state, NULL, GST_SECOND); + GST_OBJECT_LOCK (self); + if (actual_state <= GST_STATE_READY) { + g_free (priv->model_location); + priv->model_location = g_value_dup_string (value); + } else { + GST_ERROR_OBJECT (self, "Model location can only be set in the NULL or READY states"); } - priv->model_location = g_value_dup_string (value); + GST_OBJECT_UNLOCK (self); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -679,9 +685,8 @@ gst_video_inference_finalize (GObject * object) priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; - if (priv->model_location) { - g_free (priv->model_location); - } + g_free (priv->model_location); + priv->model_location = NULL; g_clear_object (&priv->backend); From 27e8dd8e4fdd0d15548a9b2bcd1d1d5d91561519 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 9 Nov 2018 11:15:44 -0600 Subject: [PATCH 21/49] Start R2Inference Engine and Model The Engine, Loader and Model instances are created and the Model is loaded into the Engine to start processing frames. --- gst-libs/gst/r2inference/gstvideoinference.cc | 55 ++++++++++++++----- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index f39a4d67..b8c24683 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -176,9 +176,6 @@ static void gst_video_inference_init (GstVideoInference * self) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - r2i::RuntimeError error; - auto factory = r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, - error); priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; @@ -194,11 +191,7 @@ gst_video_inference_init (GstVideoInference * self) gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, (gpointer) (self)); - priv->backend = GST_BACKEND (g_object_new (GST_TYPE_NCSDK, NULL)); - - priv->engine = factory->MakeEngine (error); - priv->loader = factory->MakeLoader (error); - priv->model = nullptr; + // priv->backend = GST_BACKEND (g_object_new (GST_TYPE_NCSDK, NULL)); priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); } @@ -307,25 +300,58 @@ gst_video_inference_start (GstVideoInference * self) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - r2i::RuntimeError error; gboolean ret = TRUE; + r2i::RuntimeError error; + + /* TODO: Avoid hardcoded NCSDK, it should be queried from the backend */ + auto factory = r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, + error); GST_INFO_OBJECT (self, "Starting video inference"); - priv->model = priv->loader->Load (priv->model_location, error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Unable to load model: (%d) %s", error.GetCode (), - error.GetDescription ().c_str ()); + if (NULL == priv->model_location) { + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("Model Location has not been set"), (NULL)); ret = FALSE; goto out; } + priv->engine = factory->MakeEngine (error); + if (error.IsError ()) { + goto error; + } + + priv->loader = factory->MakeLoader (error); + if (error.IsError ()) { + goto error; + } + + priv->model = priv->loader->Load (priv->model_location, error); + if (error.IsError ()) { + goto error; + } + + error = priv->engine->SetModel (priv->model); + if (error.IsError ()) { + goto error; + } + + error = priv->engine->Start (); + if (error.IsError ()) { + goto error; + } + if (klass->start != NULL) { ret = klass->start (self); } -out: + out: return ret; + error: + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("R2Inference Error: (Code:%d) %s", error.GetCode(), + error.GetDescription ().c_str()), (NULL)); + return FALSE; } static gboolean @@ -516,7 +542,6 @@ gst_video_inference_forward_buffer (GstVideoInference * self, gpointer prediction; gsize predsize; - /* User didn't request this pad */ if (NULL == data) { goto out; From 771b24f293ad0b65d05e332a4d0f32f3a594bba6 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 9 Nov 2018 13:43:49 -0600 Subject: [PATCH 22/49] Forward event only when the src pad was requested --- gst-libs/gst/r2inference/gstvideoinference.cc | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index b8c24683..ad6f185b 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -671,13 +671,20 @@ gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, srcpad = gst_video_inference_get_src_pad (self, priv, pad->pad); - /* Collect pads will decrease the refcount of the event when we return */ - gst_event_ref (event); - - if (FALSE == gst_pad_push_event (srcpad, event)) { - GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", - GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); - goto out; + if (NULL != srcpad) { + GST_LOG_OBJECT (self, "Forwarding event %s from %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); + /* Collect pads will decrease the refcount of the event when we return */ + gst_event_ref (event); + if (FALSE == gst_pad_push_event (srcpad, event)) { + GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); + goto out; + } + } + else { + GST_LOG_OBJECT (self, "Dropping event %s from %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); } ret = gst_collect_pads_event_default (priv->cpads, pad, event, FALSE); From b04cab35e914075a9e93534f2b99aa9ddeffc553 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 9 Nov 2018 15:17:06 -0600 Subject: [PATCH 23/49] Separate Bypass and Model Buffers Processing Model Processing function will do the Prediction, while Bypass Processink will attach the GstMetas to the bypass buffer. --- gst-libs/gst/r2inference/gstvideoinference.cc | 176 +++++++++++------- 1 file changed, 107 insertions(+), 69 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index ad6f185b..a2b17e4d 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -108,7 +108,11 @@ static GstPad *gst_video_inference_create_pad (GstVideoInference * self, static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * - self, GstVideoInferencePrivate * priv, GstCollectData * data, GstPad * pad); + self, GstBuffer * buffer, GstCollectData * data, GstPad * pad); +static GstBuffer * gst_video_inference_model_buffer_process (GstVideoInference * + self, GstBuffer * buffer); +static GstBuffer * gst_video_inference_bypass_buffer_process (GstVideoInference * + self, GstBuffer * buffer_bypass, GstBuffer * buffer_model); static gboolean gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, GstEvent * event, gpointer user_data); static gboolean gst_video_inference_src_event (GstPad * pad, GstObject * parent, @@ -191,8 +195,6 @@ gst_video_inference_init (GstVideoInference * self) gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, (gpointer) (self)); - // priv->backend = GST_BACKEND (g_object_new (GST_TYPE_NCSDK, NULL)); - priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); } @@ -247,7 +249,6 @@ gst_video_inference_get_property (GObject * object, } } - static void gst_video_inference_child_proxy_init (GstChildProxyInterface * iface) { @@ -532,92 +533,90 @@ gst_video_inference_release_pad (GstElement * element, GstPad * pad) static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, - GstVideoInferencePrivate * priv, GstCollectData * data, GstPad * pad) + GstBuffer * buffer, GstCollectData * data, GstPad * pad) { - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); - GstBuffer *buffer = NULL; GstFlowReturn ret = GST_FLOW_OK; - gboolean subret = FALSE; - GstBuffer *prepared; - gpointer prediction; - gsize predsize; /* User didn't request this pad */ if (NULL == data) { goto out; } - buffer = gst_collect_pads_pop (priv->cpads, data); - if (NULL == buffer) { - GST_INFO_OBJECT (self, "EOS requested on %s:%s", + if (NULL != pad) { + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", + GST_DEBUG_PAD_NAME (pad)); + ret = gst_pad_push (pad, buffer); + } else { + GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", GST_DEBUG_PAD_NAME (data->pad)); - ret = GST_FLOW_EOS; + gst_buffer_unref (buffer); goto out; } - if (data->pad == priv->sink_model && NULL != klass->preprocess) { + if (GST_FLOW_OK != ret) { + GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", + GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); + } + +out: + return ret; +} + +static GstBuffer * +gst_video_inference_model_buffer_process (GstVideoInference * self, + GstBuffer * buffer) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstBuffer *buffer_out = NULL; + GstBuffer *prepared = NULL; + gpointer prediction; + gsize predsize; + + if (NULL != klass->preprocess) { /* Allocate a new buffer for the preprocessing stage */ GstAllocationParams params; gsize size = gst_buffer_get_size (buffer); - GstBufferCopyFlags flags = - (GstBufferCopyFlags) (GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS - | GST_BUFFER_COPY_META); - gst_allocation_params_init (¶ms); prepared = gst_buffer_new_allocate (NULL, size, ¶ms); - gst_buffer_copy_into (prepared, buffer, flags, 0, size); - subret = klass->preprocess (self, buffer, prepared); + + if (!klass->preprocess (self, buffer, prepared)) { + GST_ERROR_OBJECT (self, "Subclass failed Preprocess"); + gst_buffer_unref (prepared); + goto out; + } } else { prepared = gst_buffer_ref (buffer); - subret = TRUE; } - /* This reference is no longer needed because the pre-process copied - it or now its called prepared */ - gst_buffer_unref (buffer); - - if (FALSE == subret) { - GST_ERROR_OBJECT (self, "Subclass failed preprocess"); - goto suberror; - } - // TODO: Do processing here + /* TODO: Create R2I Frame */ + /* TODO: Creat R2I Prediction */ predsize = 1000 * sizeof (float); prediction = g_malloc (predsize); + /* TODO: Predict using the Prepared Buffer */ if (NULL != klass->postprocess) { - subret = klass->postprocess (self, prepared, prediction, predsize); - } - - /* Prediction is no longer needed */ - g_free (prediction); - - if (FALSE == subret) { - GST_ERROR_OBJECT (self, "Subclass failed postprocess"); - goto suberror; - } - - if (NULL != pad) { - GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", - GST_DEBUG_PAD_NAME (pad)); - ret = gst_pad_push (pad, prepared); - } else { - GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", - GST_DEBUG_PAD_NAME (data->pad)); - gst_buffer_unref (prepared); - goto out; + if (!klass->postprocess (self, buffer, prediction, predsize)) { + GST_ERROR_OBJECT (self, "Subclass failed postprocess"); + gst_buffer_unref (prepared); + goto out; + } } - if (GST_FLOW_OK != ret) { - GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", - GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); - } + gst_buffer_unref (prepared); + buffer_out = buffer; out: - return ret; + return buffer_out; +} -suberror: - gst_buffer_unref (prepared); - return GST_FLOW_ERROR; +static GstBuffer * +gst_video_inference_bypass_buffer_process (GstVideoInference * self, + GstBuffer * buffer_bypass, GstBuffer * buffer_model) +{ + /* TODO: Get GstMetas from Model buffer and attach them to the + Bypass Buffer */ + + return buffer_bypass; } static GstFlowReturn @@ -625,18 +624,57 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) { GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - GstFlowReturn ret; + GstFlowReturn ret = GST_FLOW_OK; + GstBuffer * buffer_model; + GstBuffer * buffer_bypass; + + /* Process Model Buffer */ + if (priv->sink_model_data) { + buffer_model = gst_collect_pads_pop (priv->cpads, priv->sink_model_data); + if (NULL == buffer_model) { + GST_INFO_OBJECT (self, "EOS requested on %s:%s", + GST_DEBUG_PAD_NAME (priv->sink_model_data->pad)); + ret = GST_FLOW_EOS; + goto out; + } + buffer_model = gst_video_inference_model_buffer_process (self, buffer_model); + if (NULL == buffer_model) { + ret = GST_FLOW_ERROR; + goto out; + } + } - ret = gst_video_inference_forward_buffer (self, priv, priv->sink_bypass_data, - priv->src_bypass); - if (GST_FLOW_OK != ret) { - goto out; + /* Process Bypass Buffer */ + if (priv->sink_bypass_data) { + buffer_bypass = gst_collect_pads_pop (priv->cpads, priv->sink_bypass_data); + if (NULL == buffer_bypass) { + GST_INFO_OBJECT (self, "EOS requested on %s:%s", + GST_DEBUG_PAD_NAME (priv->sink_bypass_data->pad)); + ret = GST_FLOW_EOS; + goto out; + } + buffer_bypass = gst_video_inference_bypass_buffer_process (self, buffer_bypass, buffer_model); + if (NULL == buffer_bypass) { + ret = GST_FLOW_ERROR; + goto out; + } } - ret = gst_video_inference_forward_buffer (self, priv, priv->sink_model_data, - priv->src_model); - if (GST_FLOW_OK != ret) { - goto out; + /* Forward buffers to src pads */ + if (NULL != buffer_model) { + ret = gst_video_inference_forward_buffer (self, buffer_model, + priv->sink_model_data, priv->src_model); + if (GST_FLOW_OK != ret) { + goto out; + } + } + + if (NULL != buffer_bypass) { + ret = gst_video_inference_forward_buffer (self, buffer_bypass, + priv->sink_bypass_data, priv->src_bypass); + if (GST_FLOW_OK != ret) { + goto out; + } } out: From 73f7e051a09092de5e124361a3023ffdd4723626 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Wed, 5 Dec 2018 16:41:21 -0600 Subject: [PATCH 24/49] Add base GstTinyYolo element implementation --- ext/r2inference/Makefile.am | 6 +- ext/r2inference/gstinference.c | 17 ++- ext/r2inference/gsttinyyolo.c | 237 +++++++++++++++++++++++++++++++++ ext/r2inference/gsttinyyolo.h | 34 +++++ 4 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 ext/r2inference/gsttinyyolo.c create mode 100644 ext/r2inference/gsttinyyolo.h diff --git a/ext/r2inference/Makefile.am b/ext/r2inference/Makefile.am index 0ebfbc1c..b31a60b0 100644 --- a/ext/r2inference/Makefile.am +++ b/ext/r2inference/Makefile.am @@ -2,7 +2,8 @@ plugin_LTLIBRARIES = libgstinference.la libgstinference_la_SOURCES = \ gstgooglenet.c \ - gstinference.c + gstinference.c \ + gsttinyyolo.c libgstinference_la_CFLAGS = \ $(GST_CFLAGS) \ @@ -23,4 +24,5 @@ libgstinference_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ - gstgooglenet.h + gstgooglenet.h \ + gsttinyyolo.h diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index 511b2b7f..68b78baa 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -24,6 +24,7 @@ #endif #include "gstgooglenet.h" +#include "gsttinyyolo.h" static gboolean plugin_init (GstPlugin * plugin) @@ -31,8 +32,22 @@ plugin_init (GstPlugin * plugin) /* FIXME Remember to set the rank if it's an element that is meant to be autoplugged by decodebin. */ - return gst_element_register (plugin, "googlenet", GST_RANK_NONE, + gboolean ret = TRUE; + + ret = gst_element_register (plugin, "googlenet", GST_RANK_NONE, GST_TYPE_GOOGLENET); + if (!ret) { + goto out; + } + + ret = gst_element_register (plugin, "tinyyolo", GST_RANK_NONE, + GST_TYPE_TINYYOLO); + if (!ret) { + goto out; + } + + out: + return ret; } GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, diff --git a/ext/r2inference/gsttinyyolo.c b/ext/r2inference/gsttinyyolo.c new file mode 100644 index 00000000..3f03fee6 --- /dev/null +++ b/ext/r2inference/gsttinyyolo.c @@ -0,0 +1,237 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gsttinyyolo + * + * The tinyyolo element allows the user to infer/execute a pretrained model + * based on the TinyYolo architecture on incoming image frames. + * + * + * Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! tinyyolo ! xvimagesink + * ]| + * Process video frames from the camera using a TinyYolo model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gsttinyyolo.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tinyyolo_debug_category); +#define GST_CAT_DEFAULT gst_tinyyolo_debug_category + +/* prototypes */ + + +static void gst_tinyyolo_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_tinyyolo_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_tinyyolo_dispose (GObject * object); +static void gst_tinyyolo_finalize (GObject * object); + +static gboolean gst_tinyyolo_preprocess (GstVideoInference * vi, + GstBuffer * inbuf, GstBuffer * outbuf); +static gboolean gst_tinyyolo_postprocess (GstVideoInference * vi, + GstBuffer * buf, const gpointer prediction, gsize predsize); +static gboolean gst_tinyyolo_start (GstVideoInference * vi); +static gboolean gst_tinyyolo_stop (GstVideoInference * vi); + +enum +{ + PROP_0 +}; + +/* pad templates */ + +#define CAPS "video/x-raw,format=BGR,width=448,height=448" + +static GstStaticPadTemplate sink_model_factory = +GST_STATIC_PAD_TEMPLATE ("sink_model", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +static GstStaticPadTemplate src_model_factory = +GST_STATIC_PAD_TEMPLATE ("src_model", + GST_PAD_SRC, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +struct _GstTinyyolo +{ + GstVideoInference parent; +}; + +struct _GstTinyyoloClass +{ + GstVideoInferenceClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstTinyyolo, gst_tinyyolo, GST_TYPE_VIDEO_INFERENCE, + GST_DEBUG_CATEGORY_INIT (gst_tinyyolo_debug_category, "tinyyolo", 0, + "debug category for tinyyolo element")); + +static void +gst_tinyyolo_class_init (GstTinyyoloClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoInferenceClass *vi_class = GST_VIDEO_INFERENCE_CLASS (klass); + + gst_element_class_add_static_pad_template (element_class, + &sink_model_factory); + gst_element_class_add_static_pad_template (element_class, &src_model_factory); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "tinyyolo", "Filter", + "Infers incoming image frames using a pretrained TinyYolo model", + "Carlos Rodriguez \n\t\t\t" + "Jose Jimenez \n\t\t\t" + "Michael Gruner \n\t\t\t" + "Carlos Aguero \n\t\t\t" + "Miguel Taylor \n\t\t\t" + "Greivin Fallas "); + + gobject_class->set_property = gst_tinyyolo_set_property; + gobject_class->get_property = gst_tinyyolo_get_property; + gobject_class->dispose = gst_tinyyolo_dispose; + gobject_class->finalize = gst_tinyyolo_finalize; + + vi_class->start = GST_DEBUG_FUNCPTR (gst_tinyyolo_start); + vi_class->stop = GST_DEBUG_FUNCPTR (gst_tinyyolo_stop); + vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_tinyyolo_preprocess); + vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_tinyyolo_postprocess); +} + +static void +gst_tinyyolo_init (GstTinyyolo * tinyyolo) +{ +} + +void +gst_tinyyolo_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "set_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_tinyyolo_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "get_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_tinyyolo_dispose (GObject * object) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "dispose"); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS (gst_tinyyolo_parent_class)->dispose (object); +} + +void +gst_tinyyolo_finalize (GObject * object) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS (gst_tinyyolo_parent_class)->finalize (object); +} + +static gboolean +gst_tinyyolo_preprocess (GstVideoInference * vi, + GstBuffer * inbuf, GstBuffer * outbuf) +{ + GstMapInfo ininfo; + GstMapInfo outinfo; + + GST_LOG_OBJECT (vi, "Preprocess"); + + gst_buffer_map (inbuf, &ininfo, GST_MAP_READ); + gst_buffer_map (outbuf, &outinfo, GST_MAP_WRITE); + + memcpy (outinfo.data, ininfo.data, ininfo.size); + + gst_buffer_unmap (inbuf, &ininfo); + gst_buffer_unmap (outbuf, &outinfo); + + return TRUE; +} + +static gboolean +gst_tinyyolo_postprocess (GstVideoInference * vi, + GstBuffer * buf, const gpointer prediction, gsize predsize) +{ + GST_LOG_OBJECT (vi, "Postprocess"); + + return TRUE; +} + +static gboolean +gst_tinyyolo_start (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Starting TinyYolo"); + + return TRUE; +} + +static gboolean +gst_tinyyolo_stop (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Stopping TinyYolo"); + + return TRUE; +} diff --git a/ext/r2inference/gsttinyyolo.h b/ext/r2inference/gsttinyyolo.h new file mode 100644 index 00000000..fd36996f --- /dev/null +++ b/ext/r2inference/gsttinyyolo.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_TINYYOLO_H_ +#define _GST_TINYYOLO_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TINYYOLO gst_tinyyolo_get_type () +G_DECLARE_FINAL_TYPE (GstTinyyolo, gst_tinyyolo, GST, TINYYOLO, GstVideoInference) + +G_END_DECLS + +#endif /* _GST_TINYYOLO_H_ */ From 2c96aa45c40bd2dccc83536e61b8ffe7be7e5b80 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Wed, 5 Dec 2018 17:01:19 -0600 Subject: [PATCH 25/49] Fix indentation at Authors --- ext/r2inference/gsttinyyolo.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/r2inference/gsttinyyolo.c b/ext/r2inference/gsttinyyolo.c index 3f03fee6..f6fae39e 100644 --- a/ext/r2inference/gsttinyyolo.c +++ b/ext/r2inference/gsttinyyolo.c @@ -115,11 +115,11 @@ gst_tinyyolo_class_init (GstTinyyoloClass * klass) "tinyyolo", "Filter", "Infers incoming image frames using a pretrained TinyYolo model", "Carlos Rodriguez \n\t\t\t" - "Jose Jimenez \n\t\t\t" - "Michael Gruner \n\t\t\t" - "Carlos Aguero \n\t\t\t" - "Miguel Taylor \n\t\t\t" - "Greivin Fallas "); + " Jose Jimenez \n\t\t\t" + " Michael Gruner \n\t\t\t" + " Carlos Aguero \n\t\t\t" + " Miguel Taylor \n\t\t\t" + " Greivin Fallas "); gobject_class->set_property = gst_tinyyolo_set_property; gobject_class->get_property = gst_tinyyolo_get_property; From c3a8405c232b0280ff164897790ee316682fe94e Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Tue, 8 Jan 2019 23:17:29 +0000 Subject: [PATCH 26/49] Add inference to gstvideoinference class --- gst-libs/gst/r2inference/Makefile.am | 3 +- gst-libs/gst/r2inference/gstvideoinference.cc | 218 +++++++++++------- gst-libs/gst/r2inference/gstvideoinference.h | 5 +- 3 files changed, 144 insertions(+), 82 deletions(-) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index d0e4b564..e29ae8c5 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -24,7 +24,8 @@ libgstinference_@GST_API_VERSION@_la_CFLAGS= \ libgstinference_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ - $(GST_BASE_LIBS) \ + $(GST_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index a2b17e4d..dee0d5d7 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -21,7 +21,6 @@ #include "gstvideoinference.h" #include "gstinferencebackends.h" - #include #include @@ -109,9 +108,9 @@ static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, GstBuffer * buffer, GstCollectData * data, GstPad * pad); -static GstBuffer * gst_video_inference_model_buffer_process (GstVideoInference * +static gboolean gst_video_inference_model_buffer_process (GstVideoInference * self, GstBuffer * buffer); -static GstBuffer * gst_video_inference_bypass_buffer_process (GstVideoInference * +static gboolean gst_video_inference_bypass_buffer_process (GstVideoInference * self, GstBuffer * buffer_bypass, GstBuffer * buffer_model); static gboolean gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, GstEvent * event, gpointer user_data); @@ -212,13 +211,15 @@ gst_video_inference_set_property (GObject * object, break; case PROP_MODEL_LOCATION: GstState actual_state; - gst_element_get_state (GST_ELEMENT(self), &actual_state, NULL, GST_SECOND); + gst_element_get_state (GST_ELEMENT (self), &actual_state, NULL, + GST_SECOND); GST_OBJECT_LOCK (self); if (actual_state <= GST_STATE_READY) { g_free (priv->model_location); priv->model_location = g_value_dup_string (value); } else { - GST_ERROR_OBJECT (self, "Model location can only be set in the NULL or READY states"); + GST_ERROR_OBJECT (self, + "Model location can only be set in the NULL or READY states"); } GST_OBJECT_UNLOCK (self); break; @@ -312,7 +313,7 @@ gst_video_inference_start (GstVideoInference * self) if (NULL == priv->model_location) { GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, - ("Model Location has not been set"), (NULL)); + ("Model Location has not been set"), (NULL)); ret = FALSE; goto out; } @@ -346,12 +347,12 @@ gst_video_inference_start (GstVideoInference * self) ret = klass->start (self); } - out: +out: return ret; - error: +error: GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("R2Inference Error: (Code:%d) %s", error.GetCode(), - error.GetDescription ().c_str()), (NULL)); + ("R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()), (NULL)); return FALSE; } @@ -533,90 +534,145 @@ gst_video_inference_release_pad (GstElement * element, GstPad * pad) static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, - GstBuffer * buffer, GstCollectData * data, GstPad * pad) + GstBuffer * buffer, GstCollectData * data, GstPad * pad) { GstFlowReturn ret = GST_FLOW_OK; /* User didn't request this pad */ - if (NULL == data) { - goto out; - } - - if (NULL != pad) { - GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", - GST_DEBUG_PAD_NAME (pad)); - ret = gst_pad_push (pad, buffer); - } else { + if (NULL == data || pad == NULL) { GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", GST_DEBUG_PAD_NAME (data->pad)); - gst_buffer_unref (buffer); goto out; } + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", + GST_DEBUG_PAD_NAME (pad)); + ret = gst_pad_push (pad, gst_buffer_ref (buffer)); + if (GST_FLOW_OK != ret) { GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); } out: + gst_buffer_unref (buffer); return ret; } -static GstBuffer * +static gboolean gst_video_inference_model_buffer_process (GstVideoInference * self, GstBuffer * buffer) { - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); - GstBuffer *buffer_out = NULL; - GstBuffer *prepared = NULL; - gpointer prediction; - gsize predsize; - - if (NULL != klass->preprocess) { - /* Allocate a new buffer for the preprocessing stage */ - GstAllocationParams params; - gsize size = gst_buffer_get_size (buffer); - gst_allocation_params_init (¶ms); - prepared = gst_buffer_new_allocate (NULL, size, ¶ms); - - if (!klass->preprocess (self, buffer, prepared)) { - GST_ERROR_OBJECT (self, "Subclass failed Preprocess"); - gst_buffer_unref (prepared); - goto out; - } - } else { - prepared = gst_buffer_ref (buffer); + GstVideoInferenceClass *klass; + GstVideoInferencePrivate *priv; + GstBuffer *outbuf; + GstVideoFrame inframe, outframe; + GstAllocationParams params; + GstVideoInfo vininfo; + GstCaps *pad_caps; + std::shared_ptr < r2i::IPrediction > prediction; + std::unique_ptr < r2i::IFrameworkFactory > factory; + std::shared_ptr < r2i::IFrame > frame; + r2i::RuntimeError error; + gboolean ret; + + klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + outbuf = NULL; + ret = TRUE; + + if (NULL == klass->preprocess) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass did not implement preprocess"), (NULL)); + ret = FALSE; + goto out; + } + + gst_video_info_init (&vininfo); + pad_caps = gst_pad_get_current_caps (priv->sink_model); + gst_video_info_from_caps (&vininfo, pad_caps); + gst_caps_unref (pad_caps); + + gst_allocation_params_init (¶ms); + outbuf = + gst_buffer_new_allocate (NULL, + gst_buffer_get_size (buffer) * sizeof (gfloat), ¶ms); + gst_video_frame_map (&inframe, &vininfo, buffer, GST_MAP_READ); + gst_video_frame_map (&outframe, &vininfo, outbuf, GST_MAP_WRITE); + + if (!klass->preprocess (self, &inframe, &outframe)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed to preprocess"), (NULL)); + ret = FALSE; + goto free_frames; + } + + factory = + r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, error); + if (error.IsError ()) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Failed to create factory backend (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()), (NULL)); + goto free_frames; } - /* TODO: Create R2I Frame */ - /* TODO: Creat R2I Prediction */ - predsize = 1000 * sizeof (float); - prediction = g_malloc (predsize); - /* TODO: Predict using the Prepared Buffer */ + frame = factory->MakeFrame (error); + if (error.IsError ()) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Failed to created backend frame (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()), (NULL)); + goto free_frames; + } + + error = + frame->Configure (outframe.data[0], inframe.info.width, + inframe.info.height, r2i::ImageFormat::Id::RGB); + if (error.IsError ()) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Failed to configure backend frame (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()), (NULL)); + goto free_frames; + } + + prediction = priv->engine->Predict (frame, error); + if (error.IsError ()) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Backend prediction failed (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()), (NULL)); + goto free_frames; + } if (NULL != klass->postprocess) { - if (!klass->postprocess (self, buffer, prediction, predsize)) { - GST_ERROR_OBJECT (self, "Subclass failed postprocess"); - gst_buffer_unref (prepared); - goto out; + if (!klass->postprocess (self, &outframe, prediction->GetResultData (), + prediction->GetResultSize ())) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed at preprocess"), (NULL)); + goto free_frames; } } - gst_buffer_unref (prepared); - - buffer_out = buffer; +free_frames: + gst_video_frame_unmap (&outframe); + gst_video_frame_unmap (&inframe); + gst_buffer_unref (outbuf); out: - return buffer_out; + return ret; } -static GstBuffer * +static gboolean gst_video_inference_bypass_buffer_process (GstVideoInference * self, GstBuffer * buffer_bypass, GstBuffer * buffer_model) { /* TODO: Get GstMetas from Model buffer and attach them to the Bypass Buffer */ - return buffer_bypass; + return TRUE; } static GstFlowReturn @@ -625,22 +681,21 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstFlowReturn ret = GST_FLOW_OK; - GstBuffer * buffer_model; - GstBuffer * buffer_bypass; + GstBuffer *buffer_model; + GstBuffer *buffer_bypass; /* Process Model Buffer */ if (priv->sink_model_data) { buffer_model = gst_collect_pads_pop (priv->cpads, priv->sink_model_data); if (NULL == buffer_model) { GST_INFO_OBJECT (self, "EOS requested on %s:%s", - GST_DEBUG_PAD_NAME (priv->sink_model_data->pad)); + GST_DEBUG_PAD_NAME (priv->sink_model_data->pad)); ret = GST_FLOW_EOS; goto out; } - buffer_model = gst_video_inference_model_buffer_process (self, buffer_model); - if (NULL == buffer_model) { + if (!gst_video_inference_model_buffer_process (self, buffer_model)) { ret = GST_FLOW_ERROR; - goto out; + goto model_free; } } @@ -649,34 +704,40 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) buffer_bypass = gst_collect_pads_pop (priv->cpads, priv->sink_bypass_data); if (NULL == buffer_bypass) { GST_INFO_OBJECT (self, "EOS requested on %s:%s", - GST_DEBUG_PAD_NAME (priv->sink_bypass_data->pad)); + GST_DEBUG_PAD_NAME (priv->sink_bypass_data->pad)); ret = GST_FLOW_EOS; - goto out; + goto model_free; } - buffer_bypass = gst_video_inference_bypass_buffer_process (self, buffer_bypass, buffer_model); - if (NULL == buffer_bypass) { + if (!gst_video_inference_bypass_buffer_process (self, buffer_bypass, + buffer_model)) { ret = GST_FLOW_ERROR; - goto out; + goto bypass_free; } } /* Forward buffers to src pads */ if (NULL != buffer_model) { - ret = gst_video_inference_forward_buffer (self, buffer_model, - priv->sink_model_data, priv->src_model); + ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), + priv->sink_model_data, priv->src_model); if (GST_FLOW_OK != ret) { - goto out; + goto bypass_free; } } if (NULL != buffer_bypass) { - ret = gst_video_inference_forward_buffer (self, buffer_bypass, - priv->sink_bypass_data, priv->src_bypass); + ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_bypass), + priv->sink_bypass_data, priv->src_bypass); if (GST_FLOW_OK != ret) { - goto out; + goto bypass_free; } } +bypass_free: + gst_buffer_unref (buffer_bypass); + +model_free: + gst_buffer_unref (buffer_model); + out: return ret; } @@ -711,18 +772,17 @@ gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, if (NULL != srcpad) { GST_LOG_OBJECT (self, "Forwarding event %s from %s:%s", - GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); /* Collect pads will decrease the refcount of the event when we return */ gst_event_ref (event); if (FALSE == gst_pad_push_event (srcpad, event)) { GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", - GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); goto out; } - } - else { + } else { GST_LOG_OBJECT (self, "Dropping event %s from %s:%s", - GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); } ret = gst_collect_pads_event_default (priv->cpads, pad, event, FALSE); diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 459880e2..de6d573c 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -23,6 +23,7 @@ #define __GST_VIDEO_INFERENCE_H__ #include +#include G_BEGIN_DECLS @@ -35,8 +36,8 @@ struct _GstVideoInferenceClass gboolean (* start) (GstVideoInference *self); gboolean (* stop) (GstVideoInference *self); - gboolean (* preprocess) (GstVideoInference *self, GstBuffer * inbuf, GstBuffer * outbuf); - gboolean (* postprocess) (GstVideoInference *self, GstBuffer *buf, const gpointer prediction, + gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, GstVideoFrame *outframe); + gboolean (* postprocess) (GstVideoInference *self, GstVideoFrame *outframe, const gpointer prediction, gsize size); }; From 9cbc7ff92b912562586ae957cac779b7f86039ea Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Tue, 8 Jan 2019 23:21:04 +0000 Subject: [PATCH 27/49] Add preprocess and postprocess to GstGoogleNet --- ext/r2inference/Makefile.am | 15 ++++++++-- ext/r2inference/gstgooglenet.c | 51 ++++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ext/r2inference/Makefile.am b/ext/r2inference/Makefile.am index b31a60b0..ea66bab7 100644 --- a/ext/r2inference/Makefile.am +++ b/ext/r2inference/Makefile.am @@ -9,12 +9,23 @@ libgstinference_la_CFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ -I$(top_srcdir)/gst-libs + +libgstinference_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + + libgstinference_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la libgstinference_la_LDFLAGS = \ @@ -23,6 +34,6 @@ libgstinference_la_LDFLAGS = \ libgstinference_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) -noinst_HEADERS = \ - gstgooglenet.h \ +noinst_HEADERS = \ + gstgooglenet.h \ gsttinyyolo.h diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index dbea8a8b..1f36469f 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -45,7 +45,12 @@ GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); #define GST_CAT_DEFAULT gst_googlenet_debug_category /* prototypes */ +#define CHANNELS 3 +const float GoogleNetMean[] = { 0.40787054 * 255.0, + 0.45752458 * 255.0, + 0.48109378 * 255.0 +}; static void gst_googlenet_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); @@ -55,9 +60,9 @@ static void gst_googlenet_dispose (GObject * object); static void gst_googlenet_finalize (GObject * object); static gboolean gst_googlenet_preprocess (GstVideoInference * vi, - GstBuffer * inbuf, GstBuffer * outbuf); + GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_googlenet_postprocess (GstVideoInference * vi, - GstBuffer * buf, const gpointer prediction, gsize predsize); + GstVideoFrame * outframe, const gpointer prediction, gsize predsize); static gboolean gst_googlenet_start (GstVideoInference * vi); static gboolean gst_googlenet_stop (GstVideoInference * vi); @@ -190,29 +195,47 @@ gst_googlenet_finalize (GObject * object) static gboolean gst_googlenet_preprocess (GstVideoInference * vi, - GstBuffer * inbuf, GstBuffer * outbuf) + GstVideoFrame * inframe, GstVideoFrame * outframe) { - GstMapInfo ininfo; - GstMapInfo outinfo; + gint size; GST_LOG_OBJECT (vi, "Preprocess"); - - gst_buffer_map (inbuf, &ininfo, GST_MAP_READ); - gst_buffer_map (outbuf, &outinfo, GST_MAP_WRITE); - - memcpy (outinfo.data, ininfo.data, ininfo.size); - - gst_buffer_unmap (inbuf, &ininfo); - gst_buffer_unmap (outbuf, &outinfo); + size = CHANNELS * inframe->info.width * inframe->info.height; + + for (gint i = 0; i < size; i += CHANNELS) { + /* BGR = RGB - Mean */ + ((gfloat *) outframe->data[0])[i + 0] = + (((guchar *) inframe->data[0])[i + 2]) - GoogleNetMean[0]; + ((gfloat *) outframe->data[0])[i + 1] = + (((guchar *) inframe->data[0])[i + 1]) - GoogleNetMean[1]; + ((gfloat *) outframe->data[0])[i + 2] = + (((guchar *) inframe->data[0])[i + 0]) - GoogleNetMean[2]; + } return TRUE; } static gboolean gst_googlenet_postprocess (GstVideoInference * vi, - GstBuffer * buf, const gpointer prediction, gsize predsize) + GstVideoFrame * outframe, const gpointer prediction, gsize predsize) { + gint index; + gint size; + gdouble max; GST_LOG_OBJECT (vi, "Postprocess"); + index = 0; + max = -1; + size = predsize / sizeof (gfloat); + + for (gint i = 0; i < size; ++i) { + gfloat current = ((gfloat *) prediction)[i]; + if (current > max) { + max = current; + index = i; + } + } + + g_print ("Highest probability is label %i : (%f)\n", index, max); return TRUE; } From 2224b53b5418fbfb0199d500ecea9a138145fcc2 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Tue, 8 Jan 2019 23:22:04 +0000 Subject: [PATCH 28/49] Add preprocess and postprocess to GstTinyYolo --- ext/r2inference/gsttinyyolo.c | 266 ++++++++++++++++++- gst-libs/gst/r2inference/gstvideoinference.h | 7 +- 2 files changed, 258 insertions(+), 15 deletions(-) diff --git a/ext/r2inference/gsttinyyolo.c b/ext/r2inference/gsttinyyolo.c index f6fae39e..2effd416 100644 --- a/ext/r2inference/gsttinyyolo.c +++ b/ext/r2inference/gsttinyyolo.c @@ -46,6 +46,31 @@ GST_DEBUG_CATEGORY_STATIC (gst_tinyyolo_debug_category); /* prototypes */ +#define CHANNELS 3 +#define GRID_H 7 +#define GRID_W 7 +/* Number of classes */ +#define CLASSES 20 +/* Number of boxes per cell */ +#define BOXES 2 +/* Box dim */ +#define BOX_DIM 4 +/* Probability threshold */ +#define PROB_THRESH 0.08 +/* Intersection over union threshold */ +#define IOU_THRESH 0.30 + +typedef struct _Box Box; +struct _Box +{ + const gchar *label; + gdouble x_center; + gdouble y_center; + gdouble width; + gdouble height; + gdouble prob; +}; + static void gst_tinyyolo_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); @@ -55,12 +80,29 @@ static void gst_tinyyolo_dispose (GObject * object); static void gst_tinyyolo_finalize (GObject * object); static gboolean gst_tinyyolo_preprocess (GstVideoInference * vi, - GstBuffer * inbuf, GstBuffer * outbuf); + GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_tinyyolo_postprocess (GstVideoInference * vi, - GstBuffer * buf, const gpointer prediction, gsize predsize); + GstVideoFrame * outframe, const gpointer prediction, gsize predsize); static gboolean gst_tinyyolo_start (GstVideoInference * vi); static gboolean gst_tinyyolo_stop (GstVideoInference * vi); +static void print_box (Box in_box); + +static void print_top_predictions (gpointer prediction, + gint input_image_width, gint input_image_height); +static void get_boxes_from_prediction (gpointer prediction, + gint input_image_width, gint input_image_height, + Box * boxes, gint * elements); + +static void box_to_pixels (Box * normalized_box, gint row, gint col, + gint image_width, gint image_height); + +static void remove_duplicated_boxes (Box * boxes, gint * num_boxes); + +static void delete_box (Box * boxes, gint * num_boxes, gint index); + +static gdouble intersection_over_union (Box box_1, Box box_2); + enum { PROP_0 @@ -191,32 +233,232 @@ gst_tinyyolo_finalize (GObject * object) G_OBJECT_CLASS (gst_tinyyolo_parent_class)->finalize (object); } +void +print_box (Box in_box) +{ + g_print ("Box:"); + g_print ("[class:'%s', ", in_box.label); + g_print ("x_center:%f, ", in_box.x_center); + g_print ("y_center:%f, ", in_box.y_center); + g_print ("width:%f, ", in_box.width); + g_print ("height:%f, ", in_box.height); + g_print ("prob:%f]\n", in_box.prob); +} + +void +box_to_pixels (Box * normalized_box, gint row, gint col, gint image_width, + gint image_height) +{ + + /* adjust the box center according to its cell and grid dim */ + normalized_box->x_center += col; + normalized_box->y_center += row; + normalized_box->x_center /= GRID_H; + normalized_box->y_center /= GRID_W; + + /* adjust the lengths and widths */ + normalized_box->width *= normalized_box->width; + normalized_box->height *= normalized_box->height; + + /* scale the boxes to the image size in pixels */ + normalized_box->x_center *= image_width; + normalized_box->y_center *= image_height; + normalized_box->width *= image_width; + normalized_box->height *= image_height; +} + +void +get_boxes_from_prediction (gpointer prediction, gint input_image_width, + gint input_image_height, Box * boxes, gint * elements) +{ + + gint i; + gint j; + gint c; + gint b; + gint box_probs_start = GRID_H * GRID_W * CLASSES; + gint all_boxes_start = GRID_H * GRID_W * CLASSES + GRID_H * GRID_W * BOXES; + gint index; + gdouble class_prob; + gdouble box_prob; + gdouble prob; + gint counter = 0; + + const gchar *labels[] = { "aeroplane", "bicycle", "bird", "boat", "bottle", + "bus", "car", "cat", "chair", "cow", "diningtable", + "dog", "horse", "motorbike", "person", "pottedplant", + "sheep", "sofa", "train", "tvmonitor" + }; + + /* Iterate rows */ + for (i = 0; i < GRID_H; i++) { + /* Iterate colums */ + for (j = 0; j < GRID_W; j++) { + /* Iterate classes */ + for (c = 0; c < CLASSES; c++) { + index = (i * GRID_W + j) * CLASSES + c; + class_prob = ((gfloat *) prediction)[index]; + for (b = 0; b < BOXES; b++) { + index = (i * GRID_W + j) * BOXES + b; + box_prob = ((gfloat *) prediction)[box_probs_start + index]; + prob = class_prob * box_prob; + /* If the probability is over the threshold add it to the boxes list */ + if (prob > PROB_THRESH) { + Box result; + index = ((i * GRID_W + j) * BOXES + b) * BOX_DIM; + result.label = labels[c]; + result.x_center = ((gfloat *) prediction)[all_boxes_start + index]; + result.y_center = + ((gfloat *) prediction)[all_boxes_start + index + 1]; + result.width = ((gfloat *) prediction)[all_boxes_start + index + 2]; + result.height = + ((gfloat *) prediction)[all_boxes_start + index + 3]; + result.prob = prob; + box_to_pixels (&result, i, j, input_image_width, + input_image_height); + boxes[counter] = result; + counter = counter + 1; + } + } + } + } + + *elements = counter; + + } +} + +void +print_top_predictions (gpointer prediction, + gint input_image_width, gint input_image_height) +{ + + Box boxes[GRID_H * GRID_W * BOXES]; + gint elements = 0; + gint index = 0; + + get_boxes_from_prediction (prediction, input_image_width, input_image_height, + boxes, &elements); + + remove_duplicated_boxes (boxes, &elements); + + for (index = 0; index < elements; index = index + 1) { + print_box (boxes[index]); + } + +} + +gdouble +intersection_over_union (Box box_1, Box box_2) +{ + + /* + * Evaluate the intersection-over-union for two boxes + * The intersection-over-union metric determines how close + * two boxes are to being the same box. + */ + gdouble intersection_dim_1; + gdouble intersection_dim_2; + gdouble intersection_area; + gdouble union_area; + + /* First diminsion of the intersecting box */ + intersection_dim_1 = MIN (box_1.x_center + 0.5 * box_1.width, + box_2.x_center + 0.5 * box_2.width) - + MAX (box_1.x_center - 0.5 * box_1.width, + box_2.x_center - 0.5 * box_2.width); + + /* Second dimension of the intersecting box */ + intersection_dim_2 = MIN (box_1.y_center + 0.5 * box_1.height, + box_2.y_center + 0.5 * box_2.height) - + MAX (box_1.y_center - 0.5 * box_1.height, + box_2.y_center - 0.5 * box_2.height); + + if ((intersection_dim_1 < 0) || (intersection_dim_2 < 0)) { + intersection_area = 0; + } else { + intersection_area = intersection_dim_1 * intersection_dim_2; + } + union_area = box_1.width * box_1.height + box_2.width * box_2.height - + intersection_area; + return intersection_area / union_area; +} + +void +delete_box (Box * boxes, gint * num_boxes, gint index) +{ + + gint i, last_index; + if (*num_boxes > 0 && index < *num_boxes && index > -1) { + last_index = *num_boxes - 1; + for (i = index; i < last_index; i++) { + boxes[i] = boxes[i + 1]; + } + *num_boxes -= 1; + } +} + +void +remove_duplicated_boxes (Box * boxes, gint * num_boxes) +{ + + /* Remove duplicated boxes. A box is considered a duplicate if its + * intersection over union metric is above a threshold + */ + gdouble iou; + gint it1, it2; + + for (it1 = 0; it1 < *num_boxes - 1; it1++) { + for (it2 = it1 + 1; it2 < *num_boxes; it2++) { + if (strcmp (boxes[it1].label, boxes[it2].label) == 0) { + iou = intersection_over_union (boxes[it1], boxes[it2]); + if (iou > IOU_THRESH) { + if (boxes[it1].prob > boxes[it2].prob) { + delete_box (boxes, num_boxes, it2); + it2--; + } else { + delete_box (boxes, num_boxes, it1); + it1--; + break; + } + } + } + } + } +} + static gboolean gst_tinyyolo_preprocess (GstVideoInference * vi, - GstBuffer * inbuf, GstBuffer * outbuf) + GstVideoFrame * inframe, GstVideoFrame * outframe) { - GstMapInfo ininfo; - GstMapInfo outinfo; + gint size; GST_LOG_OBJECT (vi, "Preprocess"); - gst_buffer_map (inbuf, &ininfo, GST_MAP_READ); - gst_buffer_map (outbuf, &outinfo, GST_MAP_WRITE); - - memcpy (outinfo.data, ininfo.data, ininfo.size); + size = CHANNELS * inframe->info.width * inframe->info.height; - gst_buffer_unmap (inbuf, &ininfo); - gst_buffer_unmap (outbuf, &outinfo); + for (gint i = 0; i < size; i += CHANNELS) { + ((gfloat *) outframe->data[0])[i + 2] = + (((guchar *) inframe->data[0])[i + 2]) / 255.0; + ((gfloat *) outframe->data[0])[i + 1] = + (((guchar *) inframe->data[0])[i + 1]) / 255.0; + ((gfloat *) outframe->data[0])[i + 0] = + (((guchar *) inframe->data[0])[i + 0]) / 255.0; + } return TRUE; } static gboolean gst_tinyyolo_postprocess (GstVideoInference * vi, - GstBuffer * buf, const gpointer prediction, gsize predsize) + GstVideoFrame * outframe, const gpointer prediction, gsize predsize) { + GST_LOG_OBJECT (vi, "Postprocess"); + print_top_predictions (prediction, outframe->info.width, + outframe->info.height); + return TRUE; } diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index de6d573c..416f30a7 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -36,9 +36,10 @@ struct _GstVideoInferenceClass gboolean (* start) (GstVideoInference *self); gboolean (* stop) (GstVideoInference *self); - gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, GstVideoFrame *outframe); - gboolean (* postprocess) (GstVideoInference *self, GstVideoFrame *outframe, const gpointer prediction, - gsize size); + gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, + GstVideoFrame *outframe); + gboolean (* postprocess) (GstVideoInference *self, GstVideoFrame *outframe, + const gpointer prediction, gsize size); }; G_END_DECLS From f93ba14b20fdacb59fb600dd59156e2bf6ae4310 Mon Sep 17 00:00:00 2001 From: Jose Jimenez Date: Wed, 9 Jan 2019 14:12:17 -0600 Subject: [PATCH 29/49] Add support for tensorflow backend Create a class for tensorflow r2i backend and add it to the available backends. Allow properties to be set by each backend instead of adding them on GstBackend class since each one has different properties. --- gst-libs/gst/r2inference/Makefile.am | 6 +- gst-libs/gst/r2inference/gstbackend.cc | 8 +-- gst-libs/gst/r2inference/gstbackend.h | 9 +++ .../gst/r2inference/gstinferencebackends.cc | 7 ++- gst-libs/gst/r2inference/gstncsdk.cc | 6 ++ gst-libs/gst/r2inference/gsttensorflow.cc | 57 +++++++++++++++++++ gst-libs/gst/r2inference/gsttensorflow.h | 35 ++++++++++++ gst-libs/gst/r2inference/gstvideoinference.cc | 1 + 8 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 gst-libs/gst/r2inference/gsttensorflow.cc create mode 100644 gst-libs/gst/r2inference/gsttensorflow.h diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index e29ae8c5..7d5db676 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -6,7 +6,8 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstchildinspector.c \ gstinferencebackends.cc \ gstbackend.cc \ - gstncsdk.cc + gstncsdk.cc \ + gsttensorflow.cc libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ @@ -36,4 +37,5 @@ gstinferenceinclude_HEADERS= \ gstchildinspector.h \ gstinferencebackends.h \ gstbackend.h \ - gstncsdk.h + gstncsdk.h \ + gsttensorflow.h diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 90ee1bd3..c7e37a42 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -41,11 +41,6 @@ G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta * param); static int gst_backend_param_flags (int flags); -static void gst_backend_install_properties (GstBackendClass * klass); -static void gst_backend_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec); -static void gst_backend_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec); static void gst_backend_class_init (GstBackendClass * klass) @@ -55,7 +50,6 @@ gst_backend_class_init (GstBackendClass * klass) oclass->set_property = gst_backend_set_property; oclass->get_property = gst_backend_get_property; - gst_backend_install_properties (klass); } static void @@ -63,7 +57,7 @@ gst_backend_init (GstBackend * self) { } -static void +void gst_backend_install_properties (GstBackendClass * klass) { GObjectClass *oclass = G_OBJECT_CLASS (klass); diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 5a66cc20..7d726ff0 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -37,6 +37,15 @@ struct _GstBackendClass GHashTable *props; }; +void gst_backend_install_properties (GstBackendClass * klass); + +void gst_backend_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec); +void gst_backend_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec); + + + gboolean gst_backend_configure (GstBackend * self, std::shared_ptr < r2i::IEngine > engine, std::shared_ptr < r2i::IModel > model); diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index ed4a7ddc..3d68913e 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -22,6 +22,7 @@ #include "gstinferencebackends.h" #include "gstchildinspector.h" #include "gstncsdk.h" +#include "gsttensorflow.h" #include "gstbackend.h" #include #include @@ -32,6 +33,7 @@ static std::unordered_map backend_types ({ {r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, + {r2i::FrameworkCode::TENSORFLOW, GST_TYPE_TENSORFLOW}, {r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} }); @@ -49,6 +51,7 @@ gst_inference_backends_get_type (void) static const GEnumValue backend_desc[] = { {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, + {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", "tensorflow"}, {0, NULL, NULL} }; if (!backend_type) { @@ -98,8 +101,8 @@ gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, if (NULL == *backends_parameters) *backends_parameters = parameters; else - *backends_parameters = g_strconcat (*backends_parameters, parameters, NULL); - + *backends_parameters = g_strconcat (*backends_parameters,"\n", parameters, NULL); + g_object_unref (backend); } diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc index a2f8948f..70f15256 100644 --- a/gst-libs/gst/r2inference/gstncsdk.cc +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -39,8 +39,14 @@ static void gst_ncsdk_class_init (GstNcsdkClass * klass) { GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); bclass->backend = r2i::FrameworkCode::NCSDK; + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + + gst_backend_install_properties (bclass); } static void diff --git a/gst-libs/gst/r2inference/gsttensorflow.cc b/gst-libs/gst/r2inference/gsttensorflow.cc new file mode 100644 index 00000000..d669f25a --- /dev/null +++ b/gst-libs/gst/r2inference/gsttensorflow.cc @@ -0,0 +1,57 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gsttensorflow.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tensorflow_debug_category); +#define GST_CAT_DEFAULT gst_tensorflow_debug_category + +struct _GstTensorflow +{ + GstBackend parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstTensorflow, gst_tensorflow, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_tensorflow_debug_category, "tensorflow", 0, + "debug category for tensorflow parameters")); + +static void +gst_tensorflow_class_init (GstTensorflowClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + bclass->backend = r2i::FrameworkCode::TENSORFLOW; + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + + + gst_backend_install_properties (bclass); +} + +static void +gst_tensorflow_init (GstTensorflow * self) +{ + +} diff --git a/gst-libs/gst/r2inference/gsttensorflow.h b/gst-libs/gst/r2inference/gsttensorflow.h new file mode 100644 index 00000000..dd7e0adc --- /dev/null +++ b/gst-libs/gst/r2inference/gsttensorflow.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_TENSORFLOW_H__ +#define __GST_TENSORFLOW_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TENSORFLOW gst_tensorflow_get_type () +G_DECLARE_FINAL_TYPE(GstTensorflow, gst_tensorflow, GST, TENSORFLOW, GstBackend); + +G_END_DECLS + +#endif //__GST_TENSORFLOW_H__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.cc index dee0d5d7..dc255324 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.cc @@ -25,6 +25,7 @@ #include #include "gstncsdk.h" +#include "gsttensorflow.h" static GstStaticPadTemplate sink_bypass_factory = GST_STATIC_PAD_TEMPLATE ("sink_bypass", From 11ef07b254bef8da3462ee6af6db815c91200bdc Mon Sep 17 00:00:00 2001 From: Jose Jimenez Date: Wed, 9 Jan 2019 21:25:19 -0600 Subject: [PATCH 30/49] Add support to change between backends using the VideoInference property Use GstBackend as a interface to change between Frameworks on GstInference. Encapsulate r2i in GstBackend. --- gst-libs/gst/r2inference/Makefile.am | 3 +- gst-libs/gst/r2inference/gstbackend.cc | 126 ++++++++++++- gst-libs/gst/r2inference/gstbackend.h | 20 +-- gst-libs/gst/r2inference/gstbackendsubclass.h | 40 +++++ .../gst/r2inference/gstinferencebackends.cc | 17 +- .../gst/r2inference/gstinferencebackends.h | 4 +- gst-libs/gst/r2inference/gstncsdk.cc | 9 +- gst-libs/gst/r2inference/gstncsdk.h | 2 +- gst-libs/gst/r2inference/gsttensorflow.cc | 9 +- gst-libs/gst/r2inference/gsttensorflow.h | 2 +- ...tvideoinference.cc => gstvideoinference.c} | 168 ++++++++---------- 11 files changed, 259 insertions(+), 141 deletions(-) create mode 100644 gst-libs/gst/r2inference/gstbackendsubclass.h rename gst-libs/gst/r2inference/{gstvideoinference.cc => gstvideoinference.c} (87%) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 7d5db676..f3da0c9b 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -2,7 +2,7 @@ lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la libgstinference_@GST_API_VERSION@_la_SOURCES= \ - gstvideoinference.cc \ + gstvideoinference.c \ gstchildinspector.c \ gstinferencebackends.cc \ gstbackend.cc \ @@ -37,5 +37,6 @@ gstinferenceinclude_HEADERS= \ gstchildinspector.h \ gstinferencebackends.h \ gstbackend.h \ + gstbackendsubclass.h \ gstncsdk.h \ gsttensorflow.h diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index c7e37a42..4a925acf 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -20,7 +20,11 @@ */ #include "gstbackend.h" +#include "gstbackendsubclass.h" +#include + +#include #include GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); @@ -29,7 +33,12 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); typedef struct _GstBackendPrivate GstBackendPrivate; struct _GstBackendPrivate { + r2i::FrameworkCode code; + std::shared_ptr < r2i::IEngine > engine; + std::shared_ptr < r2i::ILoader > loader; + std::shared_ptr < r2i::IModel > model; std::unique_ptr < r2i::IParameters > params; + std::unique_ptr < r2i::IFrameworkFactory > factory; }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -58,7 +67,8 @@ gst_backend_init (GstBackend * self) } void -gst_backend_install_properties (GstBackendClass * klass) +gst_backend_install_properties (GstBackendClass * klass, + r2i::FrameworkCode code) { GObjectClass *oclass = G_OBJECT_CLASS (klass); r2i::RuntimeError error; @@ -67,7 +77,7 @@ gst_backend_install_properties (GstBackendClass * klass) klass->props = g_hash_table_new (g_direct_hash, g_direct_equal); - auto factory = r2i::IFrameworkFactory::MakeFactory (klass->backend, error); + auto factory = r2i::IFrameworkFactory::MakeFactory (code, error); auto pfactory = factory->MakeParameters (error); error = pfactory->List (params); @@ -162,19 +172,117 @@ gst_backend_get_property (GObject * object, guint property_id, } gboolean -gst_backend_configure (GstBackend * self, - std::shared_ptr < r2i::IEngine > engine, - std::shared_ptr < r2i::IModel > model) +gst_backend_start (GstBackend * self, const gchar * model_location) +{ + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + r2i::RuntimeError error; + + g_return_val_if_fail (priv, FALSE); + + priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, + error); + if (error.IsError ()) { + goto error; + } + + priv->engine = priv->factory->MakeEngine (error); + if (error.IsError ()) { + goto error; + } + + priv->loader = priv->factory->MakeLoader (error); + if (error.IsError ()) { + goto error; + } + + priv->model = priv->loader->Load (model_location, error); + if (error.IsError ()) { + goto error; + } + + error = priv->engine->SetModel (priv->model); + if (error.IsError ()) { + goto error; + } + + error = priv->engine->Start (); + if (error.IsError ()) { + goto error; + } + + return TRUE; +error: + GST_ERROR ("R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + +gboolean +gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, + gpointer *prediction_data, gsize *prediction_size) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + std::shared_ptr < r2i::IPrediction > prediction; + std::shared_ptr < r2i::IFrame > frame; r2i::RuntimeError error; + + g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (input_frame, FALSE); + g_return_val_if_fail (prediction_data, FALSE); + g_return_val_if_fail (prediction_size, FALSE); + + frame = priv->factory->MakeFrame (error); + if (error.IsError ()) { + goto error; + } - error = priv->params->Configure (engine, model); + GST_LOG_OBJECT (self, "Processing Frame of size %d x %d", + input_frame->info.width , input_frame->info.height); + + error = + frame->Configure (input_frame->data[0], input_frame->info.width, + input_frame->info.height, r2i::ImageFormat::Id::RGB); + if (error.IsError ()) { + goto error; + } + + prediction = priv->engine->Predict (frame, error); if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Error configuring backend parameters: (%d): %s", - error.GetCode (), error.GetDescription ().c_str ()); - return FALSE; + goto error; } + + *prediction_size = prediction->GetResultSize (); + + /*could we avoid memory copy ?*/ + *prediction_data = g_malloc(*prediction_size); + memcpy(*prediction_data , prediction->GetResultData(),*prediction_size); + + GST_LOG_OBJECT (self, "Size of prediction %p is %lu", + *prediction_data, *prediction_size); return TRUE; +error: + GST_ERROR ("R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + +gboolean +gst_backend_set_framework_code (GstBackend * backend, r2i::FrameworkCode code) +{ + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); + g_return_val_if_fail (priv, FALSE); + + priv->code = code; + return TRUE; + +} + +guint +gst_backend_get_framework_code (GstBackend * backend) +{ + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); + g_return_val_if_fail (priv, -1); + + return priv->code; } diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 7d726ff0..a275a220 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -23,7 +23,7 @@ #define __GST_BACKEND_H__ #include -#include +#include G_BEGIN_DECLS #define GST_TYPE_BACKEND gst_backend_get_type () @@ -33,22 +33,12 @@ struct _GstBackendClass { GObjectClass parent_class; - r2i::FrameworkCode backend; GHashTable *props; }; -void gst_backend_install_properties (GstBackendClass * klass); - -void gst_backend_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec); -void gst_backend_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec); - - - -gboolean gst_backend_configure (GstBackend * self, - std::shared_ptr < r2i::IEngine > engine, - std::shared_ptr < r2i::IModel > model); - +gboolean gst_backend_start (GstBackend *, const gchar *); +guint gst_backend_get_framework_code (GstBackend *); +gboolean gst_backend_process_frame (GstBackend *, GstVideoFrame *, + gpointer *, gsize *); G_END_DECLS #endif //__GST_BACKEND_H__ diff --git a/gst-libs/gst/r2inference/gstbackendsubclass.h b/gst-libs/gst/r2inference/gstbackendsubclass.h new file mode 100644 index 00000000..37aef953 --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackendsubclass.h @@ -0,0 +1,40 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ +#ifndef __GST_BACKENDSUBCLASS_H__ +#define __GST_BACKENDSUBCLASS_H__ + +#include "gstbackend.h" + +#include + +G_BEGIN_DECLS + +void gst_backend_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec); +void gst_backend_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec); +void gst_backend_install_properties (GstBackendClass * klass, + r2i::FrameworkCode code); +gboolean gst_backend_set_framework_code (GstBackend * backend, + r2i::FrameworkCode code); + +G_END_DECLS +#endif //__GST_BACKENDSUBCLASS_H__ diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index 3d68913e..79c5c0e0 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -41,8 +41,6 @@ static void gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, gchar ** backends_parameters, r2i::RuntimeError error, guint alignment); -static GType -gst_inference_backends_search_type (r2i::FrameworkCode id); GType gst_inference_backends_get_type (void) @@ -51,18 +49,20 @@ gst_inference_backends_get_type (void) static const GEnumValue backend_desc[] = { {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, - {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", "tensorflow"}, + {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", + "tensorflow"}, {0, NULL, NULL} }; if (!backend_type) { backend_type = - g_enum_register_static ("GstInferenceBackends", (GEnumValue *) backend_desc); + g_enum_register_static ("GstInferenceBackends", + (GEnumValue *) backend_desc); } return backend_type; } -static GType -gst_inference_backends_search_type (r2i::FrameworkCode id) +GType +gst_inference_backends_search_type (guint id) { auto search = backend_types.find (id); if (backend_types.end () == search) { @@ -101,8 +101,9 @@ gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, if (NULL == *backends_parameters) *backends_parameters = parameters; else - *backends_parameters = g_strconcat (*backends_parameters,"\n", parameters, NULL); - + *backends_parameters = + g_strconcat (*backends_parameters, "\n", parameters, NULL); + g_object_unref (backend); } diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h index 33e71f61..80cf88db 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.h +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -28,9 +28,9 @@ G_BEGIN_DECLS #define GST_TYPE_INFERENCE_BACKENDS (gst_inference_backends_get_type()) -GType gst_inference_backends_get_type(void); +GType gst_inference_backends_get_type (void); gchar * gst_inference_backends_get_string_properties (void); +GType gst_inference_backends_search_type (guint); G_END_DECLS - #endif //__GST_INFERENCE_BACKENDS_H__ diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc index 70f15256..ec797872 100644 --- a/gst-libs/gst/r2inference/gstncsdk.cc +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -41,16 +41,15 @@ gst_ncsdk_class_init (GstNcsdkClass * klass) GstBackendClass *bclass = GST_BACKEND_CLASS (klass); GObjectClass *oclass = G_OBJECT_CLASS (klass); - bclass->backend = r2i::FrameworkCode::NCSDK; - oclass->set_property = gst_backend_set_property; oclass->get_property = gst_backend_get_property; - - gst_backend_install_properties (bclass); + + gst_backend_install_properties (bclass, r2i::FrameworkCode::NCSDK); } static void gst_ncsdk_init (GstNcsdk * self) { - + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::NCSDK); } diff --git a/gst-libs/gst/r2inference/gstncsdk.h b/gst-libs/gst/r2inference/gstncsdk.h index f29a2f87..08689b68 100644 --- a/gst-libs/gst/r2inference/gstncsdk.h +++ b/gst-libs/gst/r2inference/gstncsdk.h @@ -23,7 +23,7 @@ #define __GST_NCSDK_H__ #include -#include +#include G_BEGIN_DECLS diff --git a/gst-libs/gst/r2inference/gsttensorflow.cc b/gst-libs/gst/r2inference/gsttensorflow.cc index d669f25a..889cedf7 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.cc +++ b/gst-libs/gst/r2inference/gsttensorflow.cc @@ -41,17 +41,14 @@ gst_tensorflow_class_init (GstTensorflowClass * klass) GstBackendClass *bclass = GST_BACKEND_CLASS (klass); GObjectClass *oclass = G_OBJECT_CLASS (klass); - bclass->backend = r2i::FrameworkCode::TENSORFLOW; - oclass->set_property = gst_backend_set_property; oclass->get_property = gst_backend_get_property; - - - gst_backend_install_properties (bclass); + gst_backend_install_properties (bclass, r2i::FrameworkCode::TENSORFLOW); } static void gst_tensorflow_init (GstTensorflow * self) { - + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::TENSORFLOW); } diff --git a/gst-libs/gst/r2inference/gsttensorflow.h b/gst-libs/gst/r2inference/gsttensorflow.h index dd7e0adc..793eb6dc 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.h +++ b/gst-libs/gst/r2inference/gsttensorflow.h @@ -23,7 +23,7 @@ #define __GST_TENSORFLOW_H__ #include -#include +#include G_BEGIN_DECLS diff --git a/gst-libs/gst/r2inference/gstvideoinference.cc b/gst-libs/gst/r2inference/gstvideoinference.c similarity index 87% rename from gst-libs/gst/r2inference/gstvideoinference.cc rename to gst-libs/gst/r2inference/gstvideoinference.c index dc255324..0540d715 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.cc +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -21,11 +21,10 @@ #include "gstvideoinference.h" #include "gstinferencebackends.h" +#include "gstbackend.h" + #include -#include -#include "gstncsdk.h" -#include "gsttensorflow.h" static GstStaticPadTemplate sink_bypass_factory = GST_STATIC_PAD_TEMPLATE ("sink_bypass", @@ -69,10 +68,6 @@ struct _GstVideoInferencePrivate GstBackend *backend; - std::shared_ptr < r2i::IEngine > engine; - std::shared_ptr < r2i::ILoader > loader; - std::shared_ptr < r2i::IModel > model; - gchar *model_location; }; @@ -119,6 +114,9 @@ static gboolean gst_video_inference_src_event (GstPad * pad, GstObject * parent, GstEvent * event); static GstPad *gst_video_inference_get_src_pad (GstVideoInference * self, GstVideoInferencePrivate * priv, GstPad * pad); +static void +gst_video_inference_set_backend (GstVideoInference * self, gint backend); +static guint gst_video_inference_get_backend_type (GstVideoInference * self); G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, GST_TYPE_ELEMENT, @@ -168,7 +166,7 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) g_object_class_install_property (oclass, PROP_MODEL_LOCATION, g_param_spec_string ("model-location", "Model Location", "Path to the model to use", DEFAULT_MODEL_LOCATION, - G_PARAM_READWRITE)); + G_PARAM_READWRITE)); klass->start = NULL; klass->stop = NULL; @@ -196,6 +194,8 @@ gst_video_inference_init (GstVideoInference * self) gst_video_inference_sink_event, (gpointer) (self)); priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); + + gst_video_inference_set_backend (self, DEFAULT_BACKEND); } static void @@ -204,14 +204,17 @@ gst_video_inference_set_property (GObject * object, { GstVideoInference *self = GST_VIDEO_INFERENCE (object); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstState actual_state; GST_LOG_OBJECT (self, "Set Property"); switch (property_id) { case PROP_BACKEND: + GST_OBJECT_LOCK (self); + gst_video_inference_set_backend (self, g_value_get_enum (value)); + GST_OBJECT_UNLOCK (self); break; case PROP_MODEL_LOCATION: - GstState actual_state; gst_element_get_state (GST_ELEMENT (self), &actual_state, NULL, GST_SECOND); GST_OBJECT_LOCK (self); @@ -241,6 +244,7 @@ gst_video_inference_get_property (GObject * object, switch (property_id) { case PROP_BACKEND: + g_value_set_enum (value, gst_video_inference_get_backend_type (self)); break; case PROP_MODEL_LOCATION: g_value_set_string (value, priv->model_location); @@ -304,11 +308,6 @@ gst_video_inference_start (GstVideoInference * self) GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); gboolean ret = TRUE; - r2i::RuntimeError error; - - /* TODO: Avoid hardcoded NCSDK, it should be queried from the backend */ - auto factory = r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, - error); GST_INFO_OBJECT (self, "Starting video inference"); @@ -319,29 +318,10 @@ gst_video_inference_start (GstVideoInference * self) goto out; } - priv->engine = factory->MakeEngine (error); - if (error.IsError ()) { - goto error; - } - - priv->loader = factory->MakeLoader (error); - if (error.IsError ()) { - goto error; - } - - priv->model = priv->loader->Load (priv->model_location, error); - if (error.IsError ()) { - goto error; - } - - error = priv->engine->SetModel (priv->model); - if (error.IsError ()) { - goto error; - } - - error = priv->engine->Start (); - if (error.IsError ()) { - goto error; + if (!gst_backend_start (priv->backend, priv->model_location)) { + GST_ELEMENT_ERROR (self, LIBRARY, INIT, + ("Could not start the selected backend"), (NULL)); + ret = FALSE; } if (klass->start != NULL) { @@ -350,11 +330,6 @@ gst_video_inference_start (GstVideoInference * self) out: return ret; -error: - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("R2Inference Error: (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()), (NULL)); - return FALSE; } static gboolean @@ -546,8 +521,7 @@ gst_video_inference_forward_buffer (GstVideoInference * self, goto out; } - GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", - GST_DEBUG_PAD_NAME (pad)); + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); ret = gst_pad_push (pad, gst_buffer_ref (buffer)); if (GST_FLOW_OK != ret) { @@ -571,11 +545,9 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, GstAllocationParams params; GstVideoInfo vininfo; GstCaps *pad_caps; - std::shared_ptr < r2i::IPrediction > prediction; - std::unique_ptr < r2i::IFrameworkFactory > factory; - std::shared_ptr < r2i::IFrame > frame; - r2i::RuntimeError error; gboolean ret; + gpointer prediction_data = NULL; + gsize prediction_size; klass = GST_VIDEO_INFERENCE_GET_CLASS (self); priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -585,7 +557,7 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, if (NULL == klass->preprocess) { GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass did not implement preprocess"), (NULL)); + ("Subclass did not implement preprocess"), (NULL)); ret = FALSE; goto out; } @@ -604,61 +576,31 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, if (!klass->preprocess (self, &inframe, &outframe)) { GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass failed to preprocess"), (NULL)); - ret = FALSE; - goto free_frames; - } - - factory = - r2i::IFrameworkFactory::MakeFactory (r2i::FrameworkCode::NCSDK, error); - if (error.IsError ()) { - ret = FALSE; - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Failed to create factory backend (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()), (NULL)); - goto free_frames; - } - - frame = factory->MakeFrame (error); - if (error.IsError ()) { + ("Subclass failed to preprocess"), (NULL)); ret = FALSE; - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Failed to created backend frame (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()), (NULL)); goto free_frames; } - error = - frame->Configure (outframe.data[0], inframe.info.width, - inframe.info.height, r2i::ImageFormat::Id::RGB); - if (error.IsError ()) { - ret = FALSE; - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Failed to configure backend frame (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()), (NULL)); - goto free_frames; - } + GST_LOG_OBJECT (self, "Processing frame using selected Backend"); - prediction = priv->engine->Predict (frame, error); - if (error.IsError ()) { + if (!gst_backend_process_frame (priv->backend, &outframe, + &prediction_data, &prediction_size)) { ret = FALSE; - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Backend prediction failed (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()), (NULL)); goto free_frames; } if (NULL != klass->postprocess) { - if (!klass->postprocess (self, &outframe, prediction->GetResultData (), - prediction->GetResultSize ())) { + if (!klass->postprocess (self, &outframe, prediction_data, prediction_size)) { ret = FALSE; GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass failed at preprocess"), (NULL)); + ("Subclass failed at preprocess"), (NULL)); goto free_frames; } } free_frames: + if(prediction_data) + g_free(prediction_data); gst_video_frame_unmap (&outframe); gst_video_frame_unmap (&inframe); gst_buffer_unref (outbuf); @@ -682,8 +624,8 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstFlowReturn ret = GST_FLOW_OK; - GstBuffer *buffer_model; - GstBuffer *buffer_bypass; + GstBuffer *buffer_model = NULL; + GstBuffer *buffer_bypass = NULL; /* Process Model Buffer */ if (priv->sink_model_data) { @@ -710,7 +652,7 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) goto model_free; } if (!gst_video_inference_bypass_buffer_process (self, buffer_bypass, - buffer_model)) { + buffer_model)) { ret = GST_FLOW_ERROR; goto bypass_free; } @@ -718,7 +660,8 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) /* Forward buffers to src pads */ if (NULL != buffer_model) { - ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), + ret = + gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), priv->sink_model_data, priv->src_model); if (GST_FLOW_OK != ret) { goto bypass_free; @@ -726,8 +669,10 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) } if (NULL != buffer_bypass) { - ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_bypass), - priv->sink_bypass_data, priv->src_bypass); + ret = + gst_video_inference_forward_buffer (self, + gst_buffer_ref (buffer_bypass), priv->sink_bypass_data, + priv->src_bypass); if (GST_FLOW_OK != ret) { goto bypass_free; } @@ -821,5 +766,42 @@ gst_video_inference_finalize (GObject * object) g_clear_object (&priv->backend); + if (priv->backend) + g_object_unref (priv->backend); + G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); } + +static void +gst_video_inference_set_backend (GstVideoInference * self, gint backend) +{ + GstBackend *backend_new; + GType backend_type; + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_return_if_fail (priv); + + if (GST_STATE (self) != GST_STATE_NULL) { + g_warning ("Can't set backend property if not on NULL state"); + return; + } + + if (priv->backend) + g_object_unref (priv->backend); + + backend_type = gst_inference_backends_search_type (backend); + backend_new = (GstBackend *) g_object_new (backend_type, NULL); + priv->backend = backend_new; + + return; +} + +static guint +gst_video_inference_get_backend_type (GstVideoInference * self) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_return_val_if_fail (priv, -1); + + return gst_backend_get_framework_code (priv->backend); +} From 22b5bc89fda3887ee3395d359302e38031fc0e88 Mon Sep 17 00:00:00 2001 From: Jose Jimenez Date: Tue, 15 Jan 2019 14:09:58 -0600 Subject: [PATCH 31/49] Transfer GstBackend errors to base classes Instead of printing a GST_ERROR, transfer a GError to base clases for error handling. --- gst-libs/gst/r2inference/gstbackend.cc | 27 +++++++++++++++++--- gst-libs/gst/r2inference/gstbackend.h | 6 +++-- gst-libs/gst/r2inference/gstvideoinference.c | 19 ++++++++++---- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 4a925acf..b8cdf9b1 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -51,6 +51,8 @@ G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta * param); static int gst_backend_param_flags (int flags); +#define GST_BACKEND_ERROR gst_backend_error_quark() + static void gst_backend_class_init (GstBackendClass * klass) { @@ -172,12 +174,14 @@ gst_backend_get_property (GObject * object, guint property_id, } gboolean -gst_backend_start (GstBackend * self, const gchar * model_location) +gst_backend_start (GstBackend * self, const gchar * model_location, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (model_location, FALSE); + g_return_val_if_fail (err, FALSE); priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, error); @@ -212,14 +216,15 @@ gst_backend_start (GstBackend * self, const gchar * model_location) return TRUE; error: - GST_ERROR ("R2Inference Error: (Code:%d) %s", error.GetCode (), + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), error.GetDescription ().c_str ()); return FALSE; } gboolean gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, - gpointer *prediction_data, gsize *prediction_size) + gpointer *prediction_data, gsize *prediction_size, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); std::shared_ptr < r2i::IPrediction > prediction; @@ -230,6 +235,7 @@ gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, g_return_val_if_fail (input_frame, FALSE); g_return_val_if_fail (prediction_data, FALSE); g_return_val_if_fail (prediction_size, FALSE); + g_return_val_if_fail (err, FALSE); frame = priv->factory->MakeFrame (error); if (error.IsError ()) { @@ -262,7 +268,8 @@ gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, return TRUE; error: - GST_ERROR ("R2Inference Error: (Code:%d) %s", error.GetCode (), + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), error.GetDescription ().c_str ()); return FALSE; } @@ -286,3 +293,15 @@ gst_backend_get_framework_code (GstBackend * backend) return priv->code; } + +GQuark +gst_backend_error_quark(void) +{ + static GQuark q = 0; + + if (0 == q) { + q = g_quark_from_static_string("gst-backend-error-quark"); + } + + return q; +} diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index a275a220..b61f2abe 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -36,9 +36,11 @@ struct _GstBackendClass GHashTable *props; }; -gboolean gst_backend_start (GstBackend *, const gchar *); +GQuark gst_backend_error_quark (void); +gboolean gst_backend_start (GstBackend *, const gchar *, GError **); guint gst_backend_get_framework_code (GstBackend *); gboolean gst_backend_process_frame (GstBackend *, GstVideoFrame *, - gpointer *, gsize *); + gpointer *, gsize *, GError **); + G_END_DECLS #endif //__GST_BACKEND_H__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 0540d715..f1c4de25 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -308,6 +308,7 @@ gst_video_inference_start (GstVideoInference * self) GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); gboolean ret = TRUE; + GError *err = NULL; GST_INFO_OBJECT (self, "Starting video inference"); @@ -318,9 +319,9 @@ gst_video_inference_start (GstVideoInference * self) goto out; } - if (!gst_backend_start (priv->backend, priv->model_location)) { + if (!gst_backend_start (priv->backend, priv->model_location, &err)) { GST_ELEMENT_ERROR (self, LIBRARY, INIT, - ("Could not start the selected backend"), (NULL)); + ("Could not start the selected backend: (%s)", err->message), (NULL)); ret = FALSE; } @@ -329,6 +330,8 @@ gst_video_inference_start (GstVideoInference * self) } out: + if (err) + g_error_free (err); return ret; } @@ -548,6 +551,7 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, gboolean ret; gpointer prediction_data = NULL; gsize prediction_size; + GError *err = NULL; klass = GST_VIDEO_INFERENCE_GET_CLASS (self); priv = GST_VIDEO_INFERENCE_PRIVATE (self); @@ -584,7 +588,10 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, GST_LOG_OBJECT (self, "Processing frame using selected Backend"); if (!gst_backend_process_frame (priv->backend, &outframe, - &prediction_data, &prediction_size)) { + &prediction_data, &prediction_size, &err)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Could not process using the selected backend: (%s)", + err->message), (NULL)); ret = FALSE; goto free_frames; } @@ -599,12 +606,14 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, } free_frames: - if(prediction_data) - g_free(prediction_data); + if (prediction_data) + g_free (prediction_data); gst_video_frame_unmap (&outframe); gst_video_frame_unmap (&inframe); gst_buffer_unref (outbuf); out: + if (err) + g_error_free (err); return ret; } From a13efd0bd5998208deb2636774c326c908c136a9 Mon Sep 17 00:00:00 2001 From: Jose Jimenez Date: Wed, 23 Jan 2019 11:40:42 -0600 Subject: [PATCH 32/49] Avoid sending error in case of flush or eos We don't need to report error when a pad is on flushing or eos state since this is a common case when closing down a pipeline --- gst-libs/gst/r2inference/gstvideoinference.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index f1c4de25..842795c0 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -527,7 +527,12 @@ gst_video_inference_forward_buffer (GstVideoInference * self, GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); ret = gst_pad_push (pad, gst_buffer_ref (buffer)); - if (GST_FLOW_OK != ret) { + if (GST_FLOW_FLUSHING == ret || GST_FLOW_EOS == ret) { + GST_INFO_OBJECT (self, "Pad %s:%s returned: (%d) %s", + GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); + } + + else if (GST_FLOW_OK != ret) { GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); } From 99cd0bb5ce9060e1f23e268cc0108c2634be5158 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Wed, 16 Jan 2019 19:18:19 +0000 Subject: [PATCH 33/49] Remove unused hash table --- gst-libs/gst/r2inference/gstbackend.cc | 21 --------------------- gst-libs/gst/r2inference/gstbackend.h | 1 - 2 files changed, 22 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index b8cdf9b1..85131fff 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -77,17 +77,12 @@ gst_backend_install_properties (GstBackendClass * klass, static std::vector < r2i::ParameterMeta > params; static gint nprop = 1; - klass->props = g_hash_table_new (g_direct_hash, g_direct_equal); - auto factory = r2i::IFrameworkFactory::MakeFactory (code, error); auto pfactory = factory->MakeParameters (error); error = pfactory->List (params); for (auto & param:params) { GParamSpec *spec = gst_backend_param_to_spec (¶m); - - g_hash_table_insert (klass->props, GINT_TO_POINTER (nprop), - (gpointer) (param.name.c_str ())); g_object_class_install_property (oclass, nprop, spec); nprop++; } @@ -141,36 +136,20 @@ void gst_backend_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstBackendClass *klass = GST_BACKEND_GET_CLASS (object); GstBackend *self = GST_BACKEND (object); - const gchar *param; GST_DEBUG_OBJECT (self, "set_property"); - param = (const gchar *) g_hash_table_lookup (klass->props, - GINT_TO_POINTER (property_id)); - if (NULL == param) { - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } } void gst_backend_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstBackendClass *klass = GST_BACKEND_GET_CLASS (object); GstBackend *self = GST_BACKEND (object); - const gchar *param; GST_DEBUG_OBJECT (self, "get_property"); - param = (const gchar *) g_hash_table_lookup (klass->props, - GINT_TO_POINTER (property_id)); - if (NULL == param) { - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } } gboolean diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index b61f2abe..21d41c6e 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -33,7 +33,6 @@ struct _GstBackendClass { GObjectClass parent_class; - GHashTable *props; }; GQuark gst_backend_error_quark (void); From c28a7cd32cb2260b0e86b91a78f0b50f79189b10 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Thu, 17 Jan 2019 00:03:34 +0000 Subject: [PATCH 34/49] Add safety thread logic --- gst-libs/gst/r2inference/gstbackend.cc | 144 ++++++++++++++++--------- 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 85131fff..bc6bea55 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -26,52 +26,72 @@ #include #include +#include GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category +typedef struct { + guint property_id; + GValue value; + GParamSpec pspec; +} InferenceProperty; + + typedef struct _GstBackendPrivate GstBackendPrivate; -struct _GstBackendPrivate -{ +struct _GstBackendPrivate { r2i::FrameworkCode code; std::shared_ptr < r2i::IEngine > engine; std::shared_ptr < r2i::ILoader > loader; std::shared_ptr < r2i::IModel > model; std::unique_ptr < r2i::IParameters > params; std::unique_ptr < r2i::IFrameworkFactory > factory; + std::queue property_queue; + GMutex backend_mutex; + gboolean backend_started; }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, - GST_DEBUG_CATEGORY_INIT (gst_backend_debug_category, "backend", 0, - "debug category for backend parameters"); G_ADD_PRIVATE (GstBackend)); + GST_DEBUG_CATEGORY_INIT (gst_backend_debug_category, "backend", 0, + "debug category for backend parameters"); G_ADD_PRIVATE (GstBackend)); #define GST_BACKEND_PRIVATE(self) \ (GstBackendPrivate *)(gst_backend_get_instance_private (self)) -static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta * param); +static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta *param); static int gst_backend_param_flags (int flags); +static void gst_backend_finalize (GObject *obj); #define GST_BACKEND_ERROR gst_backend_error_quark() static void -gst_backend_class_init (GstBackendClass * klass) -{ +gst_backend_class_init (GstBackendClass *klass) { GObjectClass *oclass = G_OBJECT_CLASS (klass); oclass->set_property = gst_backend_set_property; oclass->get_property = gst_backend_get_property; + oclass->finalize = gst_backend_finalize; } static void -gst_backend_init (GstBackend * self) -{ +gst_backend_init (GstBackend *self) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + g_mutex_init(&priv->backend_mutex); + priv->backend_started = false; +} + +static void +gst_backend_finalize (GObject *obj) { + GstBackend *self = GST_BACKEND (obj); + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + g_mutex_clear (&priv->backend_mutex); + G_OBJECT_CLASS (gst_backend_parent_class)->finalize (obj); } void -gst_backend_install_properties (GstBackendClass * klass, - r2i::FrameworkCode code) -{ +gst_backend_install_properties (GstBackendClass *klass, + r2i::FrameworkCode code) { GObjectClass *oclass = G_OBJECT_CLASS (klass); r2i::RuntimeError error; static std::vector < r2i::ParameterMeta > params; @@ -81,7 +101,7 @@ gst_backend_install_properties (GstBackendClass * klass, auto pfactory = factory->MakeParameters (error); error = pfactory->List (params); - for (auto & param:params) { + for (auto ¶m : params) { GParamSpec *spec = gst_backend_param_to_spec (¶m); g_object_class_install_property (oclass, nprop, spec); nprop++; @@ -89,8 +109,7 @@ gst_backend_install_properties (GstBackendClass * klass, } static int -gst_backend_param_flags (int flags) -{ +gst_backend_param_flags (int flags) { int pflags = G_PARAM_STATIC_STRINGS; if (r2i::ParameterMeta::Flags::READ & flags) { @@ -105,24 +124,23 @@ gst_backend_param_flags (int flags) } static GParamSpec * -gst_backend_param_to_spec (r2i::ParameterMeta * param) -{ +gst_backend_param_to_spec (r2i::ParameterMeta *param) { GParamSpec *spec = NULL; switch (param->type) { - case (r2i::ParameterMeta::Type::INTEGER):{ + case (r2i::ParameterMeta::Type::INTEGER): { spec = g_param_spec_int (param->name.c_str (), - param->name.c_str (), - param->description.c_str (), - G_MININT, - G_MAXINT, 0, (GParamFlags) gst_backend_param_flags (param->flags)); + param->name.c_str (), + param->description.c_str (), + G_MININT, + G_MAXINT, 0, (GParamFlags) gst_backend_param_flags (param->flags)); break; } - case (r2i::ParameterMeta::Type::STRING):{ + case (r2i::ParameterMeta::Type::STRING): { spec = g_param_spec_string (param->name.c_str (), - param->name.c_str (), - param->description.c_str (), - NULL, (GParamFlags) gst_backend_param_flags (param->flags)); + param->name.c_str (), + param->description.c_str (), + NULL, (GParamFlags) gst_backend_param_flags (param->flags)); break; } default: @@ -133,19 +151,36 @@ gst_backend_param_to_spec (r2i::ParameterMeta * param) } void -gst_backend_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec) -{ +gst_backend_set_property (GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) { GstBackend *self = GST_BACKEND (object); + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); GST_DEBUG_OBJECT (self, "set_property"); + g_mutex_lock (&priv->backend_mutex); + if (priv->backend_started) { + switch (pspec->value_type) { + case G_TYPE_STRING: + priv->params->Set(pspec->name, g_value_get_string(value)); + break; + case G_TYPE_INT: + priv->params->Set(pspec->name, g_value_get_int(value)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + } else { + /* TODO: Add params to queue */ + } + g_mutex_unlock (&priv->backend_mutex); + } void -gst_backend_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec) -{ +gst_backend_get_property (GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) { GstBackend *self = GST_BACKEND (object); GST_DEBUG_OBJECT (self, "get_property"); @@ -153,17 +188,16 @@ gst_backend_get_property (GObject * object, guint property_id, } gboolean -gst_backend_start (GstBackend * self, const gchar * model_location, GError **err) -{ +gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; - + g_return_val_if_fail (priv, FALSE); g_return_val_if_fail (model_location, FALSE); g_return_val_if_fail (err, FALSE); priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, - error); + error); if (error.IsError ()) { goto error; } @@ -192,7 +226,14 @@ gst_backend_start (GstBackend * self, const gchar * model_location, GError **err if (error.IsError ()) { goto error; } - + + g_mutex_lock (&priv->backend_mutex); + + /* TODO: Extract params and free queue */ + + priv->backend_started = true; + g_mutex_unlock (&priv->backend_mutex); + return TRUE; error: g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), @@ -202,14 +243,13 @@ gst_backend_start (GstBackend * self, const gchar * model_location, GError **err } gboolean -gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, - gpointer *prediction_data, gsize *prediction_size, GError **err) -{ +gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, + gpointer *prediction_data, gsize *prediction_size, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); std::shared_ptr < r2i::IPrediction > prediction; std::shared_ptr < r2i::IFrame > frame; r2i::RuntimeError error; - + g_return_val_if_fail (priv, FALSE); g_return_val_if_fail (input_frame, FALSE); g_return_val_if_fail (prediction_data, FALSE); @@ -222,11 +262,11 @@ gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, } GST_LOG_OBJECT (self, "Processing Frame of size %d x %d", - input_frame->info.width , input_frame->info.height); + input_frame->info.width, input_frame->info.height); error = - frame->Configure (input_frame->data[0], input_frame->info.width, - input_frame->info.height, r2i::ImageFormat::Id::RGB); + frame->Configure (input_frame->data[0], input_frame->info.width, + input_frame->info.height, r2i::ImageFormat::Id::RGB); if (error.IsError ()) { goto error; } @@ -235,12 +275,12 @@ gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, if (error.IsError ()) { goto error; } - + *prediction_size = prediction->GetResultSize (); /*could we avoid memory copy ?*/ *prediction_data = g_malloc(*prediction_size); - memcpy(*prediction_data , prediction->GetResultData(),*prediction_size); + memcpy(*prediction_data, prediction->GetResultData(), *prediction_size); GST_LOG_OBJECT (self, "Size of prediction %p is %lu", *prediction_data, *prediction_size); @@ -248,14 +288,13 @@ gst_backend_process_frame (GstBackend * self, GstVideoFrame * input_frame, return TRUE; error: g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), - "R2Inference Error: (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()); - return FALSE; + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; } gboolean -gst_backend_set_framework_code (GstBackend * backend, r2i::FrameworkCode code) -{ +gst_backend_set_framework_code (GstBackend *backend, r2i::FrameworkCode code) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); g_return_val_if_fail (priv, FALSE); @@ -265,8 +304,7 @@ gst_backend_set_framework_code (GstBackend * backend, r2i::FrameworkCode code) } guint -gst_backend_get_framework_code (GstBackend * backend) -{ +gst_backend_get_framework_code (GstBackend *backend) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); g_return_val_if_fail (priv, -1); From 568d300f036c84fdf9b56831f57d9fbc472e1955 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Thu, 17 Jan 2019 01:09:13 +0000 Subject: [PATCH 35/49] Add queue process to properties --- gst-libs/gst/r2inference/gstbackend.cc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index bc6bea55..37bdf97c 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -172,7 +172,14 @@ gst_backend_set_property (GObject *object, guint property_id, break; } } else { - /* TODO: Add params to queue */ + + InferenceProperty property; + + property.property_id = property_id; + property.value = *value; + property.pspec = *pspec; + priv->property_queue.push(property); + } g_mutex_unlock (&priv->backend_mutex); @@ -229,7 +236,22 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) g_mutex_lock (&priv->backend_mutex); - /* TODO: Extract params and free queue */ + while (!priv->property_queue.empty()) { + InferenceProperty property = priv->property_queue.front(); + switch (property.pspec.value_type) { + case G_TYPE_STRING: + priv->params->Set(property.pspec.name, g_value_get_string(&property.value)); + break; + case G_TYPE_INT: + priv->params->Set(property.pspec.name, g_value_get_int(&property.value)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + + priv->property_queue.pop(); + } priv->backend_started = true; g_mutex_unlock (&priv->backend_mutex); From 6e06da4ca2ddf373537f824967c13278ae2e2cdb Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Thu, 17 Jan 2019 14:32:37 -0600 Subject: [PATCH 36/49] Add queue logic to property set --- gst-libs/gst/r2inference/gstbackend.cc | 57 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 37bdf97c..90e30387 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -32,9 +32,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category typedef struct { - guint property_id; - GValue value; - GParamSpec pspec; + GValue *value; + GParamSpec *pspec; } InferenceProperty; @@ -44,11 +43,11 @@ struct _GstBackendPrivate { std::shared_ptr < r2i::IEngine > engine; std::shared_ptr < r2i::ILoader > loader; std::shared_ptr < r2i::IModel > model; - std::unique_ptr < r2i::IParameters > params; + std::shared_ptr < r2i::IParameters > params; std::unique_ptr < r2i::IFrameworkFactory > factory; - std::queue property_queue; GMutex backend_mutex; gboolean backend_started; + std::shared_ptr < std::queue > property_queue; }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -79,6 +78,7 @@ gst_backend_init (GstBackend *self) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); g_mutex_init(&priv->backend_mutex); priv->backend_started = false; + priv->property_queue = std::make_shared>(); } static void @@ -155,6 +155,7 @@ gst_backend_set_property (GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { GstBackend *self = GST_BACKEND (object); GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + GValue *value_copy; GST_DEBUG_OBJECT (self, "set_property"); @@ -172,14 +173,13 @@ gst_backend_set_property (GObject *object, guint property_id, break; } } else { - - InferenceProperty property; - - property.property_id = property_id; - property.value = *value; - property.pspec = *pspec; - priv->property_queue.push(property); - + value_copy = (GValue *) g_malloc(sizeof(GValue)); + *value_copy = G_VALUE_INIT; + value_copy = g_value_init (value_copy, pspec->value_type); + g_value_copy (value, value_copy); + g_param_spec_ref (pspec); + priv->property_queue->push({value_copy, pspec}); + GST_INFO_OBJECT (self, "Queueing property: %s\n", pspec->name); } g_mutex_unlock (&priv->backend_mutex); @@ -198,6 +198,7 @@ gboolean gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; + InferenceProperty property; g_return_val_if_fail (priv, FALSE); g_return_val_if_fail (model_location, FALSE); @@ -234,25 +235,37 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) goto error; } - g_mutex_lock (&priv->backend_mutex); + priv->params = priv->factory->MakeParameters (error); + if (error.IsError ()) { + goto error; + } + error = priv->params->Configure(priv->engine, priv->model); + if (error.IsError ()) { + goto error; + } - while (!priv->property_queue.empty()) { - InferenceProperty property = priv->property_queue.front(); - switch (property.pspec.value_type) { + g_mutex_lock (&priv->backend_mutex); + while (!priv->property_queue->empty()) { + property = priv->property_queue->front(); + switch (property.pspec->value_type) { case G_TYPE_STRING: - priv->params->Set(property.pspec.name, g_value_get_string(&property.value)); + GST_INFO_OBJECT (self, "Setting property: %s=%s\n", property.pspec->name, + g_value_get_string(property.value)); + priv->params->Set(property.pspec->name, g_value_get_string(property.value)); break; case G_TYPE_INT: - priv->params->Set(property.pspec.name, g_value_get_int(&property.value)); + GST_INFO_OBJECT (self, "Setting property: %s=%d\n", property.pspec->name, + g_value_get_int(property.value)); + priv->params->Set(property.pspec->name, g_value_get_int(property.value)); break; default: GST_WARNING_OBJECT (self, "Invalid property type"); break; } - - priv->property_queue.pop(); + g_param_spec_unref (property.pspec); + g_free(property.value); + priv->property_queue->pop(); } - priv->backend_started = true; g_mutex_unlock (&priv->backend_mutex); From 2b649a977056d0dcf946efe58770608f35affff4 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Wed, 23 Jan 2019 16:51:43 -0600 Subject: [PATCH 37/49] Add param write before start flag support --- gst-libs/gst/r2inference/gstbackend.cc | 58 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 90e30387..8e2ae0b2 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -26,7 +26,7 @@ #include #include -#include +#include GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category @@ -47,7 +47,7 @@ struct _GstBackendPrivate { std::unique_ptr < r2i::IFrameworkFactory > factory; GMutex backend_mutex; gboolean backend_started; - std::shared_ptr < std::queue > property_queue; + std::shared_ptr < std::list > property_list; }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -78,7 +78,7 @@ gst_backend_init (GstBackend *self) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); g_mutex_init(&priv->backend_mutex); priv->backend_started = false; - priv->property_queue = std::make_shared>(); + priv->property_list = std::make_shared>(); } static void @@ -178,7 +178,7 @@ gst_backend_set_property (GObject *object, guint property_id, value_copy = g_value_init (value_copy, pspec->value_type); g_value_copy (value, value_copy); g_param_spec_ref (pspec); - priv->property_queue->push({value_copy, pspec}); + priv->property_list->push_back({value_copy, pspec}); GST_INFO_OBJECT (self, "Queueing property: %s\n", pspec->name); } g_mutex_unlock (&priv->backend_mutex); @@ -199,6 +199,9 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; InferenceProperty property; + std::list::iterator property_it; + static std::vector params; + static std::vector::iterator param_it; g_return_val_if_fail (priv, FALSE); g_return_val_if_fail (model_location, FALSE); @@ -230,23 +233,56 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) goto error; } - error = priv->engine->Start (); + priv->params = priv->factory->MakeParameters (error); if (error.IsError ()) { goto error; } - - priv->params = priv->factory->MakeParameters (error); + error = priv->params->Configure(priv->engine, priv->model); if (error.IsError ()) { goto error; } - error = priv->params->Configure(priv->engine, priv->model); + error = priv->params->List (params); if (error.IsError ()) { goto error; } g_mutex_lock (&priv->backend_mutex); - while (!priv->property_queue->empty()) { - property = priv->property_queue->front(); + for (property_it = priv->property_list->begin(); + property_it != priv->property_list->end(); ++property_it) { + for (param_it = params.begin(); param_it != params.end(); ++param_it) { + if (!g_strcmp0(property_it->pspec->name, param_it->name.c_str())) { + if (r2i::ParameterMeta::Flags::WRITE_BEFORE_START & param_it->flags) { + switch (property_it->pspec->value_type) { + case G_TYPE_STRING: + GST_INFO_OBJECT (self, "Setting property: %s=%s\n", property_it->pspec->name, + g_value_get_string(property_it->value)); + priv->params->Set(property_it->pspec->name, + g_value_get_string(property_it->value)); + break; + case G_TYPE_INT: + GST_INFO_OBJECT (self, "Setting property: %s=%d\n", property_it->pspec->name, + g_value_get_int(property.value)); + priv->params->Set(property_it->pspec->name, + g_value_get_int(property_it->value)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + priv->property_list->erase(property_it--); + } + break; + } + } + } + + error = priv->engine->Start (); + if (error.IsError ()) { + goto error; + } + + while (!priv->property_list->empty()) { + property = priv->property_list->front(); switch (property.pspec->value_type) { case G_TYPE_STRING: GST_INFO_OBJECT (self, "Setting property: %s=%s\n", property.pspec->name, @@ -264,7 +300,7 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) } g_param_spec_unref (property.pspec); g_free(property.value); - priv->property_queue->pop(); + priv->property_list->pop_front(); } priv->backend_started = true; g_mutex_unlock (&priv->backend_mutex); From 488bfed4866fe18c301e621c5b758b9c6137d58c Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Wed, 23 Jan 2019 21:34:34 +0000 Subject: [PATCH 38/49] Remove limitation to edit string parameters --- gst-libs/gst/r2inference/gstbackend.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 8e2ae0b2..d5786620 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -110,7 +110,7 @@ gst_backend_install_properties (GstBackendClass *klass, static int gst_backend_param_flags (int flags) { - int pflags = G_PARAM_STATIC_STRINGS; + int pflags = 0; if (r2i::ParameterMeta::Flags::READ & flags) { pflags += G_PARAM_READABLE; From b0a663cf0a93ef922962cefa57dad1593254c7e3 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Wed, 23 Jan 2019 23:09:46 +0000 Subject: [PATCH 39/49] Add get parameters logic --- gst-libs/gst/r2inference/gstbackend.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index d5786620..3f469b4d 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -189,9 +189,23 @@ void gst_backend_get_property (GObject *object, guint property_id, GValue *value, GParamSpec *pspec) { GstBackend *self = GST_BACKEND (object); - + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + int int_buffer; + std::string string_buffer; GST_DEBUG_OBJECT (self, "get_property"); + if (NULL != priv->params ){ + switch (pspec->value_type) { + case G_TYPE_STRING: + priv->params->Get (pspec->name, string_buffer); + g_value_set_enum (value, int_buffer); + break; + case G_TYPE_INT: + priv->params->Get (pspec->name, int_buffer); + g_value_set_string (value, string_buffer.c_str()); + break; + } + } } gboolean From 123d69fe365206be8e8711f1528ce867e9c9d680 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Wed, 23 Jan 2019 20:29:15 -0600 Subject: [PATCH 40/49] Fix indentation --- gst-libs/gst/r2inference/gstbackend.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 3f469b4d..5e48c0be 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -194,7 +194,7 @@ gst_backend_get_property (GObject *object, guint property_id, std::string string_buffer; GST_DEBUG_OBJECT (self, "get_property"); - if (NULL != priv->params ){ + if (NULL != priv->params ) { switch (pspec->value_type) { case G_TYPE_STRING: priv->params->Get (pspec->name, string_buffer); @@ -209,7 +209,8 @@ gst_backend_get_property (GObject *object, guint property_id, } gboolean -gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) { +gst_backend_start (GstBackend *self, const gchar *model_location, + GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; InferenceProperty property; @@ -322,8 +323,8 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) return TRUE; error: g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), - "R2Inference Error: (Code:%d) %s", error.GetCode (), - error.GetDescription ().c_str ()); + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); return FALSE; } @@ -397,13 +398,12 @@ gst_backend_get_framework_code (GstBackend *backend) { } GQuark -gst_backend_error_quark(void) -{ +gst_backend_error_quark(void) { static GQuark q = 0; if (0 == q) { q = g_quark_from_static_string("gst-backend-error-quark"); } - + return q; } From af493415746474f2627b6ddd6cfa1e8f826b8569 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Thu, 24 Jan 2019 12:08:00 -0600 Subject: [PATCH 41/49] Change inference properties from struct to class --- gst-libs/gst/r2inference/gstbackend.cc | 110 ++++++++++++++----------- 1 file changed, 60 insertions(+), 50 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 5e48c0be..cdf78bde 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -31,11 +31,54 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category -typedef struct { - GValue *value; - GParamSpec *pspec; -} InferenceProperty; +class InferenceProperty { + private: + GValue *avalue; + GParamSpec *apspec; + + public: + + InferenceProperty() { + } + + InferenceProperty(const GValue *value, GParamSpec *pspec) { + avalue = (GValue *) g_malloc(sizeof(GValue)); + *avalue = G_VALUE_INIT; + avalue = g_value_init (avalue, pspec->value_type); + g_value_copy (value, avalue); + g_param_spec_ref (pspec); + apspec = pspec; + } + + ~InferenceProperty() { + g_param_spec_unref (apspec); + g_free(avalue); + } + + void apply_inference_property(GstBackend *self, + std::shared_ptr params) { + switch (apspec->value_type) { + case G_TYPE_STRING: + GST_INFO_OBJECT (self, "Setting property: %s=%s\n", apspec->name, + g_value_get_string(avalue)); + params->Set(apspec->name, g_value_get_string(avalue)); + break; + case G_TYPE_INT: + GST_INFO_OBJECT (self, "Setting property: %s=%d\n", apspec->name, + g_value_get_int(avalue)); + params->Set(apspec->name, g_value_get_int(avalue)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + } + + const gchar *get_name() { + return apspec->name; + } +}; typedef struct _GstBackendPrivate GstBackendPrivate; struct _GstBackendPrivate { @@ -47,7 +90,7 @@ struct _GstBackendPrivate { std::unique_ptr < r2i::IFrameworkFactory > factory; GMutex backend_mutex; gboolean backend_started; - std::shared_ptr < std::list > property_list; + std::shared_ptr < std::list > property_list; }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -78,7 +121,7 @@ gst_backend_init (GstBackend *self) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); g_mutex_init(&priv->backend_mutex); priv->backend_started = false; - priv->property_list = std::make_shared>(); + priv->property_list = std::make_shared>(); } static void @@ -155,7 +198,7 @@ gst_backend_set_property (GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { GstBackend *self = GST_BACKEND (object); GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); - GValue *value_copy; + InferenceProperty *property; GST_DEBUG_OBJECT (self, "set_property"); @@ -173,12 +216,8 @@ gst_backend_set_property (GObject *object, guint property_id, break; } } else { - value_copy = (GValue *) g_malloc(sizeof(GValue)); - *value_copy = G_VALUE_INIT; - value_copy = g_value_init (value_copy, pspec->value_type); - g_value_copy (value, value_copy); - g_param_spec_ref (pspec); - priv->property_list->push_back({value_copy, pspec}); + property = new InferenceProperty(value, pspec); + priv->property_list->push_back(property); GST_INFO_OBJECT (self, "Queueing property: %s\n", pspec->name); } g_mutex_unlock (&priv->backend_mutex); @@ -213,8 +252,8 @@ gst_backend_start (GstBackend *self, const gchar *model_location, GError **err) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); r2i::RuntimeError error; - InferenceProperty property; - std::list::iterator property_it; + InferenceProperty *property; + std::list::iterator property_it; static std::vector params; static std::vector::iterator param_it; @@ -264,26 +303,12 @@ gst_backend_start (GstBackend *self, const gchar *model_location, g_mutex_lock (&priv->backend_mutex); for (property_it = priv->property_list->begin(); property_it != priv->property_list->end(); ++property_it) { + property = *property_it; for (param_it = params.begin(); param_it != params.end(); ++param_it) { - if (!g_strcmp0(property_it->pspec->name, param_it->name.c_str())) { + if (!g_strcmp0(property->get_name(), param_it->name.c_str())) { if (r2i::ParameterMeta::Flags::WRITE_BEFORE_START & param_it->flags) { - switch (property_it->pspec->value_type) { - case G_TYPE_STRING: - GST_INFO_OBJECT (self, "Setting property: %s=%s\n", property_it->pspec->name, - g_value_get_string(property_it->value)); - priv->params->Set(property_it->pspec->name, - g_value_get_string(property_it->value)); - break; - case G_TYPE_INT: - GST_INFO_OBJECT (self, "Setting property: %s=%d\n", property_it->pspec->name, - g_value_get_int(property.value)); - priv->params->Set(property_it->pspec->name, - g_value_get_int(property_it->value)); - break; - default: - GST_WARNING_OBJECT (self, "Invalid property type"); - break; - } + property->apply_inference_property(self, priv->params); + property->~InferenceProperty(); priv->property_list->erase(property_it--); } break; @@ -298,23 +323,8 @@ gst_backend_start (GstBackend *self, const gchar *model_location, while (!priv->property_list->empty()) { property = priv->property_list->front(); - switch (property.pspec->value_type) { - case G_TYPE_STRING: - GST_INFO_OBJECT (self, "Setting property: %s=%s\n", property.pspec->name, - g_value_get_string(property.value)); - priv->params->Set(property.pspec->name, g_value_get_string(property.value)); - break; - case G_TYPE_INT: - GST_INFO_OBJECT (self, "Setting property: %s=%d\n", property.pspec->name, - g_value_get_int(property.value)); - priv->params->Set(property.pspec->name, g_value_get_int(property.value)); - break; - default: - GST_WARNING_OBJECT (self, "Invalid property type"); - break; - } - g_param_spec_unref (property.pspec); - g_free(property.value); + property->apply_inference_property(self, priv->params); + property->~InferenceProperty(); priv->property_list->pop_front(); } priv->backend_started = true; From 1aa744e543a1338834c54d148220bf007fad8d4a Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Thu, 24 Jan 2019 19:23:54 +0000 Subject: [PATCH 42/49] Change default backend to TensorFlow --- gst-libs/gst/r2inference/gstvideoinference.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 842795c0..1f92dfbf 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -43,7 +43,7 @@ GST_STATIC_PAD_TEMPLATE ("src_bypass", GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category -#define DEFAULT_BACKEND 0 +#define DEFAULT_BACKEND 1 #define DEFAULT_MODEL_LOCATION NULL enum From 9be1b144cae23361abea06892de61d1cf3fd3af4 Mon Sep 17 00:00:00 2001 From: Jose Jimenez Date: Mon, 28 Jan 2019 12:27:41 -0600 Subject: [PATCH 43/49] Handle mutex unlock in case of error in Engine Start or Param Set --- gst-libs/gst/r2inference/gstbackend.cc | 28 +++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index cdf78bde..a9986889 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -57,17 +57,17 @@ class InferenceProperty { } void apply_inference_property(GstBackend *self, - std::shared_ptr params) { + std::shared_ptr params, r2i::RuntimeError &error) { switch (apspec->value_type) { case G_TYPE_STRING: GST_INFO_OBJECT (self, "Setting property: %s=%s\n", apspec->name, g_value_get_string(avalue)); - params->Set(apspec->name, g_value_get_string(avalue)); + error = params->Set(apspec->name, g_value_get_string(avalue)); break; case G_TYPE_INT: GST_INFO_OBJECT (self, "Setting property: %s=%d\n", apspec->name, g_value_get_int(avalue)); - params->Set(apspec->name, g_value_get_int(avalue)); + error = params->Set(apspec->name, g_value_get_int(avalue)); break; default: GST_WARNING_OBJECT (self, "Invalid property type"); @@ -264,39 +264,47 @@ gst_backend_start (GstBackend *self, const gchar *model_location, priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, error); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend library"); goto error; } priv->engine = priv->factory->MakeEngine (error); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend engine"); goto error; } priv->loader = priv->factory->MakeLoader (error); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the model loader"); goto error; } priv->model = priv->loader->Load (model_location, error); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to load model"); goto error; } error = priv->engine->SetModel (priv->model); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set model to engine"); goto error; } priv->params = priv->factory->MakeParameters (error); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set get parameters for backend"); goto error; } error = priv->params->Configure(priv->engine, priv->model); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to configure mode to backend"); goto error; } error = priv->params->List (params); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to list the backend parameters"); goto error; } @@ -307,9 +315,13 @@ gst_backend_start (GstBackend *self, const gchar *model_location, for (param_it = params.begin(); param_it != params.end(); ++param_it) { if (!g_strcmp0(property->get_name(), param_it->name.c_str())) { if (r2i::ParameterMeta::Flags::WRITE_BEFORE_START & param_it->flags) { - property->apply_inference_property(self, priv->params); + property->apply_inference_property(self, priv->params, error); property->~InferenceProperty(); priv->property_list->erase(property_it--); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set backend parameters"); + goto error; + } } break; } @@ -318,20 +330,26 @@ gst_backend_start (GstBackend *self, const gchar *model_location, error = priv->engine->Start (); if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend engine"); goto error; } while (!priv->property_list->empty()) { property = priv->property_list->front(); - property->apply_inference_property(self, priv->params); + property->apply_inference_property(self, priv->params, error); property->~InferenceProperty(); priv->property_list->pop_front(); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set backend parameters"); + goto error; + } } priv->backend_started = true; g_mutex_unlock (&priv->backend_mutex); return TRUE; error: + g_mutex_unlock (&priv->backend_mutex); g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), "R2Inference Error: (Code:%d) %s", error.GetCode (), error.GetDescription ().c_str ()); From ff3816431e14fc47060ba8fe81fa3af7c42fa3ec Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Tue, 29 Jan 2019 15:49:41 -0600 Subject: [PATCH 44/49] Get default backend from r2inference --- gst-libs/gst/r2inference/gstinferencebackends.cc | 13 +++++++++++++ gst-libs/gst/r2inference/gstinferencebackends.h | 1 + gst-libs/gst/r2inference/gstvideoinference.c | 10 +++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index 79c5c0e0..324d23c9 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -120,3 +120,16 @@ gst_inference_backends_get_string_properties (void) return backends_parameters; } + +guint16 +gst_inference_backends_get_default_backend (void) +{ + std::vector backends; + r2i::FrameworkCode code; + r2i::RuntimeError error; + + backends = r2i::IFrameworkFactory::List (error); + code = backends.front().code; + + return code; +} diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h index 80cf88db..8d8bc8c9 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.h +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -30,6 +30,7 @@ G_BEGIN_DECLS GType gst_inference_backends_get_type (void); gchar * gst_inference_backends_get_string_properties (void); +guint16 gst_inference_backends_get_default_backend (void); GType gst_inference_backends_search_type (guint); G_END_DECLS diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 1f92dfbf..002b05eb 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -43,7 +43,6 @@ GST_STATIC_PAD_TEMPLATE ("src_bypass", GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category -#define DEFAULT_BACKEND 1 #define DEFAULT_MODEL_LOCATION NULL enum @@ -159,9 +158,9 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) g_free (backends_params); g_object_class_install_property (oclass, PROP_BACKEND, - g_param_spec_enum ("backend", "Backend", - backend_blurb, GST_TYPE_INFERENCE_BACKENDS, DEFAULT_BACKEND, - G_PARAM_READWRITE)); + g_param_spec_enum ("backend", "Backend", backend_blurb, + GST_TYPE_INFERENCE_BACKENDS, + gst_inference_backends_get_default_backend (), G_PARAM_READWRITE)); g_object_class_install_property (oclass, PROP_MODEL_LOCATION, g_param_spec_string ("model-location", "Model Location", @@ -195,7 +194,8 @@ gst_video_inference_init (GstVideoInference * self) priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); - gst_video_inference_set_backend (self, DEFAULT_BACKEND); + gst_video_inference_set_backend (self, + gst_inference_backends_get_default_backend ()); } static void From f4ef78b1c24dd783717f06d0038dd0893916c06e Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Wed, 30 Jan 2019 09:45:32 -0600 Subject: [PATCH 45/49] Fix attempt to unlock mutex that was not locked --- gst-libs/gst/r2inference/gstbackend.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index a9986889..b1a9d5d5 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -331,7 +331,7 @@ gst_backend_start (GstBackend *self, const gchar *model_location, error = priv->engine->Start (); if (error.IsError ()) { GST_ERROR_OBJECT (self, "Failed to start the backend engine"); - goto error; + goto start_error; } while (!priv->property_list->empty()) { @@ -348,8 +348,10 @@ gst_backend_start (GstBackend *self, const gchar *model_location, g_mutex_unlock (&priv->backend_mutex); return TRUE; -error: + +start_error: g_mutex_unlock (&priv->backend_mutex); +error: g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), "R2Inference Error: (Code:%d) %s", error.GetCode (), error.GetDescription ().c_str ()); From 30833ac3a3400654127b207e8fa578b7e80d9268 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Wed, 30 Jan 2019 10:08:51 -0600 Subject: [PATCH 46/49] Add a flag to create the backend only once --- gst-libs/gst/r2inference/gstbackend.cc | 87 ++++++++++++++------------ 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index b1a9d5d5..3cd08d9e 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -91,6 +91,8 @@ struct _GstBackendPrivate { GMutex backend_mutex; gboolean backend_started; std::shared_ptr < std::list > property_list; + gboolean backend_created; + }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -121,6 +123,7 @@ gst_backend_init (GstBackend *self) { GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); g_mutex_init(&priv->backend_mutex); priv->backend_started = false; + priv->backend_created = false; priv->property_list = std::make_shared>(); } @@ -261,51 +264,55 @@ gst_backend_start (GstBackend *self, const gchar *model_location, g_return_val_if_fail (model_location, FALSE); g_return_val_if_fail (err, FALSE); - priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, - error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to start the backend library"); - goto error; - } - priv->engine = priv->factory->MakeEngine (error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to start the backend engine"); - goto error; - } + if (!priv->backend_created) { + priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, + error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend library"); + goto error; + } - priv->loader = priv->factory->MakeLoader (error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to start the model loader"); - goto error; - } + priv->engine = priv->factory->MakeEngine (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend engine"); + goto error; + } - priv->model = priv->loader->Load (model_location, error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to load model"); - goto error; - } + priv->loader = priv->factory->MakeLoader (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the model loader"); + goto error; + } - error = priv->engine->SetModel (priv->model); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to set model to engine"); - goto error; - } + priv->model = priv->loader->Load (model_location, error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to load model"); + goto error; + } - priv->params = priv->factory->MakeParameters (error); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to set get parameters for backend"); - goto error; - } - error = priv->params->Configure(priv->engine, priv->model); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to configure mode to backend"); - goto error; - } - error = priv->params->List (params); - if (error.IsError ()) { - GST_ERROR_OBJECT (self, "Failed to list the backend parameters"); - goto error; + error = priv->engine->SetModel (priv->model); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set model to engine"); + goto error; + } + + priv->params = priv->factory->MakeParameters (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set get parameters for backend"); + goto error; + } + error = priv->params->Configure(priv->engine, priv->model); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to configure mode to backend"); + goto error; + } + error = priv->params->List (params); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to list the backend parameters"); + goto error; + } + priv->backend_created = true; } g_mutex_lock (&priv->backend_mutex); From 93075465ab65eae03d533da7db9fea020b6191b9 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Wed, 30 Jan 2019 10:28:23 -0600 Subject: [PATCH 47/49] Add backend stop and call it from videinference --- gst-libs/gst/r2inference/gstbackend.cc | 24 +++++++++++++++++++- gst-libs/gst/r2inference/gstbackend.h | 1 + gst-libs/gst/r2inference/gstvideoinference.c | 9 +++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 3cd08d9e..22df5841 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -92,7 +92,7 @@ struct _GstBackendPrivate { gboolean backend_started; std::shared_ptr < std::list > property_list; gboolean backend_created; - + }; G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, @@ -365,6 +365,28 @@ gst_backend_start (GstBackend *self, const gchar *model_location, return FALSE; } +gboolean +gst_backend_stop (GstBackend *self, GError **err) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + r2i::RuntimeError error; + + g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (err, FALSE); + + error = priv->engine->Stop (); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to stop the backend engine"); + goto error; + } + return TRUE; + +error: + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + gboolean gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, gpointer *prediction_data, gsize *prediction_size, GError **err) { diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 21d41c6e..933bb21f 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -37,6 +37,7 @@ struct _GstBackendClass GQuark gst_backend_error_quark (void); gboolean gst_backend_start (GstBackend *, const gchar *, GError **); +gboolean gst_backend_stop (GstBackend *, GError **); guint gst_backend_get_framework_code (GstBackend *); gboolean gst_backend_process_frame (GstBackend *, GstVideoFrame *, gpointer *, gsize *, GError **); diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 002b05eb..a257bdc4 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -311,7 +311,6 @@ gst_video_inference_start (GstVideoInference * self) GError *err = NULL; GST_INFO_OBJECT (self, "Starting video inference"); - if (NULL == priv->model_location) { GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, ("Model Location has not been set"), (NULL)); @@ -339,10 +338,18 @@ static gboolean gst_video_inference_stop (GstVideoInference * self) { GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); gboolean ret = TRUE; + GError *err = NULL; GST_INFO_OBJECT (self, "Stopping video inference"); + if (!gst_backend_stop (priv->backend, &err)) { + GST_ELEMENT_ERROR (self, LIBRARY, INIT, + ("Could not stop the selected backend: (%s)", err->message), (NULL)); + ret = FALSE; + } + if (klass->stop != NULL) { ret = klass->stop (self); } From 87d21afad06fa82d008e64cd5e522543e64bd2f2 Mon Sep 17 00:00:00 2001 From: Greivin Fallas Sanchez Date: Thu, 31 Jan 2019 17:39:30 +0000 Subject: [PATCH 48/49] Update googlenet plugin to inception version 4 --- ext/r2inference/gstgooglenet.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c index 1f36469f..f019fa82 100644 --- a/ext/r2inference/gstgooglenet.c +++ b/ext/r2inference/gstgooglenet.c @@ -47,11 +47,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); /* prototypes */ #define CHANNELS 3 -const float GoogleNetMean[] = { 0.40787054 * 255.0, - 0.45752458 * 255.0, - 0.48109378 * 255.0 -}; - static void gst_googlenet_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); static void gst_googlenet_get_property (GObject * object, @@ -73,7 +68,7 @@ enum /* pad templates */ -#define CAPS "video/x-raw,format=BGR,width=224,height=224" +#define CAPS "video/x-raw,format=BGR,width=299,height=299" static GstStaticPadTemplate sink_model_factory = GST_STATIC_PAD_TEMPLATE ("sink_model", @@ -204,12 +199,14 @@ gst_googlenet_preprocess (GstVideoInference * vi, for (gint i = 0; i < size; i += CHANNELS) { /* BGR = RGB - Mean */ + ((gfloat *) outframe->data[0])[i + 0] = - (((guchar *) inframe->data[0])[i + 2]) - GoogleNetMean[0]; + (((guchar *) inframe->data[0])[i + 0] - 128) / 128.0; ((gfloat *) outframe->data[0])[i + 1] = - (((guchar *) inframe->data[0])[i + 1]) - GoogleNetMean[1]; + (((guchar *) inframe->data[0])[i + 1] - 128) / 128.0; ((gfloat *) outframe->data[0])[i + 2] = - (((guchar *) inframe->data[0])[i + 0]) - GoogleNetMean[2]; + (((guchar *) inframe->data[0])[i + 2] - 128) / 128.0; + } return TRUE; From d0b5cf3ab10c03724a16f475afd9c1a79dcb2749 Mon Sep 17 00:00:00 2001 From: Miguel Taylor Date: Fri, 1 Feb 2019 17:01:48 -0600 Subject: [PATCH 49/49] Point README to official documentation --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c9b612a..da5e7c26 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # GstInference -> A GStreamer deep learning inference framework. +A GStreamer deep learning inference framework. -Stay tuned for the official release! \ No newline at end of file +Please visit the official documentation hosted at: +> http://developer.ridgerun.com/wiki/index.php?title=GstInference