diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c3beb265 --- /dev/null +++ b/.gitignore @@ -0,0 +1,56 @@ +# Editor backup +*~ +*.bak +\#* +*.orig + +# Autotools cruft +aclocal.m4 +autom4te.cache/ +compile +config.guess +config.h +config.h.in +config.status +config.sub +configure +install-sh +missing +Makefile +Makefile.in +depcomp +libtool +ltmain.sh +m4/ +.deps/ +.libs/ +stamp-h* +autoregen.sh +gstinference-*.pc + +# Build outputs +.deps/ +.libs/ +*.la +*.lo +*.o + +# Git conflict files +.merge_file_* + +# GtkDoc +docs/version.entities + +# Tests +test-driver +*.xml +*.log +*.trs + +# 3rd party tools +.clang_complete +cscope.files +cscope.out + +# Examples + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..3ff9d95e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "common"] + path = common + url = https://anongit.freedesktop.org/git/gstreamer/common.git diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..9e7f24e8 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,3 @@ +Carlos Rodriguez +Jose Jimenez +Michael Gruner diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 00000000..642f772d --- /dev/null +++ b/ChangeLog @@ -0,0 +1 @@ +=== release 0.1.0 === diff --git a/MAINTAINERS b/MAINTAINERS new file mode 100644 index 00000000..3fa45235 --- /dev/null +++ b/MAINTAINERS @@ -0,0 +1,11 @@ +GStreamer is currently maintained by the consensus of a number +of people, including, but not limited to: + +Carlos Rodriguez +Jose Jimenez +Michael Gruner + + +Maintainer-related issues should be addressed to: + + support@ridgerun.com diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 00000000..a33a6965 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,56 @@ +DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc + +ALWAYS_SUBDIRS = \ + gst-libs \ + ext \ + tests \ + common \ + docs \ + m4 + +SUBDIRS = $(ALWAYS_SUBDIRS) + +DIST_SUBDIRS = $(ALWAYS_SUBDIRS) + +EXTRA_DIST = \ + depcomp \ + AUTHORS NEWS README REQUIREMENTS \ + ChangeLog gst-inference.doap autogen.sh + +DISTCLEANFILES = _stdint.h + +noinst_HEADERS = + +ACLOCAL_AMFLAGS = -I m4 -I common/m4 + +include $(top_srcdir)/common/release.mak + +check-valgrind: + $(MAKE) -C tests/check check-valgrind + +if HAVE_GST_CHECK +check-torture: + $(MAKE) -C tests/check torture + +build-checks: + $(MAKE) -C tests/check build-checks +else +check-torture: + true +build-checks: + true +endif + +include $(top_srcdir)/common/coverage/lcov.mak + +# cruft: plugins that have been merged or moved or renamed + +CRUFT_FILES = \ + $(top_builddir)/common/shave \ + $(top_builddir)/common/shave-libtool + +CRUFT_DIRS = + +include $(top_srcdir)/common/cruft.mak + +all-local: check-cruft diff --git a/NEWS b/NEWS new file mode 100644 index 00000000..2ae77312 --- /dev/null +++ b/NEWS @@ -0,0 +1 @@ +# GstInference 0.1 Release Notes diff --git a/README.md b/README.md index 75d50a07..da5e7c26 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ # GstInference A GStreamer deep learning inference framework. + +Please visit the official documentation hosted at: +> http://developer.ridgerun.com/wiki/index.php?title=GstInference diff --git a/REQUIREMENTS b/REQUIREMENTS new file mode 100644 index 00000000..bd7e6748 --- /dev/null +++ b/REQUIREMENTS @@ -0,0 +1,54 @@ +GStreamer uses a *large* array of tools and libraries, most of which are +optional. We have attempted to make sure that any code that depends on +optional libraries doesn't get built unless you have those libraries. If +you find this not to be the case, please, let us know by filing a bug +report at http://bugzilla.gnome.org/. + +Required tools: +=============== + +An extra set of tools is required if you wish to build GStreamer +from git (using autogen.sh): + +autoconf >= 2.68 https://www.gnu.org/software/autoconf/ +automake >= 1.11 https://www.gnu.org/software/automake/ +libtool >= 2.2.6 https://www.gnu.org/software/libtool/ +pkgconfig >= 0.9.0 https://www.freedesktop.org/software/pkgconfig/ + +Required libraries: +=================== + +Package: GStreamer +Version: 1.x (same 1.x version as this package) +Recommended: Latest 1.x +URL: http://gstreamer.freedesktop.org/ +DebianPackage: libgstreamer1.0-dev +Notes: The required version is updated frequently, so the version + listed in this file is often out of date. If you are compiling + from git master, you will usually need GStreamer core and + gst-plugins-base from git master as well. + +Package: GStreamer Base Plugins +Version: 1.x (same 1.x version as this package) +Recommended: Latest 1.x +URL: http://gstreamer.freedesktop.org/ +DebianPackage: libgstreamer-plugins-base1.0-dev +Notes: The required version is updated frequently, so the version + listed in this file is often out of date. If you are compiling + from git master, you will usually need GStreamer core and + gst-plugins-base from git master as well. + + +Optional libraries: +=================== + +This file lists supporting libraries for which gst-plugins-good contains +plugins, as well as their minimum required version. You can find the +corresponding plugins in ext/(library) + + +Optional (debian) packages: +=========================== + +gtk-doc-tools >= 1.12 -- needed to build documentation +python-xml -- needed to build plugin documentation diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..a6237b6e --- /dev/null +++ b/autogen.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# gst-inference autogen.sh +# +# Run this to generate all the initial makefiles, etc. +# +# This file has been generated from common/autogen.sh.in via common/update-autogen + + +test -n "$srcdir" || srcdir=`dirname "$0"` +test -n "$srcdir" || srcdir=. + +olddir=`pwd` +cd "$srcdir" + +package=gst-inference +srcfile=gst-inference.doap + +# Make sure we have common +if test ! -f common/gst-autogen.sh; +then + echo "+ Setting up common submodule" + git submodule init +fi +git submodule update + +# source helper functions +if test ! -f common/gst-autogen.sh; +then + echo There is something wrong with your source tree. + echo You are missing common/gst-autogen.sh + exit 1 +fi +. common/gst-autogen.sh + +# install pre-commit hook for doing clean commits +if test ! \( -x .git/hooks/pre-commit -a -L .git/hooks/pre-commit \); +then + rm -f .git/hooks/pre-commit + if ! ln -s ../../common/hooks/pre-commit.hook .git/hooks/pre-commit 2> /dev/null + then + echo "Failed to create commit hook symlink, copying instead ..." + cp common/hooks/pre-commit.hook .git/hooks/pre-commit + fi +fi + +CONFIGURE_DEF_OPT='--enable-maintainer-mode --enable-gtk-doc' + +if test "x$package" = "xgstreamer"; then + CONFIGURE_DEF_OPT="$CONFIGURE_DEF_OPT --enable-failing-tests --enable-poisoning" +elif test "x$package" = "xgst-plugins-bad"; then + CONFIGURE_DEF_OPT="$CONFIGURE_DEF_OPT --with-player-tests" +fi + +autogen_options $@ + +printf "+ check for build tools" +if test -z "$NOCHECK"; then + echo + + printf " checking for autoreconf ... " + echo + which "autoreconf" 2>/dev/null || { + echo "not found! Please install the autoconf package." + exit 1 + } + + printf " checking for pkg-config ... " + echo + which "pkg-config" 2>/dev/null || { + echo "not found! Please install pkg-config." + exit 1 + } +else + echo ": skipped version checks" +fi + +# if no arguments specified then this will be printed +if test -z "$*" && test -z "$NOCONFIGURE"; then + echo "+ checking for autogen.sh options" + echo " This autogen script will automatically run ./configure as:" + echo " ./configure $CONFIGURE_DEF_OPT" + echo " To pass any additional options, please specify them on the $0" + echo " command line." +fi + +toplevel_check $srcfile + +# aclocal +if test -f acinclude.m4; then rm acinclude.m4; fi + +autoreconf --force --install || exit 1 + +test -n "$NOCONFIGURE" && { + echo "+ skipping configure stage for package $package, as requested." + echo "+ autogen.sh done." + exit 0 +} + +cd "$olddir" + +echo "+ running configure ... " +test ! -z "$CONFIGURE_DEF_OPT" && echo " default flags: $CONFIGURE_DEF_OPT" +test ! -z "$CONFIGURE_EXT_OPT" && echo " external flags: $CONFIGURE_EXT_OPT" +echo + +echo "$srcdir/configure" $CONFIGURE_DEF_OPT $CONFIGURE_EXT_OPT +"$srcdir/configure" $CONFIGURE_DEF_OPT $CONFIGURE_EXT_OPT || { + echo " configure failed" + exit 1 +} + +echo "Now type 'make' to compile $package." diff --git a/common b/common new file mode 160000 index 00000000..cd1dee06 --- /dev/null +++ b/common @@ -0,0 +1 @@ +Subproject commit cd1dee06bf07f094677d0cf3eea4a2e8c2636b24 diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..bfa7cd94 --- /dev/null +++ b/configure.ac @@ -0,0 +1,365 @@ +AC_PREREQ([2.69]) + +dnl please read gstreamer/docs/random/autotools before changing this file + +dnl initialize autoconf +dnl releases only do -Wall, git and prerelease does -Werror too +dnl use a three digit version number for releases, and four for git/pre +AC_INIT([GStreamer Inference],[0.1.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) + +AG_GST_INIT + +dnl initialize automake +AM_INIT_AUTOMAKE([-Wno-portability 1.14 no-dist-gzip dist-xz tar-ustar subdir-objects foreign]) + +dnl define PACKAGE_VERSION_* variables +AS_VERSION + +dnl check if this is a release version +AS_NANO(GST_GIT="no", GST_GIT="yes") + +dnl can autoconf find the source ? +AC_CONFIG_SRCDIR([ext/r2inference/gstinference.c]) + +dnl define the output header for config +AC_CONFIG_HEADERS([config.h]) + +dnl AM_MAINTAINER_MODE only provides the option to configure to enable it +AM_MAINTAINER_MODE([enable]) + +dnl sets host_* variables +AC_CANONICAL_HOST + +dnl use pretty build output with automake >= 1.11 +m4_ifdef([AM_SILENT_RULES],[AM_SILENT_RULES([yes])], + [AM_DEFAULT_VERBOSITY=1 + AC_SUBST(AM_DEFAULT_VERBOSITY)]) + +dnl our libraries and install dirs use GST_API_VERSION in the filename +dnl to allow side-by-side installation of different API versions +GST_API_VERSION=1.0 +AC_SUBST(GST_API_VERSION) +AC_DEFINE_UNQUOTED(GST_API_VERSION, "$GST_API_VERSION", + [GStreamer API Version]) + +AG_GST_LIBTOOL_PREPARE +AS_LIBTOOL(GST, 100, 0, 100) + +dnl *** required versions of GStreamer stuff *** +GST_REQ=1.8.0.1 +GSTPB_REQ=1.8.0.1 + +dnl *** autotools stuff **** + +dnl allow for different autotools +AS_AUTOTOOLS_ALTERNATE + +dnl Add parameters for aclocal +AC_SUBST(ACLOCAL_AMFLAGS, "-I m4 -I common/m4") + +dnl *** check for arguments to configure *** + +AG_GST_ARG_DISABLE_FATAL_WARNINGS +AG_GST_ARG_ENABLE_EXTRA_CHECKS + +AG_GST_ARG_DEBUG +AG_GST_ARG_PROFILING +AG_GST_ARG_VALGRIND +AG_GST_ARG_GCOV + +AG_GST_ARG_EXAMPLES + +AG_GST_ARG_WITH_PKG_CONFIG_PATH +AG_GST_ARG_WITH_PACKAGE_NAME +AG_GST_ARG_WITH_PACKAGE_ORIGIN + +AG_GST_ARG_WITH_PLUGINS + +AG_GST_ARG_ENABLE_EXTERNAL + +AG_GST_ARG_ENABLE_EXPERIMENTAL + +AG_GST_PKG_CONFIG_PATH + +dnl *** checks for platform *** + +dnl * hardware/architecture * + +dnl common/m4/gst-arch.m4 +dnl check CPU type +AG_GST_ARCH + +dnl Determine endianness +AC_C_BIGENDIAN + +dnl *** checks for programs *** + +dnl find a compiler +AC_PROG_CC +AC_PROG_CC_STDC + +dnl determine c++ compiler +AC_PROG_CXX +dnl determine if c++ is available on this system +AC_CHECK_PROG(HAVE_CXX, $CXX, yes, no) + +dnl determine c++ preprocessor +dnl FIXME: do we need this ? +AC_PROG_CXXCPP + +dnl check if the compiler supports '-c' and '-o' options +AM_PROG_CC_C_O + +dnl find an assembler +AM_PROG_AS + +dnl check if the compiler supports do while(0) macros +AG_GST_CHECK_DOWHILE_MACROS + +AC_PATH_PROG(VALGRIND_PATH, valgrind, no) +AM_CONDITIONAL(HAVE_VALGRIND, test ! "x$VALGRIND_PATH" = "xno") + +dnl check for documentation tools +GTK_DOC_CHECK([1.12]) +AG_GST_PLUGIN_DOCS([1.12]) + +dnl *** checks for libraries *** + +dnl check for pthreads +AX_PTHREAD + +dnl *** checks for header files *** + +dnl check if we have ANSI C header files +AC_HEADER_STDC + +dnl *** checks for types/defines *** + +dnl *** checks for structures *** + +dnl *** checks for compiler characteristics *** + +dnl *** checks for library functions *** + +dnl *** checks for dependency libraries *** + + +dnl GLib +GLIB_REQ=2.40.0 +AG_GST_GLIB_CHECK([$GLIB_REQ]) + +dnl checks for gstreamer +dnl uninstalled is selected preferentially -- see pkg-config(1) +AG_GST_CHECK_GST($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_BASE($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_CONTROLLER($GST_API_VERSION, [$GST_REQ], yes) +AG_GST_CHECK_GST_CHECK($GST_API_VERSION, [$GST_REQ], no) +AG_GST_CHECK_GST_PLUGINS_BASE($GST_API_VERSION, [$GSTPB_REQ], yes) + +GST_TOOLS_DIR=`$PKG_CONFIG --variable=toolsdir gstreamer-$GST_API_VERSION` +if test -z $GST_TOOLS_DIR; then + AC_MSG_ERROR([no tools dir defined in GStreamer pkg-config file; core upgrade needed.]) +fi +AC_SUBST(GST_TOOLS_DIR) + +AC_MSG_NOTICE(Using GStreamer Core Plugins in $GST_PLUGINS_DIR) +AC_MSG_NOTICE(Using GStreamer Base Plugins in $GSTPB_PLUGINS_DIR) + +AM_CONDITIONAL(HAVE_GST_CHECK, test "x$HAVE_GST_CHECK" = "xyes") + +dnl Check for documentation xrefs +GLIB_PREFIX="`$PKG_CONFIG --variable=prefix glib-2.0`" +GST_PREFIX="`$PKG_CONFIG --variable=prefix gstreamer-$GST_API_VERSION`" +GSTPB_PREFIX="`$PKG_CONFIG --variable=prefix gstreamer-plugins-base-$GST_API_VERSION`" +AC_SUBST(GLIB_PREFIX) +AC_SUBST(GST_PREFIX) +AC_SUBST(GSTPB_PREFIX) + +dnl Check for -Bsymbolic-functions linker flag used to avoid +dnl intra-library PLT jumps, if available. +AC_ARG_ENABLE(Bsymbolic, + [AS_HELP_STRING([--disable-Bsymbolic],[avoid linking with -Bsymbolic])],, + [SAVED_LDFLAGS="${LDFLAGS}" SAVED_LIBS="${LIBS}" + AC_MSG_CHECKING([for -Bsymbolic-functions linker flag]) + LDFLAGS=-Wl,-Bsymbolic-functions + LIBS= + AC_TRY_LINK([], [return 0], + AC_MSG_RESULT(yes) + enable_Bsymbolic=yes, + AC_MSG_RESULT(no) + enable_Bsymbolic=no) + LDFLAGS="${SAVED_LDFLAGS}" LIBS="${SAVED_LIBS}"]) + +dnl *** set variables based on configure arguments *** + +dnl set license and copyright notice +GST_LICENSE="Proprietary" +AC_DEFINE_UNQUOTED(GST_LICENSE, "$GST_LICENSE", [GStreamer Inference license]) +AC_SUBST(GST_LICENSE) + +dnl set location of plugin directory +AG_GST_SET_PLUGINDIR + +dnl set release date/time +AG_GST_SET_PACKAGE_RELEASE_DATETIME_WITH_NANO([$PACKAGE_VERSION_NANO], + ["${srcdir}/gst-inference.doap"], + [$PACKAGE_VERSION_MAJOR.$PACKAGE_VERSION_MINOR.$PACKAGE_VERSION_MICRO]) + +# set by AG_GST_PARSE_SUBSYSTEM_DISABLES above +dnl make sure it does not complain about unused variables if debugging is disabled +NO_WARNINGS="" +AG_GST_CHECK_GST_DEBUG_DISABLED([NO_WARNINGS="-Wno-unused"], [NO_WARNINGS=""]) + +dnl define an ERROR_CFLAGS Makefile variable +dnl -Wundef: too many broken headers +AG_GST_SET_ERROR_CFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wmissing-prototypes -Wredundant-decls + -Wwrite-strings -Wold-style-definition -Waggregate-return + -Winit-self -Wmissing-include-dirs -Waddress -Wno-multichar + -Wnested-externs $NO_WARNINGS]) + +dnl define an ERROR_CXXFLAGS Makefile variable +AG_GST_SET_ERROR_CXXFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wredundant-decls + -Wwrite-strings + -Winit-self -Wmissing-include-dirs -Waddress -Wno-multichar + $NO_WARNINGS]) + +dnl define an ERROR_OBJCFLAGS Makefile variable +AG_GST_SET_ERROR_OBJCFLAGS($FATAL_WARNINGS, [ + -Wmissing-declarations -Wmissing-prototypes -Wredundant-decls + -Wwrite-strings -Wold-style-definition + -Winit-self -Wmissing-include-dirs -Wno-multichar + -Wnested-externs $NO_WARNINGS]) + +dnl define correct level for debugging messages +AG_GST_SET_LEVEL_DEFAULT($GST_GIT) + +dnl used in examples +AG_GST_DEFAULT_ELEMENTS + +dnl *** plug-ins to include *** + +dnl these are all the gst plug-ins, compilable without additional libs +dnl videofilter is at the top because others depend on it +dnl AG_GST_CHECK_PLUGIN(example) + +dnl disable experimental plug-ins +dnl if test "x$BUILD_EXPERIMENTAL" != "xyes"; then + dnl AG_GST_DISABLE_PLUGIN(example) +dnl fi + +dnl *** sys plug-ins *** + +echo +AC_MSG_NOTICE([Checking libraries for plugins in sys/]) +echo + +dnl *** ext plug-ins *** +dnl keep this list sorted alphabetically ! + +if test "x$BUILD_EXTERNAL" = "xyes"; then + +echo +AC_MSG_NOTICE([Checking libraries for plugins in ext/]) +echo + +AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun's Inference Framework], gstinference, [ + AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0) +]) + +else + +dnl not building plugins with external dependencies, +dnl but we still need to set the conditionals +AM_CONDITIONAL(USE_R2INFERENCE, false) + +fi dnl of EXT plugins + +dnl *** finalize CFLAGS, LDFLAGS, LIBS + +dnl Overview: +dnl GST_OPTION_CFLAGS: common flags for profiling, debugging, errors, ... +dnl GST_*: flags shared by all built objects +dnl GST_ALL_LDFLAGS: linker flags shared by all +dnl GST_LIB_LDFLAGS: not needed, we do not install libraries +dnl GST_LT_LDFLAGS: library versioning of our libraries +dnl GST_PLUGIN_LDFLAGS: flags to be used for all plugins + +dnl GST_OPTION_CFLAGS +if test "x$USE_DEBUG" = xyes; then + PROFILE_CFLAGS="-g" +fi +AC_SUBST(PROFILE_CFLAGS) + +if test "x$PACKAGE_VERSION_NANO" = "x1"; then + dnl Define _only_ for git (not pre-releases or releases) + DEPRECATED_CFLAGS="-DGST_DISABLE_DEPRECATED" +else + DEPRECATED_CFLAGS="" +fi +AC_SUBST(DEPRECATED_CFLAGS) + +dnl every flag in GST_OPTION_CFLAGS, GST_OPTION_CXXFLAGS and GST_OPTION_OBJCFLAGS can be overridden +dnl at make time with e.g. make ERROR_CFLAGS="" +GST_OPTION_CFLAGS="\$(WARNING_CFLAGS) \$(ERROR_CFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +GST_OPTION_CXXFLAGS="\$(WARNING_CXXFLAGS) \$(ERROR_CXXFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +GST_OPTION_OBJCFLAGS="\$(WARNING_OBJCFLAGS) \$(ERROR_OBJCFLAGS) \$(DEBUG_CFLAGS) \$(PROFILE_CFLAGS) \$(GCOV_CFLAGS) \$(OPT_CFLAGS) \$(DEPRECATED_CFLAGS)" +AC_SUBST(GST_OPTION_CFLAGS) +AC_SUBST(GST_OPTION_CXXFLAGS) +AC_SUBST(GST_OPTION_OBJCFLAGS) + +dnl our libraries need to be versioned correctly +AC_SUBST(GST_LT_LDFLAGS) + +dnl FIXME: do we want to rename to GST_ALL_* ? +dnl prefer internal headers to already installed ones +dnl also add builddir include for enumtypes and marshal +dnl add ERROR_CFLAGS, but overridable +GST_CFLAGS="$GST_CFLAGS -DGST_USE_UNSTABLE_API -Wno-error=missing-include-dirs" +GST_CXXFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_CXXFLAGS)" +GST_OBJCFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_OBJCFLAGS)" +GST_CFLAGS="-I\$(top_srcdir)/gst-libs $GST_CFLAGS $GLIB_EXTRA_CFLAGS \$(GST_OPTION_CFLAGS)" +AC_SUBST(GST_CFLAGS) +AC_SUBST(GST_CXXFLAGS) +AC_SUBST(GST_OBJCFLAGS) +AC_SUBST(GST_LIBS) + +dnl LDFLAGS really should only contain flags, not libs - they get added before +dnl whatevertarget_LIBS and -L flags here affect the rest of the linking +GST_ALL_LDFLAGS="-no-undefined" +if test "x${enable_Bsymbolic}" = "xyes"; then + GST_ALL_LDFLAGS="$GST_ALL_LDFLAGS -Wl,-Bsymbolic-functions" +fi +AC_SUBST(GST_ALL_LDFLAGS) + +dnl this really should only contain flags, not libs - they get added before +dnl whatevertarget_LIBS and -L flags here affect the rest of the linking +GST_PLUGIN_LDFLAGS="-module -avoid-version -export-symbols-regex '^[_]*gst_plugin_.*' $GST_ALL_LDFLAGS" +AC_SUBST(GST_PLUGIN_LDFLAGS) + +dnl *** output files *** + +dnl keep this alphabetic per directory, please +AC_CONFIG_FILES( +Makefile +common/Makefile +common/m4/Makefile +docs/Makefile +docs/version.entities +docs/plugins/Makefile +gst-libs/Makefile +gst-libs/gst/Makefile +gst-libs/gst/r2inference/Makefile +m4/Makefile +ext/Makefile +ext/r2inference/Makefile +tests/Makefile +tests/check/Makefile +tests/examples/Makefile +tests/files/Makefile +) +AC_OUTPUT + +AG_GST_OUTPUT_PLUGINS + diff --git a/docs/Makefile.am b/docs/Makefile.am new file mode 100644 index 00000000..b71b89a4 --- /dev/null +++ b/docs/Makefile.am @@ -0,0 +1,18 @@ +if ENABLE_GTK_DOC +if ENABLE_PLUGIN_DOCS +PLUGIN_DOCS_DIRS = plugins +else +PLUGIN_DOCS_DIRS = +endif +else +PLUGIN_DOCS_DIRS = plugins +endif + +SUBDIRS = $(PLUGIN_DOCS_DIRS) +DIST_SUBDIRS = plugins + +EXTRA_DIST = \ + version.entities.in + +upload: + @if test "x$(SUBDIRS)" != x; then for a in $(SUBDIRS); do cd $$a; make upload; cd ..; done; fi diff --git a/docs/plugins/Makefile.am b/docs/plugins/Makefile.am new file mode 100644 index 00000000..07687303 --- /dev/null +++ b/docs/plugins/Makefile.am @@ -0,0 +1,80 @@ +GST_DOC_SCANOBJ = $(top_srcdir)/common/gstdoc-scangobj + +## Process this file with automake to produce Makefile.in + +# The name of the module, e.g. 'glib'. +MODULE=gst-inference +DOC_MODULE=$(MODULE)-plugin + +# for upload-doc.mak +DOC=$(MODULE)-plugin +FORMATS=html +html: html-build.stamp +include $(top_srcdir)/common/upload-doc.mak + +# The top-level SGML file. Change it if you want. +DOC_MAIN_SGML_FILE=$(DOC_MODULE)-docs.sgml + +# The directory containing the source code. +# gtk-doc will search all .c & .h files beneath here for inline comments +# documenting functions and macros. +DOC_SOURCE_DIR = $(top_srcdir)/ext + +# Extra options to supply to gtkdoc-scan. +SCAN_OPTIONS= + +# Extra options to supply to gtkdoc-mkdb. +MKDB_OPTIONS=--sgml-mode --source-suffixes=c,h,cc + +# Extra options to supply to gtkdoc-fixref. +FIXXREF_OPTIONS=--extra-dir=$(GLIB_PREFIX)/share/gtk-doc/html \ + --extra-dir=$(GST_PREFIX)/share/gtk-doc/html \ + --extra-dir=$(GSTPB_PREFIX)/share/gtk-doc/html + +# Used for dependencies. +HFILE_GLOB= \ + $(top_srcdir)/ext/*/*.h +CFILE_GLOB= \ + $(top_srcdir)/ext/*/*.c + +# Header files to ignore when scanning. +IGNORE_HFILES = +IGNORE_CFILES = + +# we add all .h files of elements that have signals/args we want +# sadly this also pulls in the private methods - maybe we should +# move those around in the source ? +# also, we should add some stuff here conditionally based on whether +# or not the plugin will actually build +# but I'm not sure about that - it might be this Just Works given that +# the registry won't have the element + +EXTRA_HFILES = +# $(top_srcdir)/ext/example/example.h# +# $(top_srcdir)/gst/example/example.h# +# $(top_srcdir)/sys/example/example.h + +# example code that needs to be converted to xml and placed in xml/ +EXAMPLE_CFILES = +# $(top_srcdir)/tests/examples/example/example.c + +# Images to copy into HTML directory. +HTML_IMAGES = + +# Extra SGML files that are included by $(DOC_MAIN_SGML_FILE). +content_files = + +# Other files to distribute. +extra_files = + +# CFLAGS and LDFLAGS for compiling scan program. Only needed if your app/lib +# contains GtkObjects/GObjects and you want to document signals and properties. +GTKDOC_CFLAGS = $(GST_BASE_CFLAGS) -I$(top_builddir) +GTKDOC_LIBS = $(GST_BASE_LIBS) + +# If you need to override some of the declarations, place them in this file +# and uncomment this line. +#DOC_OVERRIDES = $(DOC_MODULE)-overrides.txt +DOC_OVERRIDES = + +include $(top_srcdir)/common/gtk-doc-plugins.mak diff --git a/docs/plugins/gst-inference-plugin-sections.txt b/docs/plugins/gst-inference-plugin-sections.txt new file mode 100644 index 00000000..a0c620d9 --- /dev/null +++ b/docs/plugins/gst-inference-plugin-sections.txt @@ -0,0 +1,15 @@ +
+element-example +example +GstExample + +GstExampleClass +GST_EXAMPLE +GST_EXAMPLE_CAST +GST_IS_EXAMPLE +GST_EXAMPLE_CLASS +GST_IS_EXAMPLE_CLASS +GST_TYPE_EXAMPLE + +gst_example_get_type +
diff --git a/docs/plugins/gst-inference-plugin.types b/docs/plugins/gst-inference-plugin.types new file mode 100644 index 00000000..e69de29b diff --git a/docs/version.entities.in b/docs/version.entities.in new file mode 100644 index 00000000..286989f5 --- /dev/null +++ b/docs/version.entities.in @@ -0,0 +1,2 @@ + + diff --git a/ext/Makefile.am b/ext/Makefile.am new file mode 100644 index 00000000..30d735f3 --- /dev/null +++ b/ext/Makefile.am @@ -0,0 +1,11 @@ +if USE_R2INFERENCE +R2INFERENCE_DIR=r2inference +else +R2INFERENCE_DIR= +endif + +SUBDIRS=$(R2INFERENCE_DIR) + +DIST_SUBDIRS=r2inference + +include $(top_srcdir)/common/parallel-subdirs.mak diff --git a/ext/r2inference/Makefile.am b/ext/r2inference/Makefile.am new file mode 100644 index 00000000..ea66bab7 --- /dev/null +++ b/ext/r2inference/Makefile.am @@ -0,0 +1,39 @@ +plugin_LTLIBRARIES = libgstinference.la + +libgstinference_la_SOURCES = \ + gstgooglenet.c \ + gstinference.c \ + gsttinyyolo.c + +libgstinference_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + + +libgstinference_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + + +libgstinference_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinference_la_LDFLAGS = \ + $(GST_PLUGIN_LDFLAGS) + +libgstinference_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstgooglenet.h \ + gsttinyyolo.h diff --git a/ext/r2inference/gstgooglenet.c b/ext/r2inference/gstgooglenet.c new file mode 100644 index 00000000..f019fa82 --- /dev/null +++ b/ext/r2inference/gstgooglenet.c @@ -0,0 +1,254 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstgooglenet + * + * The googlenet element allows the user to infer/execute a pretrained model + * based on the GoogLeNet architecture on incoming image frames. + * + * + * Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! googlenet ! xvimagesink + * ]| + * Process video frames from the camera using a GoogLeNet model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstgooglenet.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_googlenet_debug_category); +#define GST_CAT_DEFAULT gst_googlenet_debug_category + +/* prototypes */ +#define CHANNELS 3 + +static void gst_googlenet_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_googlenet_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_googlenet_dispose (GObject * object); +static void gst_googlenet_finalize (GObject * object); + +static gboolean gst_googlenet_preprocess (GstVideoInference * vi, + GstVideoFrame * inframe, GstVideoFrame * outframe); +static gboolean gst_googlenet_postprocess (GstVideoInference * vi, + GstVideoFrame * outframe, const gpointer prediction, gsize predsize); +static gboolean gst_googlenet_start (GstVideoInference * vi); +static gboolean gst_googlenet_stop (GstVideoInference * vi); + +enum +{ + PROP_0 +}; + +/* pad templates */ + +#define CAPS "video/x-raw,format=BGR,width=299,height=299" + +static GstStaticPadTemplate sink_model_factory = +GST_STATIC_PAD_TEMPLATE ("sink_model", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +static GstStaticPadTemplate src_model_factory = +GST_STATIC_PAD_TEMPLATE ("src_model", + GST_PAD_SRC, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +struct _GstGooglenet +{ + GstVideoInference parent; +}; + +struct _GstGooglenetClass +{ + GstVideoInferenceClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstGooglenet, gst_googlenet, GST_TYPE_VIDEO_INFERENCE, + GST_DEBUG_CATEGORY_INIT (gst_googlenet_debug_category, "googlenet", 0, + "debug category for googlenet element")); + +static void +gst_googlenet_class_init (GstGooglenetClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoInferenceClass *vi_class = GST_VIDEO_INFERENCE_CLASS (klass); + + gst_element_class_add_static_pad_template (element_class, + &sink_model_factory); + gst_element_class_add_static_pad_template (element_class, &src_model_factory); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "googlenet", "Filter", + "Infers incoming image frames using a pretrained GoogLeNet model", + "Carlos Rodriguez \n\t\t\t" + " Jose Jimenez \n\t\t\t" + " Michael Gruner "); + + gobject_class->set_property = gst_googlenet_set_property; + gobject_class->get_property = gst_googlenet_get_property; + gobject_class->dispose = gst_googlenet_dispose; + gobject_class->finalize = gst_googlenet_finalize; + + vi_class->start = GST_DEBUG_FUNCPTR (gst_googlenet_start); + vi_class->stop = GST_DEBUG_FUNCPTR (gst_googlenet_stop); + vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_googlenet_preprocess); + vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_googlenet_postprocess); +} + +static void +gst_googlenet_init (GstGooglenet * googlenet) +{ +} + +void +gst_googlenet_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "set_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_googlenet_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "get_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_googlenet_dispose (GObject * object) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "dispose"); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS (gst_googlenet_parent_class)->dispose (object); +} + +void +gst_googlenet_finalize (GObject * object) +{ + GstGooglenet *googlenet = GST_GOOGLENET (object); + + GST_DEBUG_OBJECT (googlenet, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS (gst_googlenet_parent_class)->finalize (object); +} + +static gboolean +gst_googlenet_preprocess (GstVideoInference * vi, + GstVideoFrame * inframe, GstVideoFrame * outframe) +{ + gint size; + + GST_LOG_OBJECT (vi, "Preprocess"); + size = CHANNELS * inframe->info.width * inframe->info.height; + + for (gint i = 0; i < size; i += CHANNELS) { + /* BGR = RGB - Mean */ + + ((gfloat *) outframe->data[0])[i + 0] = + (((guchar *) inframe->data[0])[i + 0] - 128) / 128.0; + ((gfloat *) outframe->data[0])[i + 1] = + (((guchar *) inframe->data[0])[i + 1] - 128) / 128.0; + ((gfloat *) outframe->data[0])[i + 2] = + (((guchar *) inframe->data[0])[i + 2] - 128) / 128.0; + + } + + return TRUE; +} + +static gboolean +gst_googlenet_postprocess (GstVideoInference * vi, + GstVideoFrame * outframe, const gpointer prediction, gsize predsize) +{ + gint index; + gint size; + gdouble max; + GST_LOG_OBJECT (vi, "Postprocess"); + index = 0; + max = -1; + size = predsize / sizeof (gfloat); + + for (gint i = 0; i < size; ++i) { + gfloat current = ((gfloat *) prediction)[i]; + if (current > max) { + max = current; + index = i; + } + } + + g_print ("Highest probability is label %i : (%f)\n", index, max); + + return TRUE; +} + +static gboolean +gst_googlenet_start (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Starting GoogLeNet"); + + return TRUE; +} + +static gboolean +gst_googlenet_stop (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Stopping GoogLeNet"); + + return TRUE; +} diff --git a/ext/r2inference/gstgooglenet.h b/ext/r2inference/gstgooglenet.h new file mode 100644 index 00000000..000e9cd1 --- /dev/null +++ b/ext/r2inference/gstgooglenet.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_GOOGLENET_H_ +#define _GST_GOOGLENET_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_GOOGLENET gst_googlenet_get_type () +G_DECLARE_FINAL_TYPE (GstGooglenet, gst_googlenet, GST, GOOGLENET, GstVideoInference) + +G_END_DECLS + +#endif diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c new file mode 100644 index 00000000..68b78baa --- /dev/null +++ b/ext/r2inference/gstinference.c @@ -0,0 +1,57 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstgooglenet.h" +#include "gsttinyyolo.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + + /* FIXME Remember to set the rank if it's an element that is meant + to be autoplugged by decodebin. */ + gboolean ret = TRUE; + + ret = gst_element_register (plugin, "googlenet", GST_RANK_NONE, + GST_TYPE_GOOGLENET); + if (!ret) { + goto out; + } + + ret = gst_element_register (plugin, "tinyyolo", GST_RANK_NONE, + GST_TYPE_TINYYOLO); + if (!ret) { + goto out; + } + + out: + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inference, + "Infer pre-trained model on incomming image frames on a variety of architectures and different backends", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/ext/r2inference/gsttinyyolo.c b/ext/r2inference/gsttinyyolo.c new file mode 100644 index 00000000..2effd416 --- /dev/null +++ b/ext/r2inference/gsttinyyolo.c @@ -0,0 +1,479 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gsttinyyolo + * + * The tinyyolo element allows the user to infer/execute a pretrained model + * based on the TinyYolo architecture on incoming image frames. + * + * + * Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! tinyyolo ! xvimagesink + * ]| + * Process video frames from the camera using a TinyYolo model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gsttinyyolo.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tinyyolo_debug_category); +#define GST_CAT_DEFAULT gst_tinyyolo_debug_category + +/* prototypes */ + +#define CHANNELS 3 +#define GRID_H 7 +#define GRID_W 7 +/* Number of classes */ +#define CLASSES 20 +/* Number of boxes per cell */ +#define BOXES 2 +/* Box dim */ +#define BOX_DIM 4 +/* Probability threshold */ +#define PROB_THRESH 0.08 +/* Intersection over union threshold */ +#define IOU_THRESH 0.30 + +typedef struct _Box Box; +struct _Box +{ + const gchar *label; + gdouble x_center; + gdouble y_center; + gdouble width; + gdouble height; + gdouble prob; +}; + + +static void gst_tinyyolo_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_tinyyolo_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_tinyyolo_dispose (GObject * object); +static void gst_tinyyolo_finalize (GObject * object); + +static gboolean gst_tinyyolo_preprocess (GstVideoInference * vi, + GstVideoFrame * inframe, GstVideoFrame * outframe); +static gboolean gst_tinyyolo_postprocess (GstVideoInference * vi, + GstVideoFrame * outframe, const gpointer prediction, gsize predsize); +static gboolean gst_tinyyolo_start (GstVideoInference * vi); +static gboolean gst_tinyyolo_stop (GstVideoInference * vi); + +static void print_box (Box in_box); + +static void print_top_predictions (gpointer prediction, + gint input_image_width, gint input_image_height); +static void get_boxes_from_prediction (gpointer prediction, + gint input_image_width, gint input_image_height, + Box * boxes, gint * elements); + +static void box_to_pixels (Box * normalized_box, gint row, gint col, + gint image_width, gint image_height); + +static void remove_duplicated_boxes (Box * boxes, gint * num_boxes); + +static void delete_box (Box * boxes, gint * num_boxes, gint index); + +static gdouble intersection_over_union (Box box_1, Box box_2); + +enum +{ + PROP_0 +}; + +/* pad templates */ + +#define CAPS "video/x-raw,format=BGR,width=448,height=448" + +static GstStaticPadTemplate sink_model_factory = +GST_STATIC_PAD_TEMPLATE ("sink_model", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +static GstStaticPadTemplate src_model_factory = +GST_STATIC_PAD_TEMPLATE ("src_model", + GST_PAD_SRC, + GST_PAD_REQUEST, + GST_STATIC_CAPS (CAPS) + ); + +struct _GstTinyyolo +{ + GstVideoInference parent; +}; + +struct _GstTinyyoloClass +{ + GstVideoInferenceClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstTinyyolo, gst_tinyyolo, GST_TYPE_VIDEO_INFERENCE, + GST_DEBUG_CATEGORY_INIT (gst_tinyyolo_debug_category, "tinyyolo", 0, + "debug category for tinyyolo element")); + +static void +gst_tinyyolo_class_init (GstTinyyoloClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoInferenceClass *vi_class = GST_VIDEO_INFERENCE_CLASS (klass); + + gst_element_class_add_static_pad_template (element_class, + &sink_model_factory); + gst_element_class_add_static_pad_template (element_class, &src_model_factory); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "tinyyolo", "Filter", + "Infers incoming image frames using a pretrained TinyYolo model", + "Carlos Rodriguez \n\t\t\t" + " Jose Jimenez \n\t\t\t" + " Michael Gruner \n\t\t\t" + " Carlos Aguero \n\t\t\t" + " Miguel Taylor \n\t\t\t" + " Greivin Fallas "); + + gobject_class->set_property = gst_tinyyolo_set_property; + gobject_class->get_property = gst_tinyyolo_get_property; + gobject_class->dispose = gst_tinyyolo_dispose; + gobject_class->finalize = gst_tinyyolo_finalize; + + vi_class->start = GST_DEBUG_FUNCPTR (gst_tinyyolo_start); + vi_class->stop = GST_DEBUG_FUNCPTR (gst_tinyyolo_stop); + vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_tinyyolo_preprocess); + vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_tinyyolo_postprocess); +} + +static void +gst_tinyyolo_init (GstTinyyolo * tinyyolo) +{ +} + +void +gst_tinyyolo_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "set_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_tinyyolo_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "get_property"); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_tinyyolo_dispose (GObject * object) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "dispose"); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS (gst_tinyyolo_parent_class)->dispose (object); +} + +void +gst_tinyyolo_finalize (GObject * object) +{ + GstTinyyolo *tinyyolo = GST_TINYYOLO (object); + + GST_DEBUG_OBJECT (tinyyolo, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS (gst_tinyyolo_parent_class)->finalize (object); +} + +void +print_box (Box in_box) +{ + g_print ("Box:"); + g_print ("[class:'%s', ", in_box.label); + g_print ("x_center:%f, ", in_box.x_center); + g_print ("y_center:%f, ", in_box.y_center); + g_print ("width:%f, ", in_box.width); + g_print ("height:%f, ", in_box.height); + g_print ("prob:%f]\n", in_box.prob); +} + +void +box_to_pixels (Box * normalized_box, gint row, gint col, gint image_width, + gint image_height) +{ + + /* adjust the box center according to its cell and grid dim */ + normalized_box->x_center += col; + normalized_box->y_center += row; + normalized_box->x_center /= GRID_H; + normalized_box->y_center /= GRID_W; + + /* adjust the lengths and widths */ + normalized_box->width *= normalized_box->width; + normalized_box->height *= normalized_box->height; + + /* scale the boxes to the image size in pixels */ + normalized_box->x_center *= image_width; + normalized_box->y_center *= image_height; + normalized_box->width *= image_width; + normalized_box->height *= image_height; +} + +void +get_boxes_from_prediction (gpointer prediction, gint input_image_width, + gint input_image_height, Box * boxes, gint * elements) +{ + + gint i; + gint j; + gint c; + gint b; + gint box_probs_start = GRID_H * GRID_W * CLASSES; + gint all_boxes_start = GRID_H * GRID_W * CLASSES + GRID_H * GRID_W * BOXES; + gint index; + gdouble class_prob; + gdouble box_prob; + gdouble prob; + gint counter = 0; + + const gchar *labels[] = { "aeroplane", "bicycle", "bird", "boat", "bottle", + "bus", "car", "cat", "chair", "cow", "diningtable", + "dog", "horse", "motorbike", "person", "pottedplant", + "sheep", "sofa", "train", "tvmonitor" + }; + + /* Iterate rows */ + for (i = 0; i < GRID_H; i++) { + /* Iterate colums */ + for (j = 0; j < GRID_W; j++) { + /* Iterate classes */ + for (c = 0; c < CLASSES; c++) { + index = (i * GRID_W + j) * CLASSES + c; + class_prob = ((gfloat *) prediction)[index]; + for (b = 0; b < BOXES; b++) { + index = (i * GRID_W + j) * BOXES + b; + box_prob = ((gfloat *) prediction)[box_probs_start + index]; + prob = class_prob * box_prob; + /* If the probability is over the threshold add it to the boxes list */ + if (prob > PROB_THRESH) { + Box result; + index = ((i * GRID_W + j) * BOXES + b) * BOX_DIM; + result.label = labels[c]; + result.x_center = ((gfloat *) prediction)[all_boxes_start + index]; + result.y_center = + ((gfloat *) prediction)[all_boxes_start + index + 1]; + result.width = ((gfloat *) prediction)[all_boxes_start + index + 2]; + result.height = + ((gfloat *) prediction)[all_boxes_start + index + 3]; + result.prob = prob; + box_to_pixels (&result, i, j, input_image_width, + input_image_height); + boxes[counter] = result; + counter = counter + 1; + } + } + } + } + + *elements = counter; + + } +} + +void +print_top_predictions (gpointer prediction, + gint input_image_width, gint input_image_height) +{ + + Box boxes[GRID_H * GRID_W * BOXES]; + gint elements = 0; + gint index = 0; + + get_boxes_from_prediction (prediction, input_image_width, input_image_height, + boxes, &elements); + + remove_duplicated_boxes (boxes, &elements); + + for (index = 0; index < elements; index = index + 1) { + print_box (boxes[index]); + } + +} + +gdouble +intersection_over_union (Box box_1, Box box_2) +{ + + /* + * Evaluate the intersection-over-union for two boxes + * The intersection-over-union metric determines how close + * two boxes are to being the same box. + */ + gdouble intersection_dim_1; + gdouble intersection_dim_2; + gdouble intersection_area; + gdouble union_area; + + /* First diminsion of the intersecting box */ + intersection_dim_1 = MIN (box_1.x_center + 0.5 * box_1.width, + box_2.x_center + 0.5 * box_2.width) - + MAX (box_1.x_center - 0.5 * box_1.width, + box_2.x_center - 0.5 * box_2.width); + + /* Second dimension of the intersecting box */ + intersection_dim_2 = MIN (box_1.y_center + 0.5 * box_1.height, + box_2.y_center + 0.5 * box_2.height) - + MAX (box_1.y_center - 0.5 * box_1.height, + box_2.y_center - 0.5 * box_2.height); + + if ((intersection_dim_1 < 0) || (intersection_dim_2 < 0)) { + intersection_area = 0; + } else { + intersection_area = intersection_dim_1 * intersection_dim_2; + } + union_area = box_1.width * box_1.height + box_2.width * box_2.height - + intersection_area; + return intersection_area / union_area; +} + +void +delete_box (Box * boxes, gint * num_boxes, gint index) +{ + + gint i, last_index; + if (*num_boxes > 0 && index < *num_boxes && index > -1) { + last_index = *num_boxes - 1; + for (i = index; i < last_index; i++) { + boxes[i] = boxes[i + 1]; + } + *num_boxes -= 1; + } +} + +void +remove_duplicated_boxes (Box * boxes, gint * num_boxes) +{ + + /* Remove duplicated boxes. A box is considered a duplicate if its + * intersection over union metric is above a threshold + */ + gdouble iou; + gint it1, it2; + + for (it1 = 0; it1 < *num_boxes - 1; it1++) { + for (it2 = it1 + 1; it2 < *num_boxes; it2++) { + if (strcmp (boxes[it1].label, boxes[it2].label) == 0) { + iou = intersection_over_union (boxes[it1], boxes[it2]); + if (iou > IOU_THRESH) { + if (boxes[it1].prob > boxes[it2].prob) { + delete_box (boxes, num_boxes, it2); + it2--; + } else { + delete_box (boxes, num_boxes, it1); + it1--; + break; + } + } + } + } + } +} + +static gboolean +gst_tinyyolo_preprocess (GstVideoInference * vi, + GstVideoFrame * inframe, GstVideoFrame * outframe) +{ + + gint size; + GST_LOG_OBJECT (vi, "Preprocess"); + + size = CHANNELS * inframe->info.width * inframe->info.height; + + for (gint i = 0; i < size; i += CHANNELS) { + ((gfloat *) outframe->data[0])[i + 2] = + (((guchar *) inframe->data[0])[i + 2]) / 255.0; + ((gfloat *) outframe->data[0])[i + 1] = + (((guchar *) inframe->data[0])[i + 1]) / 255.0; + ((gfloat *) outframe->data[0])[i + 0] = + (((guchar *) inframe->data[0])[i + 0]) / 255.0; + } + + return TRUE; +} + +static gboolean +gst_tinyyolo_postprocess (GstVideoInference * vi, + GstVideoFrame * outframe, const gpointer prediction, gsize predsize) +{ + + GST_LOG_OBJECT (vi, "Postprocess"); + + print_top_predictions (prediction, outframe->info.width, + outframe->info.height); + + return TRUE; +} + +static gboolean +gst_tinyyolo_start (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Starting TinyYolo"); + + return TRUE; +} + +static gboolean +gst_tinyyolo_stop (GstVideoInference * vi) +{ + GST_INFO_OBJECT (vi, "Stopping TinyYolo"); + + return TRUE; +} diff --git a/ext/r2inference/gsttinyyolo.h b/ext/r2inference/gsttinyyolo.h new file mode 100644 index 00000000..fd36996f --- /dev/null +++ b/ext/r2inference/gsttinyyolo.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_TINYYOLO_H_ +#define _GST_TINYYOLO_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TINYYOLO gst_tinyyolo_get_type () +G_DECLARE_FINAL_TYPE (GstTinyyolo, gst_tinyyolo, GST, TINYYOLO, GstVideoInference) + +G_END_DECLS + +#endif /* _GST_TINYYOLO_H_ */ diff --git a/gst-inference.doap b/gst-inference.doap new file mode 100644 index 00000000..a50cff64 --- /dev/null +++ b/gst-inference.doap @@ -0,0 +1,45 @@ + + + GStreamer Deep Learning Inference Framework + gst-inference + + 2018-10-22 + +GStreamer Inference Framework + + +GstInference is a GStreamer based deep learning inference framework. It provides +base clases, elements and utilities to easlily and efficiently execute deep-learning +models to detect or classify objects in a scene. + + + + + + C + + + + + + + + + + + Carlos Rodriguez + + + Jose Jimenez + + + Michael Gruner + + + + diff --git a/gst-libs/Makefile.am b/gst-libs/Makefile.am new file mode 100644 index 00000000..99c71898 --- /dev/null +++ b/gst-libs/Makefile.am @@ -0,0 +1,3 @@ + +SUBDIRS=gst + diff --git a/gst-libs/gst/Makefile.am b/gst-libs/gst/Makefile.am new file mode 100644 index 00000000..a2d5c66e --- /dev/null +++ b/gst-libs/gst/Makefile.am @@ -0,0 +1,8 @@ + +SUBDIRS= + +if USE_R2INFERENCE +SUBDIRS += r2inference +endif #USE_R2INFERENCE + +DIST_SUBDIRS = r2inference diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am new file mode 100644 index 00000000..f3da0c9b --- /dev/null +++ b/gst-libs/gst/r2inference/Makefile.am @@ -0,0 +1,42 @@ + +lib_LTLIBRARIES = libgstinference-@GST_API_VERSION@.la + +libgstinference_@GST_API_VERSION@_la_SOURCES= \ + gstvideoinference.c \ + gstchildinspector.c \ + gstinferencebackends.cc \ + gstbackend.cc \ + gstncsdk.cc \ + gsttensorflow.cc + +libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ + $(GST_CXXFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) \ + -std=c++11 + +libgstinference_@GST_API_VERSION@_la_CFLAGS= \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -lgstvideo-@GST_API_VERSION@ \ + $(R2INFERENCE_CFLAGS) + +libgstinference_@GST_API_VERSION@_la_LIBADD= \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) + +gstinferenceincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/r2inference/ + +gstinferenceinclude_HEADERS= \ + gstvideoinference.h \ + gstchildinspector.h \ + gstinferencebackends.h \ + gstbackend.h \ + gstbackendsubclass.h \ + gstncsdk.h \ + gsttensorflow.h diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc new file mode 100644 index 00000000..22df5841 --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -0,0 +1,468 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstbackend.h" +#include "gstbackendsubclass.h" + +#include + +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); +#define GST_CAT_DEFAULT gst_backend_debug_category + +class InferenceProperty { + private: + + GValue *avalue; + GParamSpec *apspec; + + public: + + InferenceProperty() { + } + + InferenceProperty(const GValue *value, GParamSpec *pspec) { + avalue = (GValue *) g_malloc(sizeof(GValue)); + *avalue = G_VALUE_INIT; + avalue = g_value_init (avalue, pspec->value_type); + g_value_copy (value, avalue); + g_param_spec_ref (pspec); + apspec = pspec; + } + + ~InferenceProperty() { + g_param_spec_unref (apspec); + g_free(avalue); + } + + void apply_inference_property(GstBackend *self, + std::shared_ptr params, r2i::RuntimeError &error) { + switch (apspec->value_type) { + case G_TYPE_STRING: + GST_INFO_OBJECT (self, "Setting property: %s=%s\n", apspec->name, + g_value_get_string(avalue)); + error = params->Set(apspec->name, g_value_get_string(avalue)); + break; + case G_TYPE_INT: + GST_INFO_OBJECT (self, "Setting property: %s=%d\n", apspec->name, + g_value_get_int(avalue)); + error = params->Set(apspec->name, g_value_get_int(avalue)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + } + + const gchar *get_name() { + return apspec->name; + } +}; + +typedef struct _GstBackendPrivate GstBackendPrivate; +struct _GstBackendPrivate { + r2i::FrameworkCode code; + std::shared_ptr < r2i::IEngine > engine; + std::shared_ptr < r2i::ILoader > loader; + std::shared_ptr < r2i::IModel > model; + std::shared_ptr < r2i::IParameters > params; + std::unique_ptr < r2i::IFrameworkFactory > factory; + GMutex backend_mutex; + gboolean backend_started; + std::shared_ptr < std::list > property_list; + gboolean backend_created; + +}; + +G_DEFINE_TYPE_WITH_CODE (GstBackend, gst_backend, G_TYPE_OBJECT, + GST_DEBUG_CATEGORY_INIT (gst_backend_debug_category, "backend", 0, + "debug category for backend parameters"); G_ADD_PRIVATE (GstBackend)); + +#define GST_BACKEND_PRIVATE(self) \ + (GstBackendPrivate *)(gst_backend_get_instance_private (self)) + +static GParamSpec *gst_backend_param_to_spec (r2i::ParameterMeta *param); +static int gst_backend_param_flags (int flags); +static void gst_backend_finalize (GObject *obj); + +#define GST_BACKEND_ERROR gst_backend_error_quark() + +static void +gst_backend_class_init (GstBackendClass *klass) { + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + oclass->finalize = gst_backend_finalize; + +} + +static void +gst_backend_init (GstBackend *self) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + g_mutex_init(&priv->backend_mutex); + priv->backend_started = false; + priv->backend_created = false; + priv->property_list = std::make_shared>(); +} + +static void +gst_backend_finalize (GObject *obj) { + GstBackend *self = GST_BACKEND (obj); + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + g_mutex_clear (&priv->backend_mutex); + G_OBJECT_CLASS (gst_backend_parent_class)->finalize (obj); +} + +void +gst_backend_install_properties (GstBackendClass *klass, + r2i::FrameworkCode code) { + GObjectClass *oclass = G_OBJECT_CLASS (klass); + r2i::RuntimeError error; + static std::vector < r2i::ParameterMeta > params; + static gint nprop = 1; + + auto factory = r2i::IFrameworkFactory::MakeFactory (code, error); + auto pfactory = factory->MakeParameters (error); + error = pfactory->List (params); + + for (auto ¶m : params) { + GParamSpec *spec = gst_backend_param_to_spec (¶m); + g_object_class_install_property (oclass, nprop, spec); + nprop++; + } +} + +static int +gst_backend_param_flags (int flags) { + int pflags = 0; + + if (r2i::ParameterMeta::Flags::READ & flags) { + pflags += G_PARAM_READABLE; + } + + if (r2i::ParameterMeta::Flags::WRITE & flags) { + pflags += G_PARAM_WRITABLE; + } + + return pflags; +} + +static GParamSpec * +gst_backend_param_to_spec (r2i::ParameterMeta *param) { + GParamSpec *spec = NULL; + + switch (param->type) { + case (r2i::ParameterMeta::Type::INTEGER): { + spec = g_param_spec_int (param->name.c_str (), + param->name.c_str (), + param->description.c_str (), + G_MININT, + G_MAXINT, 0, (GParamFlags) gst_backend_param_flags (param->flags)); + break; + } + case (r2i::ParameterMeta::Type::STRING): { + spec = g_param_spec_string (param->name.c_str (), + param->name.c_str (), + param->description.c_str (), + NULL, (GParamFlags) gst_backend_param_flags (param->flags)); + break; + } + default: + g_return_val_if_reached (NULL); + } + + return spec; +} + +void +gst_backend_set_property (GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) { + GstBackend *self = GST_BACKEND (object); + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + InferenceProperty *property; + + GST_DEBUG_OBJECT (self, "set_property"); + + g_mutex_lock (&priv->backend_mutex); + if (priv->backend_started) { + switch (pspec->value_type) { + case G_TYPE_STRING: + priv->params->Set(pspec->name, g_value_get_string(value)); + break; + case G_TYPE_INT: + priv->params->Set(pspec->name, g_value_get_int(value)); + break; + default: + GST_WARNING_OBJECT (self, "Invalid property type"); + break; + } + } else { + property = new InferenceProperty(value, pspec); + priv->property_list->push_back(property); + GST_INFO_OBJECT (self, "Queueing property: %s\n", pspec->name); + } + g_mutex_unlock (&priv->backend_mutex); + +} + +void +gst_backend_get_property (GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) { + GstBackend *self = GST_BACKEND (object); + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + int int_buffer; + std::string string_buffer; + GST_DEBUG_OBJECT (self, "get_property"); + + if (NULL != priv->params ) { + switch (pspec->value_type) { + case G_TYPE_STRING: + priv->params->Get (pspec->name, string_buffer); + g_value_set_enum (value, int_buffer); + break; + case G_TYPE_INT: + priv->params->Get (pspec->name, int_buffer); + g_value_set_string (value, string_buffer.c_str()); + break; + } + } +} + +gboolean +gst_backend_start (GstBackend *self, const gchar *model_location, + GError **err) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + r2i::RuntimeError error; + InferenceProperty *property; + std::list::iterator property_it; + static std::vector params; + static std::vector::iterator param_it; + + g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (model_location, FALSE); + g_return_val_if_fail (err, FALSE); + + + if (!priv->backend_created) { + priv->factory = r2i::IFrameworkFactory::MakeFactory (priv->code, + error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend library"); + goto error; + } + + priv->engine = priv->factory->MakeEngine (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend engine"); + goto error; + } + + priv->loader = priv->factory->MakeLoader (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the model loader"); + goto error; + } + + priv->model = priv->loader->Load (model_location, error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to load model"); + goto error; + } + + error = priv->engine->SetModel (priv->model); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set model to engine"); + goto error; + } + + priv->params = priv->factory->MakeParameters (error); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set get parameters for backend"); + goto error; + } + error = priv->params->Configure(priv->engine, priv->model); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to configure mode to backend"); + goto error; + } + error = priv->params->List (params); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to list the backend parameters"); + goto error; + } + priv->backend_created = true; + } + + g_mutex_lock (&priv->backend_mutex); + for (property_it = priv->property_list->begin(); + property_it != priv->property_list->end(); ++property_it) { + property = *property_it; + for (param_it = params.begin(); param_it != params.end(); ++param_it) { + if (!g_strcmp0(property->get_name(), param_it->name.c_str())) { + if (r2i::ParameterMeta::Flags::WRITE_BEFORE_START & param_it->flags) { + property->apply_inference_property(self, priv->params, error); + property->~InferenceProperty(); + priv->property_list->erase(property_it--); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set backend parameters"); + goto error; + } + } + break; + } + } + } + + error = priv->engine->Start (); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to start the backend engine"); + goto start_error; + } + + while (!priv->property_list->empty()) { + property = priv->property_list->front(); + property->apply_inference_property(self, priv->params, error); + property->~InferenceProperty(); + priv->property_list->pop_front(); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to set backend parameters"); + goto error; + } + } + priv->backend_started = true; + g_mutex_unlock (&priv->backend_mutex); + + return TRUE; + +start_error: + g_mutex_unlock (&priv->backend_mutex); +error: + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + +gboolean +gst_backend_stop (GstBackend *self, GError **err) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + r2i::RuntimeError error; + + g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (err, FALSE); + + error = priv->engine->Stop (); + if (error.IsError ()) { + GST_ERROR_OBJECT (self, "Failed to stop the backend engine"); + goto error; + } + return TRUE; + +error: + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + +gboolean +gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, + gpointer *prediction_data, gsize *prediction_size, GError **err) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); + std::shared_ptr < r2i::IPrediction > prediction; + std::shared_ptr < r2i::IFrame > frame; + r2i::RuntimeError error; + + g_return_val_if_fail (priv, FALSE); + g_return_val_if_fail (input_frame, FALSE); + g_return_val_if_fail (prediction_data, FALSE); + g_return_val_if_fail (prediction_size, FALSE); + g_return_val_if_fail (err, FALSE); + + frame = priv->factory->MakeFrame (error); + if (error.IsError ()) { + goto error; + } + + GST_LOG_OBJECT (self, "Processing Frame of size %d x %d", + input_frame->info.width, input_frame->info.height); + + error = + frame->Configure (input_frame->data[0], input_frame->info.width, + input_frame->info.height, r2i::ImageFormat::Id::RGB); + if (error.IsError ()) { + goto error; + } + + prediction = priv->engine->Predict (frame, error); + if (error.IsError ()) { + goto error; + } + + *prediction_size = prediction->GetResultSize (); + + /*could we avoid memory copy ?*/ + *prediction_data = g_malloc(*prediction_size); + memcpy(*prediction_data, prediction->GetResultData(), *prediction_size); + + GST_LOG_OBJECT (self, "Size of prediction %p is %lu", + *prediction_data, *prediction_size); + + return TRUE; +error: + g_set_error (err, GST_BACKEND_ERROR, error.GetCode (), + "R2Inference Error: (Code:%d) %s", error.GetCode (), + error.GetDescription ().c_str ()); + return FALSE; +} + +gboolean +gst_backend_set_framework_code (GstBackend *backend, r2i::FrameworkCode code) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); + g_return_val_if_fail (priv, FALSE); + + priv->code = code; + return TRUE; + +} + +guint +gst_backend_get_framework_code (GstBackend *backend) { + GstBackendPrivate *priv = GST_BACKEND_PRIVATE (backend); + g_return_val_if_fail (priv, -1); + + return priv->code; +} + +GQuark +gst_backend_error_quark(void) { + static GQuark q = 0; + + if (0 == q) { + q = g_quark_from_static_string("gst-backend-error-quark"); + } + + return q; +} diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h new file mode 100644 index 00000000..933bb21f --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -0,0 +1,46 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_BACKEND_H__ +#define __GST_BACKEND_H__ + +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_BACKEND gst_backend_get_type () +G_DECLARE_DERIVABLE_TYPE (GstBackend, gst_backend, GST, BACKEND, GObject); + +struct _GstBackendClass +{ + GObjectClass parent_class; + +}; + +GQuark gst_backend_error_quark (void); +gboolean gst_backend_start (GstBackend *, const gchar *, GError **); +gboolean gst_backend_stop (GstBackend *, GError **); +guint gst_backend_get_framework_code (GstBackend *); +gboolean gst_backend_process_frame (GstBackend *, GstVideoFrame *, + gpointer *, gsize *, GError **); + +G_END_DECLS +#endif //__GST_BACKEND_H__ diff --git a/gst-libs/gst/r2inference/gstbackendsubclass.h b/gst-libs/gst/r2inference/gstbackendsubclass.h new file mode 100644 index 00000000..37aef953 --- /dev/null +++ b/gst-libs/gst/r2inference/gstbackendsubclass.h @@ -0,0 +1,40 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ +#ifndef __GST_BACKENDSUBCLASS_H__ +#define __GST_BACKENDSUBCLASS_H__ + +#include "gstbackend.h" + +#include + +G_BEGIN_DECLS + +void gst_backend_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec); +void gst_backend_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec); +void gst_backend_install_properties (GstBackendClass * klass, + r2i::FrameworkCode code); +gboolean gst_backend_set_framework_code (GstBackend * backend, + r2i::FrameworkCode code); + +G_END_DECLS +#endif //__GST_BACKENDSUBCLASS_H__ diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c new file mode 100644 index 00000000..932c69fd --- /dev/null +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -0,0 +1,215 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstchildinspector.h" + +typedef struct _GstChildInspectorFlag GstChildInspectorFlag; +typedef struct _GstChildInspectorType GstChildInspectorType; + +typedef gchar *(*GstChildInspectorTypeToString) (GParamSpec * pspec, + GValue * value); + +static gchar *gst_child_inspector_type_int_to_string (GParamSpec * pspec, + GValue * value); +static gchar *gst_child_inspector_type_string_to_string (GParamSpec * pspec, + GValue * value); + +struct _GstChildInspectorFlag +{ + gint value; + const gchar *to_string; +}; + +struct _GstChildInspectorType +{ + GType value; + GstChildInspectorTypeToString to_string; +}; + +static GstChildInspectorFlag flags[] = { + {G_PARAM_READABLE, "readable"}, + {G_PARAM_WRITABLE, "writable"}, + {} +}; + +static GstChildInspectorType types[] = { + {G_TYPE_INT, gst_child_inspector_type_int_to_string}, + {G_TYPE_STRING, gst_child_inspector_type_string_to_string}, + {} +}; + +static gchar * +gst_child_inspector_type_string_to_string (GParamSpec * pspec, GValue * value) +{ + return g_strdup_printf ("String. Default: \"%s\"", + g_value_get_string (value)); +} + +static gchar * +gst_child_inspector_type_int_to_string (GParamSpec * pspec, GValue * value) +{ + GParamSpecInt *pint = G_PARAM_SPEC_INT (pspec); + + return g_strdup_printf ("Integer. Range: %d - %d Default: %d", + pint->minimum, pint->maximum, g_value_get_int (value)); +} + +static const gchar * +gst_child_inspector_flag_to_string (GParamFlags flag) +{ + GstChildInspectorFlag *current_flag; + const gchar *to_string = NULL; + + for (current_flag = flags; current_flag->to_string; ++current_flag) { + if (current_flag->value == flag) { + to_string = current_flag->to_string; + break; + } + } + + return to_string; +} + +static gchar * +gst_child_inspector_flags_to_string (GParamFlags flags) +{ + guint32 bitflags; + gint i; + gchar *sflags = NULL; + + /* Walk through all the bits in the flags */ + bitflags = flags; + for (i = 31; i >= 0; --i) { + const gchar *sflag = NULL; + guint32 bitflag = 0; + + /* Filter the desired bit */ + bitflag = bitflags & (1 << i); + sflag = gst_child_inspector_flag_to_string (bitflag); + + /* Is this a flag we want to serialize? */ + if (sflag) { + /* No trailing comma needed on the first case */ + if (!sflags) { + sflags = g_strdup (sflag); + } else { + sflags = g_strdup_printf ("%s, %s", sflag, sflags); + } + } + } + + return sflags; +} + +static gchar * +gst_child_inspector_type_to_string (GParamSpec * pspec, GValue * value) +{ + GstChildInspectorType *current_type; + const GType value_type = G_VALUE_TYPE (value); + gchar *to_string = NULL; + + for (current_type = types; current_type->to_string; ++current_type) { + if (current_type->value == value_type) { + to_string = current_type->to_string (pspec, value); + break; + } + } + + return to_string; +} + +gchar * +gst_child_inspector_property_to_string (GObject * object, GParamSpec * param, + guint alignment) +{ + GValue value = { 0, }; + const gchar *name; + const gchar *blurb; + gchar *flags = NULL; + gchar *type; + gchar *prop; + + g_return_val_if_fail (param, NULL); + g_return_val_if_fail (G_IS_OBJECT (object), NULL); + + name = g_param_spec_get_name (param); + blurb = g_param_spec_get_blurb (param); + + flags = gst_child_inspector_flags_to_string (param->flags); + + g_value_init (&value, param->value_type); + if (param->flags & G_PARAM_READABLE) { + g_object_get_property (object, param->name, &value); + } else { + g_param_value_set_default (param, &value); + } + + type = gst_child_inspector_type_to_string (param, &value); + g_value_unset (&value); + + prop = g_strdup_printf ("%*s%-20s: %s\n" + "%*s%-21.21s flags: %s\n" + "%*s%-21.21s %s", + alignment, "", name, blurb, + alignment, "", "", flags, alignment, "", "", type); + + g_free (type); + g_free (flags); + + return prop; +} + + +gchar * +gst_child_inspector_properties_to_string (GObject * object, guint alignment, + gchar * title) +{ + GParamSpec **property_specs; + guint num_properties, i; + gchar *prop, *props = NULL; + + g_return_val_if_fail (G_IS_OBJECT (object), NULL); + + property_specs = g_object_class_list_properties + (G_OBJECT_GET_CLASS (object), &num_properties); + + if (title) + props = title; + + for (i = 0; i < num_properties; i++) { + prop = + gst_child_inspector_property_to_string (object, property_specs[i], + alignment); + if (props) { + gchar *tmp; + tmp = g_strdup_printf ("%s\n%s", props, prop); + g_free (prop); + g_free (props); + props = tmp; + } else { + props = prop; + } + } + + g_free (property_specs); + + return props; +} diff --git a/gst-libs/gst/r2inference/gstchildinspector.h b/gst-libs/gst/r2inference/gstchildinspector.h new file mode 100644 index 00000000..28392601 --- /dev/null +++ b/gst-libs/gst/r2inference/gstchildinspector.h @@ -0,0 +1,36 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_CHILD_INSPECTOR_H__ +#define __GST_CHILD_INSPECTOR_H__ + +#include + +G_BEGIN_DECLS + +gchar * gst_child_inspector_property_to_string (GObject * object, + GParamSpec * param, guint alignment); +gchar * gst_child_inspector_properties_to_string (GObject * object, + guint alignment, gchar * title); + +G_END_DECLS + +#endif /* __GST_CHILD_INSPECTOR_H__ */ diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc new file mode 100644 index 00000000..324d23c9 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -0,0 +1,135 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstinferencebackends.h" +#include "gstchildinspector.h" +#include "gstncsdk.h" +#include "gsttensorflow.h" +#include "gstbackend.h" +#include +#include +#include + +#define DEFAULT_ALIGNMENT 32 + +static std::unordered_map +backend_types ({ + {r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, + {r2i::FrameworkCode::TENSORFLOW, GST_TYPE_TENSORFLOW}, + {r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} +}); + +static void +gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, + gchar ** backends_parameters, r2i::RuntimeError error, + guint alignment); + +GType +gst_inference_backends_get_type (void) +{ + static GType backend_type = 0; + static const GEnumValue + backend_desc[] = { + {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, + {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", + "tensorflow"}, + {0, NULL, NULL} + }; + if (!backend_type) { + backend_type = + g_enum_register_static ("GstInferenceBackends", + (GEnumValue *) backend_desc); + } + return backend_type; +} + +GType +gst_inference_backends_search_type (guint id) +{ + auto search = backend_types.find (id); + if (backend_types.end () == search) { + search = backend_types.find (r2i::FrameworkCode::MAX_FRAMEWORK); + } + return search->second; +} + +static void +gst_inference_backends_add_frameworkmeta (r2i::FrameworkMeta meta, + gchar ** backends_parameters, r2i::RuntimeError error, + guint alignment) +{ + GstBackend * backend = NULL; + gchar * parameters, * backend_name; + GType backend_type; + + backend_type = gst_inference_backends_search_type (meta.code); + + if (G_TYPE_INVALID == backend_type) { + GST_ERROR_OBJECT (backend, "Failed to find Backend type: %s", + meta.name.c_str ()); + return; + } + + backend = (GstBackend *) g_object_new (backend_type, NULL); + + backend_name = + g_strdup_printf ("%*s: %s. Version: %s\n", alignment, meta.name.c_str (), + meta.description.c_str (), meta.version.c_str ()); + + parameters = + gst_child_inspector_properties_to_string (G_OBJECT (backend), alignment, + backend_name); + + if (NULL == *backends_parameters) + *backends_parameters = parameters; + else + *backends_parameters = + g_strconcat (*backends_parameters, "\n", parameters, NULL); + + g_object_unref (backend); +} + +gchar * +gst_inference_backends_get_string_properties (void) +{ + gchar * backends_parameters = NULL; + r2i::RuntimeError error; + + for (auto & meta:r2i::IFrameworkFactory::List (error)) { + gst_inference_backends_add_frameworkmeta (meta, &backends_parameters, error, + DEFAULT_ALIGNMENT); + } + + return backends_parameters; +} + +guint16 +gst_inference_backends_get_default_backend (void) +{ + std::vector backends; + r2i::FrameworkCode code; + r2i::RuntimeError error; + + backends = r2i::IFrameworkFactory::List (error); + code = backends.front().code; + + return code; +} diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h new file mode 100644 index 00000000..8d8bc8c9 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -0,0 +1,37 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_INFERENCE_BACKENDS_H__ +#define __GST_INFERENCE_BACKENDS_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_INFERENCE_BACKENDS (gst_inference_backends_get_type()) + +GType gst_inference_backends_get_type (void); +gchar * gst_inference_backends_get_string_properties (void); +guint16 gst_inference_backends_get_default_backend (void); +GType gst_inference_backends_search_type (guint); + +G_END_DECLS +#endif //__GST_INFERENCE_BACKENDS_H__ diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc new file mode 100644 index 00000000..ec797872 --- /dev/null +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -0,0 +1,55 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstncsdk.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_ncsdk_debug_category); +#define GST_CAT_DEFAULT gst_ncsdk_debug_category + +struct _GstNcsdk +{ + GstBackend parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstNcsdk, gst_ncsdk, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_ncsdk_debug_category, "ncsdk", 0, + "debug category for ncsdk parameters")); + +static void +gst_ncsdk_class_init (GstNcsdkClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + + gst_backend_install_properties (bclass, r2i::FrameworkCode::NCSDK); +} + +static void +gst_ncsdk_init (GstNcsdk * self) +{ + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::NCSDK); +} diff --git a/gst-libs/gst/r2inference/gstncsdk.h b/gst-libs/gst/r2inference/gstncsdk.h new file mode 100644 index 00000000..08689b68 --- /dev/null +++ b/gst-libs/gst/r2inference/gstncsdk.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_NCSDK_H__ +#define __GST_NCSDK_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_NCSDK gst_ncsdk_get_type () +G_DECLARE_FINAL_TYPE(GstNcsdk, gst_ncsdk, GST, NCSDK, GstBackend); + +G_END_DECLS + +#endif //__GST_NCSDK_H__ diff --git a/gst-libs/gst/r2inference/gsttensorflow.cc b/gst-libs/gst/r2inference/gsttensorflow.cc new file mode 100644 index 00000000..889cedf7 --- /dev/null +++ b/gst-libs/gst/r2inference/gsttensorflow.cc @@ -0,0 +1,54 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gsttensorflow.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tensorflow_debug_category); +#define GST_CAT_DEFAULT gst_tensorflow_debug_category + +struct _GstTensorflow +{ + GstBackend parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstTensorflow, gst_tensorflow, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_tensorflow_debug_category, "tensorflow", 0, + "debug category for tensorflow parameters")); + +static void +gst_tensorflow_class_init (GstTensorflowClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + gst_backend_install_properties (bclass, r2i::FrameworkCode::TENSORFLOW); +} + +static void +gst_tensorflow_init (GstTensorflow * self) +{ + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::TENSORFLOW); +} diff --git a/gst-libs/gst/r2inference/gsttensorflow.h b/gst-libs/gst/r2inference/gsttensorflow.h new file mode 100644 index 00000000..793eb6dc --- /dev/null +++ b/gst-libs/gst/r2inference/gsttensorflow.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_TENSORFLOW_H__ +#define __GST_TENSORFLOW_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TENSORFLOW gst_tensorflow_get_type () +G_DECLARE_FINAL_TYPE(GstTensorflow, gst_tensorflow, GST, TENSORFLOW, GstBackend); + +G_END_DECLS + +#endif //__GST_TENSORFLOW_H__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c new file mode 100644 index 00000000..a257bdc4 --- /dev/null +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -0,0 +1,828 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gstvideoinference.h" +#include "gstinferencebackends.h" +#include "gstbackend.h" + +#include + + +static GstStaticPadTemplate sink_bypass_factory = +GST_STATIC_PAD_TEMPLATE ("sink_bypass", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS ("ANY") + ); + +static GstStaticPadTemplate src_bypass_factory = +GST_STATIC_PAD_TEMPLATE ("src_bypass", + GST_PAD_SRC, + GST_PAD_REQUEST, + GST_STATIC_CAPS ("ANY") + ); + +GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); +#define GST_CAT_DEFAULT gst_video_inference_debug_category + +#define DEFAULT_MODEL_LOCATION NULL + +enum +{ + PROP_0, + PROP_BACKEND, + PROP_MODEL_LOCATION +}; + + +typedef struct _GstVideoInferencePrivate GstVideoInferencePrivate; +struct _GstVideoInferencePrivate +{ + GstCollectPads *cpads; + GstCollectData *sink_bypass_data; + GstCollectData *sink_model_data; + + GstPad *sink_bypass; + GstPad *src_bypass; + GstPad *sink_model; + GstPad *src_model; + + GstBackend *backend; + + gchar *model_location; +}; + +/* GObject methods */ +static void gst_video_inference_finalize (GObject * object); +static void gst_video_inference_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_video_inference_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); + +/* GstElement methods */ +static GstStateChangeReturn gst_video_inference_change_state (GstElement * + element, GstStateChange transition); +static GstPad *gst_video_inference_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); +static void gst_video_inference_release_pad (GstElement * element, + GstPad * pad); + +/* GstChildProxy methods */ +static void gst_video_inference_child_proxy_init (GstChildProxyInterface * + iface); +static GObject *gst_video_inference_get_child_by_name (GstChildProxy * parent, + const gchar * name); +static GObject *gst_video_inference_get_child_by_index (GstChildProxy * parent, + guint index); +static guint gst_video_inference_get_children_count (GstChildProxy * parent); + +/* GstVideoInference methods */ +static gboolean gst_video_inference_start (GstVideoInference * self); +static gboolean gst_video_inference_stop (GstVideoInference * self); +static GstPad *gst_video_inference_create_pad (GstVideoInference * self, + GstPadTemplate * templ, const gchar * name, GstCollectData ** data); +static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, + gpointer user_data); +static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * + self, GstBuffer * buffer, GstCollectData * data, GstPad * pad); +static gboolean gst_video_inference_model_buffer_process (GstVideoInference * + self, GstBuffer * buffer); +static gboolean gst_video_inference_bypass_buffer_process (GstVideoInference * + self, GstBuffer * buffer_bypass, GstBuffer * buffer_model); +static gboolean gst_video_inference_sink_event (GstCollectPads * pads, + GstCollectData * pad, GstEvent * event, gpointer user_data); +static gboolean gst_video_inference_src_event (GstPad * pad, GstObject * parent, + GstEvent * event); +static GstPad *gst_video_inference_get_src_pad (GstVideoInference * self, + GstVideoInferencePrivate * priv, GstPad * pad); +static void +gst_video_inference_set_backend (GstVideoInference * self, gint backend); +static guint gst_video_inference_get_backend_type (GstVideoInference * self); + +G_DEFINE_TYPE_WITH_CODE (GstVideoInference, gst_video_inference, + GST_TYPE_ELEMENT, + GST_DEBUG_CATEGORY_INIT (gst_video_inference_debug_category, + "videoinference", 0, "debug category for videoinference base class"); + G_ADD_PRIVATE (GstVideoInference); + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_video_inference_child_proxy_init)); + +#define GST_VIDEO_INFERENCE_PRIVATE(self) \ + (GstVideoInferencePrivate *)(gst_video_inference_get_instance_private (self)) + +static void +gst_video_inference_class_init (GstVideoInferenceClass * klass) +{ + GObjectClass *oclass = G_OBJECT_CLASS (klass); + GstElementClass *eclass = GST_ELEMENT_CLASS (klass); + gchar *backend_blurb, *backends_params = NULL; + + oclass->finalize = gst_video_inference_finalize; + oclass->set_property = gst_video_inference_set_property; + oclass->get_property = gst_video_inference_get_property; + + eclass->change_state = GST_DEBUG_FUNCPTR (gst_video_inference_change_state); + eclass->request_new_pad = + GST_DEBUG_FUNCPTR (gst_video_inference_request_new_pad); + eclass->release_pad = GST_DEBUG_FUNCPTR (gst_video_inference_release_pad); + gst_element_class_add_static_pad_template (eclass, &sink_bypass_factory); + gst_element_class_add_static_pad_template (eclass, &src_bypass_factory); + + backends_params = gst_inference_backends_get_string_properties (); + backend_blurb = g_strdup_printf ("Type of predefined backend to use.\n" + "\t\t\tAccording to the selected backend " + "different properties will be available.\n " + "\t\t\tThese properties can be accessed using the " + "\"backend::\" syntax.\n" + "\t\t\tThe following list details the properties " + "for each backend\n%s", backends_params); + + g_free (backends_params); + + g_object_class_install_property (oclass, PROP_BACKEND, + g_param_spec_enum ("backend", "Backend", backend_blurb, + GST_TYPE_INFERENCE_BACKENDS, + gst_inference_backends_get_default_backend (), G_PARAM_READWRITE)); + + g_object_class_install_property (oclass, PROP_MODEL_LOCATION, + g_param_spec_string ("model-location", "Model Location", + "Path to the model to use", DEFAULT_MODEL_LOCATION, + G_PARAM_READWRITE)); + + klass->start = NULL; + klass->stop = NULL; + klass->preprocess = NULL; + klass->postprocess = NULL; +} + +static void +gst_video_inference_init (GstVideoInference * self) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + priv->sink_bypass_data = NULL; + priv->sink_model_data = NULL; + + priv->sink_bypass = NULL; + priv->src_bypass = NULL; + priv->sink_model = NULL; + priv->src_model = NULL; + + priv->cpads = gst_collect_pads_new (); + gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, + (gpointer) (self)); + gst_collect_pads_set_event_function (priv->cpads, + gst_video_inference_sink_event, (gpointer) (self)); + + priv->model_location = g_strdup (DEFAULT_MODEL_LOCATION); + + gst_video_inference_set_backend (self, + gst_inference_backends_get_default_backend ()); +} + +static void +gst_video_inference_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstState actual_state; + + GST_LOG_OBJECT (self, "Set Property"); + + switch (property_id) { + case PROP_BACKEND: + GST_OBJECT_LOCK (self); + gst_video_inference_set_backend (self, g_value_get_enum (value)); + GST_OBJECT_UNLOCK (self); + break; + case PROP_MODEL_LOCATION: + gst_element_get_state (GST_ELEMENT (self), &actual_state, NULL, + GST_SECOND); + GST_OBJECT_LOCK (self); + if (actual_state <= GST_STATE_READY) { + g_free (priv->model_location); + priv->model_location = g_value_dup_string (value); + } else { + GST_ERROR_OBJECT (self, + "Model location can only be set in the NULL or READY states"); + } + GST_OBJECT_UNLOCK (self); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_video_inference_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_LOG_OBJECT (self, "Get Property"); + + switch (property_id) { + case PROP_BACKEND: + g_value_set_enum (value, gst_video_inference_get_backend_type (self)); + break; + case PROP_MODEL_LOCATION: + g_value_set_string (value, priv->model_location); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_video_inference_child_proxy_init (GstChildProxyInterface * iface) +{ + iface->get_child_by_name = gst_video_inference_get_child_by_name; + iface->get_child_by_index = gst_video_inference_get_child_by_index; + iface->get_children_count = gst_video_inference_get_children_count; +} + +static GObject * +gst_video_inference_get_child_by_name (GstChildProxy * parent, + const gchar * name) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_DEBUG_OBJECT (self, "Requested for child %s", name); + + if (0 == g_strcmp0 (name, "backend")) { + return G_OBJECT (g_object_ref (priv->backend)); + } else { + GST_ERROR_OBJECT (self, "No such child %s", name); + return NULL; + } +} + +static GObject * +gst_video_inference_get_child_by_index (GstChildProxy * parent, guint index) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + GST_DEBUG_OBJECT (self, "Requested for child %d", index); + + if (0 == index) { + return G_OBJECT (g_object_ref (priv->backend)); + } else { + GST_DEBUG_OBJECT (self, "No such child %d", index); + return NULL; + } +} + +static guint +gst_video_inference_get_children_count (GstChildProxy * parent) +{ + return 1; +} + +static gboolean +gst_video_inference_start (GstVideoInference * self) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + gboolean ret = TRUE; + GError *err = NULL; + + GST_INFO_OBJECT (self, "Starting video inference"); + if (NULL == priv->model_location) { + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("Model Location has not been set"), (NULL)); + ret = FALSE; + goto out; + } + + if (!gst_backend_start (priv->backend, priv->model_location, &err)) { + GST_ELEMENT_ERROR (self, LIBRARY, INIT, + ("Could not start the selected backend: (%s)", err->message), (NULL)); + ret = FALSE; + } + + if (klass->start != NULL) { + ret = klass->start (self); + } + +out: + if (err) + g_error_free (err); + return ret; +} + +static gboolean +gst_video_inference_stop (GstVideoInference * self) +{ + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + gboolean ret = TRUE; + GError *err = NULL; + + GST_INFO_OBJECT (self, "Stopping video inference"); + + if (!gst_backend_stop (priv->backend, &err)) { + GST_ELEMENT_ERROR (self, LIBRARY, INIT, + ("Could not stop the selected backend: (%s)", err->message), (NULL)); + ret = FALSE; + } + + if (klass->stop != NULL) { + ret = klass->stop (self); + } + + return ret; +} + +static GstStateChangeReturn +gst_video_inference_change_state (GstElement * element, + GstStateChange transition) +{ + GstStateChangeReturn ret; + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + if (FALSE == gst_video_inference_start (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + + gst_collect_pads_start (priv->cpads); + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_collect_pads_stop (priv->cpads); + break; + default: + break; + } + + ret = + GST_ELEMENT_CLASS (gst_video_inference_parent_class)->change_state + (element, transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + if (FALSE == gst_video_inference_stop (self)) { + GST_ERROR_OBJECT (self, "Subclass failed to stop"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + break; + default: + break; + } + +out: + return ret; +} + +static GstPad * +gst_video_inference_create_pad (GstVideoInference * self, + GstPadTemplate * templ, const gchar * name, GstCollectData ** data) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstElement *element = GST_ELEMENT (self); + GstPad *pad; + + GST_INFO_OBJECT (self, "Requested pad %s", name); + pad = gst_pad_new_from_template (templ, name); + + if (GST_PAD_IS_SINK (pad)) { + g_return_val_if_fail (data, NULL); + + *data = + gst_collect_pads_add_pad (priv->cpads, pad, sizeof (GstCollectData), + NULL, TRUE); + if (NULL == *data) { + GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); + goto free_pad; + } + } else { + gst_pad_set_event_function (pad, gst_video_inference_src_event); + } + + if (FALSE == gst_element_add_pad (element, pad)) { + GST_ERROR_OBJECT (self, "Unable to add pad %s to element", name); + goto remove_pad; + } + + return GST_PAD_CAST (gst_object_ref (pad)); + +remove_pad: + gst_collect_pads_remove_pad (priv->cpads, pad); + +free_pad: + gst_object_unref (pad); + return NULL; +} + +static GstPad * +gst_video_inference_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + const gchar *tname; + GstPad **pad; + GstCollectData **data; + + tname = GST_PAD_TEMPLATE_NAME_TEMPLATE (templ); + + if (0 == g_strcmp0 (tname, "sink_bypass")) { + pad = &priv->sink_bypass; + data = &priv->sink_bypass_data; + } else if (0 == g_strcmp0 (tname, "sink_model")) { + pad = &priv->sink_model; + data = &priv->sink_model_data; + } else if (0 == g_strcmp0 (tname, "src_bypass")) { + pad = &priv->src_bypass; + data = NULL; + } else if (0 == g_strcmp0 (tname, "src_model")) { + pad = &priv->src_model; + data = NULL; + } else { + g_return_val_if_reached (NULL); + } + + if (NULL == *pad) { + *pad = gst_video_inference_create_pad (self, templ, name, data); + } else { + GST_ERROR_OBJECT (self, "Pad %s already exists", name); + return NULL; + } + + return *pad; +} + +static void +gst_video_inference_release_pad (GstElement * element, GstPad * pad) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (element); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstPad **ourpad; + GstCollectData **data; + + GST_INFO_OBJECT (self, "Removing %s:%s", GST_DEBUG_PAD_NAME (pad)); + + if (pad == priv->sink_bypass) { + ourpad = &priv->sink_bypass; + data = &priv->sink_bypass_data; + } else if (pad == priv->src_bypass) { + ourpad = &priv->src_bypass; + data = NULL; + } else if (pad == priv->sink_model) { + ourpad = &priv->sink_model; + data = &priv->sink_model_data; + } else if (pad == priv->src_model) { + ourpad = &priv->src_model; + data = NULL; + } else { + g_return_if_reached (); + } + + if (GST_PAD_IS_SINK (pad)) { + *data = NULL; + gst_collect_pads_remove_pad (priv->cpads, pad); + } + + g_clear_object (ourpad); +} + +static GstFlowReturn +gst_video_inference_forward_buffer (GstVideoInference * self, + GstBuffer * buffer, GstCollectData * data, GstPad * pad) +{ + GstFlowReturn ret = GST_FLOW_OK; + + /* User didn't request this pad */ + if (NULL == data || pad == NULL) { + GST_LOG_OBJECT (self, "Dropping buffer from %s:%s", + GST_DEBUG_PAD_NAME (data->pad)); + goto out; + } + + GST_LOG_OBJECT (self, "Forwarding buffer to %s:%s", GST_DEBUG_PAD_NAME (pad)); + ret = gst_pad_push (pad, gst_buffer_ref (buffer)); + + if (GST_FLOW_FLUSHING == ret || GST_FLOW_EOS == ret) { + GST_INFO_OBJECT (self, "Pad %s:%s returned: (%d) %s", + GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); + } + + else if (GST_FLOW_OK != ret) { + GST_ERROR_OBJECT (self, "Pad %s:%s returned: (%d) %s", + GST_DEBUG_PAD_NAME (pad), ret, gst_flow_get_name (ret)); + } + +out: + gst_buffer_unref (buffer); + return ret; +} + +static gboolean +gst_video_inference_model_buffer_process (GstVideoInference * self, + GstBuffer * buffer) +{ + GstVideoInferenceClass *klass; + GstVideoInferencePrivate *priv; + GstBuffer *outbuf; + GstVideoFrame inframe, outframe; + GstAllocationParams params; + GstVideoInfo vininfo; + GstCaps *pad_caps; + gboolean ret; + gpointer prediction_data = NULL; + gsize prediction_size; + GError *err = NULL; + + klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + outbuf = NULL; + ret = TRUE; + + if (NULL == klass->preprocess) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass did not implement preprocess"), (NULL)); + ret = FALSE; + goto out; + } + + gst_video_info_init (&vininfo); + pad_caps = gst_pad_get_current_caps (priv->sink_model); + gst_video_info_from_caps (&vininfo, pad_caps); + gst_caps_unref (pad_caps); + + gst_allocation_params_init (¶ms); + outbuf = + gst_buffer_new_allocate (NULL, + gst_buffer_get_size (buffer) * sizeof (gfloat), ¶ms); + gst_video_frame_map (&inframe, &vininfo, buffer, GST_MAP_READ); + gst_video_frame_map (&outframe, &vininfo, outbuf, GST_MAP_WRITE); + + if (!klass->preprocess (self, &inframe, &outframe)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed to preprocess"), (NULL)); + ret = FALSE; + goto free_frames; + } + + GST_LOG_OBJECT (self, "Processing frame using selected Backend"); + + if (!gst_backend_process_frame (priv->backend, &outframe, + &prediction_data, &prediction_size, &err)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Could not process using the selected backend: (%s)", + err->message), (NULL)); + ret = FALSE; + goto free_frames; + } + + if (NULL != klass->postprocess) { + if (!klass->postprocess (self, &outframe, prediction_data, prediction_size)) { + ret = FALSE; + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed at preprocess"), (NULL)); + goto free_frames; + } + } + +free_frames: + if (prediction_data) + g_free (prediction_data); + gst_video_frame_unmap (&outframe); + gst_video_frame_unmap (&inframe); + gst_buffer_unref (outbuf); +out: + if (err) + g_error_free (err); + return ret; +} + +static gboolean +gst_video_inference_bypass_buffer_process (GstVideoInference * self, + GstBuffer * buffer_bypass, GstBuffer * buffer_model) +{ + /* TODO: Get GstMetas from Model buffer and attach them to the + Bypass Buffer */ + + return TRUE; +} + +static GstFlowReturn +gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstFlowReturn ret = GST_FLOW_OK; + GstBuffer *buffer_model = NULL; + GstBuffer *buffer_bypass = NULL; + + /* Process Model Buffer */ + if (priv->sink_model_data) { + buffer_model = gst_collect_pads_pop (priv->cpads, priv->sink_model_data); + if (NULL == buffer_model) { + GST_INFO_OBJECT (self, "EOS requested on %s:%s", + GST_DEBUG_PAD_NAME (priv->sink_model_data->pad)); + ret = GST_FLOW_EOS; + goto out; + } + if (!gst_video_inference_model_buffer_process (self, buffer_model)) { + ret = GST_FLOW_ERROR; + goto model_free; + } + } + + /* Process Bypass Buffer */ + if (priv->sink_bypass_data) { + buffer_bypass = gst_collect_pads_pop (priv->cpads, priv->sink_bypass_data); + if (NULL == buffer_bypass) { + GST_INFO_OBJECT (self, "EOS requested on %s:%s", + GST_DEBUG_PAD_NAME (priv->sink_bypass_data->pad)); + ret = GST_FLOW_EOS; + goto model_free; + } + if (!gst_video_inference_bypass_buffer_process (self, buffer_bypass, + buffer_model)) { + ret = GST_FLOW_ERROR; + goto bypass_free; + } + } + + /* Forward buffers to src pads */ + if (NULL != buffer_model) { + ret = + gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), + priv->sink_model_data, priv->src_model); + if (GST_FLOW_OK != ret) { + goto bypass_free; + } + } + + if (NULL != buffer_bypass) { + ret = + gst_video_inference_forward_buffer (self, + gst_buffer_ref (buffer_bypass), priv->sink_bypass_data, + priv->src_bypass); + if (GST_FLOW_OK != ret) { + goto bypass_free; + } + } + +bypass_free: + gst_buffer_unref (buffer_bypass); + +model_free: + gst_buffer_unref (buffer_model); + +out: + return ret; +} + +static GstPad * +gst_video_inference_get_src_pad (GstVideoInference * self, + GstVideoInferencePrivate * priv, GstPad * sinkpad) +{ + GstPad *pad; + + if (sinkpad == priv->sink_model) { + pad = priv->src_model; + } else if (sinkpad == priv->sink_bypass) { + pad = priv->src_bypass; + } else { + g_return_val_if_reached (NULL); + } + + return pad; +} + +static gboolean +gst_video_inference_sink_event (GstCollectPads * pads, GstCollectData * pad, + GstEvent * event, gpointer user_data) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + gboolean ret = FALSE; + GstPad *srcpad; + + srcpad = gst_video_inference_get_src_pad (self, priv, pad->pad); + + if (NULL != srcpad) { + GST_LOG_OBJECT (self, "Forwarding event %s from %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); + /* Collect pads will decrease the refcount of the event when we return */ + gst_event_ref (event); + if (FALSE == gst_pad_push_event (srcpad, event)) { + GST_ERROR_OBJECT (self, "Event %s failed in %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (srcpad)); + goto out; + } + } else { + GST_LOG_OBJECT (self, "Dropping event %s from %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad->pad)); + } + + ret = gst_collect_pads_event_default (priv->cpads, pad, event, FALSE); + +out: + return ret; +} + +static gboolean +gst_video_inference_src_event (GstPad * pad, GstObject * parent, + GstEvent * event) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (parent); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + return gst_collect_pads_src_event_default (priv->cpads, pad, event); +} + +static void +gst_video_inference_finalize (GObject * object) +{ + GstVideoInference *self = GST_VIDEO_INFERENCE (object); + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_clear_object (&(priv->cpads)); + g_clear_object (&(priv->sink_bypass)); + g_clear_object (&(priv->sink_model)); + g_clear_object (&(priv->src_bypass)); + g_clear_object (&(priv->src_model)); + + priv->sink_bypass_data = NULL; + priv->sink_model_data = NULL; + g_free (priv->model_location); + priv->model_location = NULL; + + g_clear_object (&priv->backend); + + if (priv->backend) + g_object_unref (priv->backend); + + G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); +} + +static void +gst_video_inference_set_backend (GstVideoInference * self, gint backend) +{ + GstBackend *backend_new; + GType backend_type; + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_return_if_fail (priv); + + if (GST_STATE (self) != GST_STATE_NULL) { + g_warning ("Can't set backend property if not on NULL state"); + return; + } + + if (priv->backend) + g_object_unref (priv->backend); + + backend_type = gst_inference_backends_search_type (backend); + backend_new = (GstBackend *) g_object_new (backend_type, NULL); + priv->backend = backend_new; + + return; +} + +static guint +gst_video_inference_get_backend_type (GstVideoInference * self) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + g_return_val_if_fail (priv, -1); + + return gst_backend_get_framework_code (priv->backend); +} diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h new file mode 100644 index 00000000..416f30a7 --- /dev/null +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -0,0 +1,47 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_VIDEO_INFERENCE_H__ +#define __GST_VIDEO_INFERENCE_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_VIDEO_INFERENCE gst_video_inference_get_type () +G_DECLARE_DERIVABLE_TYPE(GstVideoInference, gst_video_inference, GST, VIDEO_INFERENCE, GstElement); + +struct _GstVideoInferenceClass +{ + GstElementClass parent_class; + + gboolean (* start) (GstVideoInference *self); + gboolean (* stop) (GstVideoInference *self); + gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, + GstVideoFrame *outframe); + gboolean (* postprocess) (GstVideoInference *self, GstVideoFrame *outframe, + const gpointer prediction, gsize size); +}; + +G_END_DECLS + +#endif //__GST_VIDEO_INFERENCE_H__ diff --git a/m4/Makefile.am b/m4/Makefile.am new file mode 100644 index 00000000..d539a6ec --- /dev/null +++ b/m4/Makefile.am @@ -0,0 +1 @@ +EXTRA_DIST = diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 00000000..c6991a63 --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,16 @@ +if HAVE_GST_CHECK +SUBDIRS_CHECK = check files +else +SUBDIRS_CHECK = +endif + +#if BUILD_EXAMPLES +#SUBDIR_EXAMPLES = examples +#else +#SUBDIR_EXAMPLES = +#endif + +SUBDIRS = $(SUBDIRS_CHECK) $(SUBDIR_EXAMPLES) + +DIST_SUBDIRS = check files + diff --git a/tests/check/Makefile.am b/tests/check/Makefile.am new file mode 100644 index 00000000..0e9271ce --- /dev/null +++ b/tests/check/Makefile.am @@ -0,0 +1,46 @@ +include $(top_srcdir)/common/check.mak + +CHECK_REGISTRY = $(top_builddir)/tests/check/test-registry.reg +TEST_FILES_DIRECTORY = $(top_srcdir)/tests/files + +REGISTRY_ENVIRONMENT = \ + GST_REGISTRY_1_0=$(CHECK_REGISTRY) + +AM_TESTS_ENVIRONMENT += \ + $(REGISTRY_ENVIRONMENT) \ + GST_PLUGIN_SYSTEM_PATH_1_0= \ + GST_PLUGIN_PATH_1_0=$(top_builddir)/gst:$(top_builddir)/ext:$(top_builddir)/sys:$(GSTPB_PLUGINS_DIR):$(GST_PLUGINS_DIR) \ + GST_PLUGIN_LOADING_WHITELIST="gstreamer@$(GST_PLUGINS_DIR):gst-plugins-base@$(GSTPB_PLUGINS_DIR):gst-inference@$(top_builddir)" \ + GST_STATE_IGNORE_ELEMENTS= \ + GSETTINGS_BACKEND="memory" + +# the core dumps of some machines have PIDs appended +CLEANFILES = core.* test-registry.* + +clean-local: clean-local-check + +#if USE_PLUGIN_EXAMPLE +#check_example = \ +# elements/example \ +# elements/example +#else +#check_example = +#endif + +check_PROGRAMS = + +VALGRIND_TO_FIX = + +TESTS = $(check_PROGRAMS) + +AM_CFLAGS = $(GST_OBJ_CFLAGS) $(GST_CHECK_CFLAGS) $(CHECK_CFLAGS) \ + $(GST_OPTION_CFLAGS) $(GST_CFLAGS) -DGST_TEST_FILES_PATH="\"$(TEST_FILES_DIRECTORY)\"" \ + -DGST_CHECK_TEST_ENVIRONMENT_BEACON="\"GST_PLUGIN_LOADING_WHITELIST\"" \ + -UG_DISABLE_ASSERT -UG_DISABLE_CAST_CHECKS $(PTHREAD_CFLAGS) +LDADD = $(GST_OBJ_LIBS) $(GST_CHECK_LIBS) $(CHECK_LIBS) + +# valgrind testing +VALGRIND_TESTS_DISABLE = \ + $(VALGRIND_TO_FIX) + +SUPPRESSIONS = $(top_srcdir)/common/gst.supp diff --git a/tests/examples/Makefile.am b/tests/examples/Makefile.am new file mode 100644 index 00000000..f75716da --- /dev/null +++ b/tests/examples/Makefile.am @@ -0,0 +1,11 @@ +#if USE_EXAMPLE +#EXAMPLE_DIR=example +#else +#EXAMPLE_DIR= +#endif + +SUBDIRS = $(EXAMPLE_DIR) + +DIST_SUBDIRS = + +include $(top_srcdir)/common/parallel-subdirs.mak diff --git a/tests/files/Makefile.am b/tests/files/Makefile.am new file mode 100644 index 00000000..1f04c07b --- /dev/null +++ b/tests/files/Makefile.am @@ -0,0 +1,4 @@ + +EXTRA_DIST = + +