Support OpenVINO as Execution Provider

k2-fsa · Jan 24, 2024 · 946cd90 · 946cd90
1 parent b6c0209
commit 946cd90
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 1 deletion.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,12 +14,13 @@ endif()
 option(SHERPA_ONNX_ENABLE_PYTHON "Whether to build Python" OFF)
 option(SHERPA_ONNX_ENABLE_TESTS "Whether to build tests" OFF)
 option(SHERPA_ONNX_ENABLE_CHECK "Whether to build with assert" OFF)
-option(BUILD_SHARED_LIBS "Whether to build shared libraries" OFF)
+option(BUILD_SHARED_LIBS "Whether to build shared libraries" ON)
 option(SHERPA_ONNX_ENABLE_PORTAUDIO "Whether to build with portaudio" ON)
 option(SHERPA_ONNX_ENABLE_JNI "Whether to build JNI internface" OFF)
 option(SHERPA_ONNX_ENABLE_C_API "Whether to build C API" ON)
 option(SHERPA_ONNX_ENABLE_WEBSOCKET "Whether to build webscoket server/client" ON)
 option(SHERPA_ONNX_ENABLE_GPU "Enable ONNX Runtime GPU support" OFF)
+option(SHERPA_ONNX_ENABLE_OPENVINO "Enable ONNX Runtime OpenVINO support" ON)
 option(SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY "True to link libstdc++ statically. Used only when BUILD_SHARED_LIBS is OFF on Linux" ON)
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
@@ -99,6 +100,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_JNI ${SHERPA_ONNX_ENABLE_JNI}")
 message(STATUS "SHERPA_ONNX_ENABLE_C_API ${SHERPA_ONNX_ENABLE_C_API}")
 message(STATUS "SHERPA_ONNX_ENABLE_WEBSOCKET ${SHERPA_ONNX_ENABLE_WEBSOCKET}")
 message(STATUS "SHERPA_ONNX_ENABLE_GPU ${SHERPA_ONNX_ENABLE_GPU}")
+message(STATUS "SHERPA_ONNX_ENABLE_OPENVINO ${SHERPA_ONNX_ENABLE_OPENVINO}")
 
 if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ version to be used.")

diff --git a/cmake/onnxruntime-win-x64.cmake b/cmake/onnxruntime-win-x64.cmake
@@ -78,6 +78,56 @@ file(COPY ${onnxruntime_SOURCE_DIR}/lib/onnxruntime.dll
     ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}
 )
 
+if(SHERPA_ONNX_ENABLE_OPENVINO)
+
+# for onnxruntime_providers_openvino.dll
+
+find_library(location_onnxruntime_providers_openvino_lib onnxruntime_providers_openvino
+  PATHS
+  "${onnxruntime_SOURCE_DIR}/lib"
+  NO_CMAKE_SYSTEM_PATH
+)
+message(STATUS "location_onnxruntime_providers_openvino_lib: ${location_onnxruntime_providers_openvino_lib}")
+
+add_library(onnxruntime_providers_openvino SHARED IMPORTED)
+set_target_properties(onnxruntime_providers_openvino PROPERTIES
+  IMPORTED_LOCATION ${location_onnxruntime_providers_openvino_lib}
+  INTERFACE_INCLUDE_DIRECTORIES "${onnxruntime_SOURCE_DIR}/include"
+)
+
+set_property(TARGET onnxruntime_providers_openvino
+  PROPERTY
+    IMPORTED_IMPLIB "${onnxruntime_SOURCE_DIR}/lib/onnxruntime_providers_openvino.lib"
+)
+
+# for onnxruntime_providers_shared.dll
+
+find_library(location_onnxruntime_providers_shared_lib onnxruntime_providers_shared
+  PATHS
+  "${onnxruntime_SOURCE_DIR}/lib"
+  NO_CMAKE_SYSTEM_PATH
+)
+message(STATUS "location_onnxruntime_providers_shared_lib: ${location_onnxruntime_providers_shared_lib}")
+add_library(onnxruntime_providers_shared SHARED IMPORTED)
+set_target_properties(onnxruntime_providers_shared PROPERTIES
+  IMPORTED_LOCATION ${location_onnxruntime_providers_shared_lib}
+  INTERFACE_INCLUDE_DIRECTORIES "${onnxruntime_SOURCE_DIR}/include"
+)
+set_property(TARGET onnxruntime_providers_shared
+  PROPERTY
+    IMPORTED_IMPLIB "${onnxruntime_SOURCE_DIR}/lib/onnxruntime_providers_shared.lib"
+)
+
+file(
+  COPY
+    ${onnxruntime_SOURCE_DIR}/lib/onnxruntime_providers_openvino.dll
+    ${onnxruntime_SOURCE_DIR}/lib/onnxruntime_providers_shared.dll
+  DESTINATION
+    ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}
+)
+
+endif()
+
 file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/lib/*.dll")
 
 message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}")

diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt
@@ -157,6 +157,13 @@ if(SHERPA_ONNX_ENABLE_GPU)
   )
 endif()
 
+if(SHERPA_ONNX_ENABLE_OPENVINO)
+  target_link_libraries(sherpa-onnx-core
+    onnxruntime_providers_openvino
+    onnxruntime_providers_shared
+  )
+endif()
+
 target_link_libraries(sherpa-onnx-core piper_phonemize)
 
 if(SHERPA_ONNX_ENABLE_CHECK)

diff --git a/sherpa-onnx/csrc/provider.cc b/sherpa-onnx/csrc/provider.cc
@@ -16,6 +16,8 @@ Provider StringToProvider(std::string s) {
                  [](unsigned char c) { return std::tolower(c); });
   if (s == "cpu") {
     return Provider::kCPU;
+  } else if (s == "openvino") {
+    return Provider::kOpenVINO;
   } else if (s == "cuda") {
     return Provider::kCUDA;
   } else if (s == "coreml") {

diff --git a/sherpa-onnx/csrc/provider.h b/sherpa-onnx/csrc/provider.h
@@ -16,6 +16,7 @@ enum class Provider {
   kCPU = 0,     // CPUExecutionProvider
   kCUDA = 1,    // CUDAExecutionProvider
   kCoreML = 2,  // CoreMLExecutionProvider
+  kOpenVINO = 3, // OpenVINOExecutionProvider
 };
 
 /**

diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc
@@ -33,6 +33,17 @@ static Ort::SessionOptions GetSessionOptionsImpl(int32_t num_threads,
   switch (p) {
     case Provider::kCPU:
       break;  // nothing to do for the CPU provider
+    case Provider::kOpenVINO:
+      {
+        // Using OPENVINO backend
+        OrtOpenVINOProviderOptions options;
+        options.device_type = "CPU_FP32"; //Other options are: GPU_FP32, GPU_FP16, MYRIAD_FP16
+        // std::cout << "OpenVINO device type is set to: " << options.device_type << std::endl;
+        fprintf(stderr, "--------------> Execution Provider: OpenVINO\n");
+        sess_opts.AppendExecutionProvider_OpenVINO(options);
+        fprintf(stderr, "<-------------- Execution Provider: OpenVINO\n");
+      }   
+      break;
     case Provider::kCUDA: {
       std::vector<std::string> available_providers =
           Ort::GetAvailableProviders();