Add C++ API for non-streaming ASR (#1456)

k2-fsa · Oct 23, 2024 · ceb69eb · ceb69eb
1 parent effd5ef
commit ceb69eb
Show file tree

Hide file tree

Showing 31 changed files with 604 additions and 43 deletions.
diff --git a/.github/scripts/test-cxx-api.sh b/.github/scripts/test-cxx-api.sh
@@ -9,6 +9,8 @@ log() {
 }
 
 echo "CXX_STREAMING_ZIPFORMER_EXE is $CXX_STREAMING_ZIPFORMER_EXE"
+echo "CXX_WHISPER_EXE is $CXX_WHISPER_EXE"
+echo "CXX_SENSE_VOICE_EXE is $CXX_SENSE_VOICE_EXE"
 echo "PATH: $PATH"
 
 log "------------------------------------------------------------"
@@ -19,3 +21,22 @@ tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 $CXX_STREAMING_ZIPFORMER_EXE
 rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
+
+log "------------------------------------------------------------"
+log "Test Whisper CXX API"
+log "------------------------------------------------------------"
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2
+tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
+rm sherpa-onnx-whisper-tiny.en.tar.bz2
+$CXX_WHISPER_EXE
+rm -rf sherpa-onnx-whisper-tiny.en
+
+log "------------------------------------------------------------"
+log "Test SenseVoice CXX API"
+log "------------------------------------------------------------"
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+
+$CXX_SENSE_VOICE_EXE
+rm -rf sherpa-onnx-sense-voice-*
diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - cxx-api-asr-non-streaming
     paths:
       - '.github/workflows/cxx-api.yaml'
       - 'CMakeLists.txt'
@@ -82,6 +83,74 @@ jobs:
             otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib
           fi
 
+      - name: Test whisper
+        shell: bash
+        run: |
+          g++ -std=c++17 -o whisper-cxx-api ./cxx-api-examples/whisper-cxx-api.cc \
+            -I ./build/install/include \
+            -L ./build/install/lib/ \
+            -l sherpa-onnx-cxx-api \
+            -l sherpa-onnx-c-api \
+            -l onnxruntime
+
+          ls -lh whisper-cxx-api
+
+          if [[ ${{ matrix.os }} == ubuntu-latest ]]; then
+            ldd ./whisper-cxx-api
+            echo "----"
+            readelf -d ./whisper-cxx-api
+          fi
+
+          curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2
+          tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
+          rm sherpa-onnx-whisper-tiny.en.tar.bz2
+
+          ls -lh sherpa-onnx-whisper-tiny.en
+          echo "---"
+          ls -lh sherpa-onnx-whisper-tiny.en/test_wavs
+
+          export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
+          export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
+
+          ./whisper-cxx-api
+
+          rm -rf sherpa-onnx-whisper-*
+          rm ./whisper-cxx-api
+
+      - name: Test SenseVoice
+        shell: bash
+        run: |
+          g++ -std=c++17 -o sense-voice-cxx-api ./cxx-api-examples/sense-voice-cxx-api.cc \
+            -I ./build/install/include \
+            -L ./build/install/lib/ \
+            -l sherpa-onnx-cxx-api \
+            -l sherpa-onnx-c-api \
+            -l onnxruntime
+
+          ls -lh sense-voice-cxx-api
+
+          if [[ ${{ matrix.os }} == ubuntu-latest ]]; then
+            ldd ./sense-voice-cxx-api
+            echo "----"
+            readelf -d ./sense-voice-cxx-api
+          fi
+
+          curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+          tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+          rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
+
+          ls -lh sherpa-onnx-sense-voice-*
+          echo "---"
+          ls -lh sherpa-onnx-sense-voice-*/test_wavs
+
+          export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
+          export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
+
+          ./sense-voice-cxx-api
+
+          rm -rf sherpa-onnx-sense-voice-*
+          rm ./sense-voice-cxx-api
+
       - name: Test streaming zipformer
         shell: bash
         run: |

diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml
@@ -155,6 +155,8 @@ jobs:
           du -h -d1 .
           export PATH=$PWD/build/bin:$PATH
           export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api
+          export CXX_WHISPER_EXE=whisper-cxx-api
+          export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api
 
           .github/scripts/test-cxx-api.sh
           du -h -d1 .

diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml
@@ -127,6 +127,8 @@ jobs:
           du -h -d1 .
           export PATH=$PWD/build/bin:$PATH
           export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api
+          export CXX_WHISPER_EXE=whisper-cxx-api
+          export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api
 
           .github/scripts/test-cxx-api.sh
           du -h -d1 .

diff --git a/.github/workflows/sanitizer.yaml b/.github/workflows/sanitizer.yaml
@@ -81,6 +81,7 @@ jobs:
         run: |
           export PATH=$PWD/build/bin:$PATH
           export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api
+          export CXX_WHISPER_EXE=whisper-cxx-api
 
           .github/scripts/test-cxx-api.sh
 

diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml
@@ -98,6 +98,8 @@ jobs:
         run: |
           export PATH=$PWD/build/bin/Release:$PATH
           export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe
+          export CXX_WHISPER_EXE=whisper-cxx-api.exe
+          export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api.exe
 
           .github/scripts/test-cxx-api.sh
 

diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml
@@ -98,6 +98,8 @@ jobs:
         run: |
           export PATH=$PWD/build/bin/Release:$PATH
           export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe
+          export CXX_WHISPER_EXE=whisper-cxx-api.exe
+          export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api.exe
 
           .github/scripts/test-cxx-api.sh
 

diff --git a/c-api-examples/paraformer-c-api.c b/c-api-examples/paraformer-c-api.c
@@ -54,7 +54,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -63,7 +63,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+  const SherpaOnnxOfflineStream *stream =
+      SherpaOnnxCreateOfflineStream(recognizer);
 
   SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
                                   wave->num_samples);

diff --git a/c-api-examples/sense-voice-c-api.c b/c-api-examples/sense-voice-c-api.c
@@ -56,7 +56,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -65,7 +65,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+  const SherpaOnnxOfflineStream *stream =
+      SherpaOnnxCreateOfflineStream(recognizer);
 
   SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
                                   wave->num_samples);

diff --git a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c
@@ -107,7 +107,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer);
+  const SherpaOnnxOnlineStream *stream =
+      SherpaOnnxCreateOnlineStream(recognizer);
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

diff --git a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c
@@ -108,7 +108,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer);
+  const SherpaOnnxOnlineStream *stream =
+      SherpaOnnxCreateOnlineStream(recognizer);
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

diff --git a/c-api-examples/streaming-paraformer-c-api.c b/c-api-examples/streaming-paraformer-c-api.c
@@ -66,7 +66,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer);
+  const SherpaOnnxOnlineStream *stream =
+      SherpaOnnxCreateOnlineStream(recognizer);
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c
@@ -130,7 +130,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer);
+  const SherpaOnnxOnlineStream *stream =
+      SherpaOnnxCreateOnlineStream(recognizer);
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

diff --git a/c-api-examples/streaming-zipformer-c-api.c b/c-api-examples/streaming-zipformer-c-api.c
@@ -72,7 +72,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer);
+  const SherpaOnnxOnlineStream *stream =
+      SherpaOnnxCreateOnlineStream(recognizer);
 
   const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
   int32_t segment_id = 0;

diff --git a/c-api-examples/telespeech-c-api.c b/c-api-examples/telespeech-c-api.c
@@ -49,7 +49,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -58,7 +58,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+  const SherpaOnnxOfflineStream *stream =
+      SherpaOnnxCreateOfflineStream(recognizer);
 
   SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
                                   wave->num_samples);

diff --git a/c-api-examples/vad-sense-voice-c-api.c b/c-api-examples/vad-sense-voice-c-api.c
@@ -66,7 +66,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -108,8 +108,9 @@ int32_t main() {
       const SherpaOnnxSpeechSegment *segment =
           SherpaOnnxVoiceActivityDetectorFront(vad);
 
-      SherpaOnnxOfflineStream *stream =
+      const SherpaOnnxOfflineStream *stream =
           SherpaOnnxCreateOfflineStream(recognizer);
+
       SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate,
                                       segment->samples, segment->n);
 
@@ -138,7 +139,9 @@ int32_t main() {
     const SherpaOnnxSpeechSegment *segment =
         SherpaOnnxVoiceActivityDetectorFront(vad);
 
-    SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+    const SherpaOnnxOfflineStream *stream =
+        SherpaOnnxCreateOfflineStream(recognizer);
+
     SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples,
                                     segment->n);
 

diff --git a/c-api-examples/whisper-c-api.c b/c-api-examples/whisper-c-api.c
@@ -58,7 +58,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -69,7 +69,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+  const SherpaOnnxOfflineStream *stream =
+      SherpaOnnxCreateOfflineStream(recognizer);
 
   SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
                                   wave->num_samples);

diff --git a/c-api-examples/zipformer-c-api.c b/c-api-examples/zipformer-c-api.c
@@ -60,7 +60,7 @@ int32_t main() {
   recognizer_config.decoding_method = "greedy_search";
   recognizer_config.model_config = offline_model_config;
 
-  SherpaOnnxOfflineRecognizer *recognizer =
+  const SherpaOnnxOfflineRecognizer *recognizer =
       SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
 
   if (recognizer == NULL) {
@@ -69,7 +69,8 @@ int32_t main() {
     return -1;
   }
 
-  SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
+  const SherpaOnnxOfflineStream *stream =
+      SherpaOnnxCreateOfflineStream(recognizer);
 
   SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples,
                                   wave->num_samples);

diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt
@@ -2,3 +2,9 @@ include_directories(${CMAKE_SOURCE_DIR})
 
 add_executable(streaming-zipformer-cxx-api ./streaming-zipformer-cxx-api.cc)
 target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api)
+
+add_executable(whisper-cxx-api ./whisper-cxx-api.cc)
+target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api)
+
+add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc)
+target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api)