Add CXX API for Kokoro TTS 1.0 (#1802)

.github/workflows/cxx-api.yaml

@@ -103,6 +103,28 @@ jobs:
           rm kws-cxx-api
           rm -rf sherpa-onnx-kws-*
 
+      - name: Test Kokoro TTS (zh+en)
+        shell: bash
+        run: |
+          g++ -std=c++17 -o kokoro-tts-zh-en-cxx-api ./cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc \
+            -I ./build/install/include \
+            -L ./build/install/lib/ \
+            -l sherpa-onnx-cxx-api \
+            -l sherpa-onnx-c-api \
+            -l onnxruntime
+
+          curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
+          tar xf kokoro-multi-lang-v1_0.tar.bz2
+          rm kokoro-multi-lang-v1_0.tar.bz2
+
+          export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH
+          export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH
+
+          ./kokoro-tts-zh-en-cxx-api
+
+          rm kokoro-tts-zh-en-cxx-api
+          rm -rf kokoro-*
+
       - name: Test Kokoro TTS (en)
         shell: bash
         run: |

c-api-examples/kws-c-api.c

@@ -26,15 +26,15 @@ int32_t main() {
   memset(&config, 0, sizeof(config));
   config.model_config.transducer.encoder =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
-      "encoder-epoch-12-avg-2-chunk-16-left-64.onnx";
+      "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx";
 
   config.model_config.transducer.decoder =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
       "decoder-epoch-12-avg-2-chunk-16-left-64.onnx";
 
   config.model_config.transducer.joiner =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
-      "joiner-epoch-12-avg-2-chunk-16-left-64.onnx";
+      "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx";
 
   config.model_config.tokens =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
@@ -58,7 +58,8 @@ int32_t main() {
           "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n");
 
   const char *wav_filename =
-      "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav";
+      "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
+      "test_wavs/3.wav";
 
   float tail_paddings[8000] = {0};  // 0.5 seconds
 

cxx-api-examples/CMakeLists.txt

@@ -27,4 +27,7 @@ if(SHERPA_ONNX_ENABLE_TTS)
 
   add_executable(kokoro-tts-en-cxx-api ./kokoro-tts-en-cxx-api.cc)
   target_link_libraries(kokoro-tts-en-cxx-api sherpa-onnx-cxx-api)
+
+  add_executable(kokoro-tts-zh-en-cxx-api ./kokoro-tts-zh-en-cxx-api.cc)
+  target_link_libraries(kokoro-tts-zh-en-cxx-api sherpa-onnx-cxx-api)
 endif()

cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc

@@ -0,0 +1,74 @@
+// cxx-api-examples/kokoro-tts-zh-en-cxx-api.c
+//
+// Copyright (c)  2025  Xiaomi Corporation
+
+// This file shows how to use sherpa-onnx CXX API
+// for Chinese TTS with Kokoro.
+//
+// clang-format off
+/*
+Usage
+
+wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
+tar xf kokoro-multi-lang-v1_0.tar.bz2
+rm kokoro-multi-lang-v1_0.tar.bz2
+
+./kokoro-tts-zh-en-cxx-api
+
+ */
+// clang-format on
+
+#include <string>
+
+#include "sherpa-onnx/c-api/cxx-api.h"
+
+static int32_t ProgressCallback(const float *samples, int32_t num_samples,
+                                float progress, void *arg) {
+  fprintf(stderr, "Progress: %.3f%%\n", progress * 100);
+  // return 1 to continue generating
+  // return 0 to stop generating
+  return 1;
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  using namespace sherpa_onnx::cxx;  // NOLINT
+  OfflineTtsConfig config;
+
+  config.model.kokoro.model = "./kokoro-multi-lang-v1_0/model.onnx";
+  config.model.kokoro.voices = "./kokoro-multi-lang-v1_0/voices.bin";
+  config.model.kokoro.tokens = "./kokoro-multi-lang-v1_0/tokens.txt";
+  config.model.kokoro.data_dir = "./kokoro-multi-lang-v1_0/espeak-ng-data";
+  config.model.kokoro.dict_dir = "./kokoro-multi-lang-v1_0/dict";
+  config.model.kokoro.lexicon =
+      "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/"
+      "lexicon-zh.txt";
+
+  config.model.num_threads = 2;
+
+  // If you don't want to see debug messages, please set it to 0
+  config.model.debug = 1;
+
+  std::string filename = "./generated-kokoro-zh-en-cxx.wav";
+  std::string text =
+      "中英文语音合成测试。This is generated by next generation Kaldi using "
+      "Kokoro without Misaki. 你觉得中英文说的如何呢？";
+
+  auto tts = OfflineTts::Create(config);
+  int32_t sid = 50;
+  float speed = 1.0;  // larger -> faster in speech speed
+
+#if 0
+  // If you don't want to use a callback, then please enable this branch
+  GeneratedAudio audio = tts.Generate(text, sid, speed);
+#else
+  GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback);
+#endif
+
+  WriteWave(filename, {audio.samples, audio.sample_rate});
+
+  fprintf(stderr, "Input text is: %s\n", text.c_str());
+  fprintf(stderr, "Speaker ID is is: %d\n", sid);
+  fprintf(stderr, "Saved to: %s\n", filename.c_str());
+
+  return 0;
+}

cxx-api-examples/kws-cxx-api.cc

@@ -25,15 +25,15 @@ int32_t main() {
   KeywordSpotterConfig config;
   config.model_config.transducer.encoder =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
-      "encoder-epoch-12-avg-2-chunk-16-left-64.onnx";
+      "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx";
 
   config.model_config.transducer.decoder =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
       "decoder-epoch-12-avg-2-chunk-16-left-64.onnx";
 
   config.model_config.transducer.joiner =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
-      "joiner-epoch-12-avg-2-chunk-16-left-64.onnx";
+      "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx";
 
   config.model_config.tokens =
       "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
@@ -57,7 +57,8 @@ int32_t main() {
       << "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n";
 
   std::string wave_filename =
-      "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav";
+      "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/"
+      "test_wavs/3.wav";
 
   std::array<float, 8000> tail_paddings = {0};  // 0.5 seconds
 

sherpa-onnx/c-api/cxx-api.cc

@@ -343,6 +343,8 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) {
   c.model.kokoro.tokens = config.model.kokoro.tokens.c_str();
   c.model.kokoro.data_dir = config.model.kokoro.data_dir.c_str();
   c.model.kokoro.length_scale = config.model.kokoro.length_scale;
+  c.model.kokoro.dict_dir = config.model.kokoro.dict_dir.c_str();
+  c.model.kokoro.lexicon = config.model.kokoro.lexicon.c_str();
 
   c.model.num_threads = config.model.num_threads;
   c.model.debug = config.model.debug;

sherpa-onnx/c-api/cxx-api.h

@@ -343,6 +343,8 @@ struct OfflineTtsKokoroModelConfig {
   std::string voices;
   std::string tokens;
   std::string data_dir;
+  std::string dict_dir;
+  std::string lexicon;
 
   float length_scale = 1.0;  // < 1, faster in speed; > 1, slower in speed
 };