Enable to stop TTS generation (#1041)

CMakeLists.txt

@@ -8,7 +8,7 @@ project(sherpa-onnx)
 # ./nodejs-addon-examples
 # ./dart-api-examples/
 # ./sherpa-onnx/flutter/CHANGELOG.md
-set(SHERPA_ONNX_VERSION "1.10.0")
+set(SHERPA_ONNX_VERSION "1.10.1")
 
 # Disable warning about
 #

android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -26,6 +26,9 @@ class MainActivity : AppCompatActivity() {
     private lateinit var speed: EditText
     private lateinit var generate: Button
     private lateinit var play: Button
+    private lateinit var stop: Button
+    private var stopped: Boolean = false
+    private var mediaPlayer: MediaPlayer? = null
 
     // see
     // https://developer.android.com/reference/kotlin/android/media/AudioTrack
@@ -49,9 +52,11 @@ class MainActivity : AppCompatActivity() {
 
         generate = findViewById(R.id.generate)
         play = findViewById(R.id.play)
+        stop = findViewById(R.id.stop)
 
         generate.setOnClickListener { onClickGenerate() }
         play.setOnClickListener { onClickPlay() }
+        stop.setOnClickListener { onClickStop() }
 
         sid.setText("0")
         speed.setText("1.0")
@@ -70,7 +75,7 @@ class MainActivity : AppCompatActivity() {
             AudioFormat.CHANNEL_OUT_MONO,
             AudioFormat.ENCODING_PCM_FLOAT
         )
-        Log.i(TAG, "sampleRate: ${sampleRate}, buffLength: ${bufLength}")
+        Log.i(TAG, "sampleRate: $sampleRate, buffLength: $bufLength")
 
         val attr = AudioAttributes.Builder().setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
             .setUsage(AudioAttributes.USAGE_MEDIA)
@@ -90,8 +95,14 @@ class MainActivity : AppCompatActivity() {
     }
 
     // this function is called from C++
-    private fun callback(samples: FloatArray) {
-        track.write(samples, 0, samples.size, AudioTrack.WRITE_BLOCKING)
+    private fun callback(samples: FloatArray): Int {
+        if (!stopped) {
+            track.write(samples, 0, samples.size, AudioTrack.WRITE_BLOCKING)
+            return 1
+        } else {
+            track.stop()
+            return 0
+        }
     }
 
     private fun onClickGenerate() {
@@ -127,6 +138,8 @@ class MainActivity : AppCompatActivity() {
         track.play()
 
         play.isEnabled = false
+        generate.isEnabled = false
+        stopped = false
         Thread {
             val audio = tts.generateWithCallback(
                 text = textStr,
@@ -140,6 +153,7 @@ class MainActivity : AppCompatActivity() {
             if (ok) {
                 runOnUiThread {
                     play.isEnabled = true
+                    generate.isEnabled = true
                     track.stop()
                 }
             }
@@ -148,11 +162,22 @@ class MainActivity : AppCompatActivity() {
 
     private fun onClickPlay() {
         val filename = application.filesDir.absolutePath + "/generated.wav"
-        val mediaPlayer = MediaPlayer.create(
+        mediaPlayer?.stop()
+        mediaPlayer = MediaPlayer.create(
             applicationContext,
             Uri.fromFile(File(filename))
         )
-        mediaPlayer.start()
+        mediaPlayer?.start()
+    }
+
+    private fun onClickStop() {
+        stopped = true
+        play.isEnabled = true
+        generate.isEnabled = true
+        track.pause()
+        track.flush()
+        mediaPlayer?.stop()
+        mediaPlayer = null
     }
 
     private fun initTts() {

android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt

@@ -76,7 +76,7 @@ class OfflineTts(
         text: String,
         sid: Int = 0,
         speed: Float = 1.0f,
-        callback: (samples: FloatArray) -> Unit
+        callback: (samples: FloatArray) -> Int
     ): GeneratedAudio {
         val objArray = generateWithCallbackImpl(
             ptr,
@@ -146,7 +146,7 @@ class OfflineTts(
         text: String,
         sid: Int = 0,
         speed: Float = 1.0f,
-        callback: (samples: FloatArray) -> Unit
+        callback: (samples: FloatArray) -> Int
     ): Array<Any>
 
     companion object {

android/SherpaOnnxTts/app/src/main/res/layout/activity_main.xml

@@ -84,4 +84,16 @@
         app:layout_constraintLeft_toLeftOf="parent"
         app:layout_constraintRight_toRightOf="parent"
         app:layout_constraintTop_toBottomOf="@id/generate" />
+
+    <Button
+        android:id="@+id/stop"
+        android:textAllCaps="false"
+        android:layout_width="match_parent"
+        android:layout_height="50dp"
+        android:layout_marginTop="4dp"
+        android:text="@string/stop"
+        app:layout_constraintLeft_toLeftOf="parent"
+        app:layout_constraintRight_toRightOf="parent"
+        app:layout_constraintTop_toBottomOf="@id/play" />
+
 </androidx.constraintlayout.widget.ConstraintLayout>

android/SherpaOnnxTts/app/src/main/res/values/strings.xml

@@ -7,4 +7,5 @@
     <string name="text_hint">Please input your text here</string>
     <string name="generate">Generate</string>
     <string name="play">Play</string>
+    <string name="stop">Stop</string>
 </resources>

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt

@@ -126,7 +126,7 @@ class TtsService : TextToSpeechService() {
             return
         }
 
-        val ttsCallback = { floatSamples: FloatArray ->
+        val ttsCallback: (FloatArray) -> Int = fun(floatSamples): Int {
             // convert FloatArray to ByteArray
             val samples = floatArrayToByteArray(floatSamples)
             val maxBufferSize: Int = callback.maxBufferSize
@@ -137,6 +137,9 @@ class TtsService : TextToSpeechService() {
                 offset += bytesToWrite
             }
 
+            // 1 means to continue
+            // 0 means to stop
+            return 1
         }
 
         Log.i(TAG, "text: $text")
@@ -160,4 +163,4 @@ class TtsService : TextToSpeechService() {
         }
         return byteArray
     }
-}
+}

dart-api-examples/non-streaming-asr/pubspec.yaml

@@ -10,7 +10,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.0
+  sherpa_onnx: ^1.10.1
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/streaming-asr/pubspec.yaml

@@ -11,7 +11,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.0
+  sherpa_onnx: ^1.10.1
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/tts/bin/piper.dart

@@ -68,6 +68,10 @@ void main(List<String> arguments) async {
       callback: (Float32List samples) {
         print('${samples.length} samples received');
         // You can play samples in a separate thread/isolate
+
+        // 1 means to continue
+        // 0 means to stop
+        return 1;
       });
   tts.free();
 

dart-api-examples/tts/pubspec.yaml

@@ -8,7 +8,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.0
+  sherpa_onnx: ^1.10.1
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/vad/pubspec.yaml

@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.0
+  sherpa_onnx: ^1.10.1
   path: ^1.9.0
   args: ^2.5.0
 

dotnet-examples/offline-tts-play/Program.cs

@@ -187,6 +187,10 @@ private static void Run(Options options)
       Marshal.Copy(samples, data, 0, n);
 
       dataItems.Add(data);
+
+      // 1 means to keep generating
+      // 0 means to stop generating
+      return 1;
     };
 
     bool playFinished = false;

kotlin-api-examples/test_tts.kt

@@ -25,6 +25,46 @@ fun testTts() {
   println("Saved to test-en.wav")
 }
 
-fun callback(samples: FloatArray): Unit {
+/*
+1. Unzip test_tts.jar
+2.
+javap ./com/k2fsa/sherpa/onnx/Test_ttsKt\$testTts\$audio\$1.class
+
+3. It prints:
+Compiled from "test_tts.kt"
+final class com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1 extends kotlin.jvm.internal.FunctionReferenceImpl implements kotlin.jvm.functions.Function1<float[], java.lang.Integer> {
+  public static final com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1 INSTANCE;
+  com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1();
+  public final java.lang.Integer invoke(float[]);
+  public java.lang.Object invoke(java.lang.Object);
+  static {};
+}
+
+4.
+javap -s ./com/k2fsa/sherpa/onnx/Test_ttsKt\$testTts\$audio\$1.class
+
+5. It prints
+Compiled from "test_tts.kt"
+final class com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1 extends kotlin.jvm.internal.FunctionReferenceImpl implements kotlin.jvm.functions.Function1<float[], java.lang.Integer> {
+  public static final com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1 INSTANCE;
+    descriptor: Lcom/k2fsa/sherpa/onnx/Test_ttsKt$testTts$audio$1;
+  com.k2fsa.sherpa.onnx.Test_ttsKt$testTts$audio$1();
+    descriptor: ()V
+
+  public final java.lang.Integer invoke(float[]);
+    descriptor: ([F)Ljava/lang/Integer;
+
+  public java.lang.Object invoke(java.lang.Object);
+    descriptor: (Ljava/lang/Object;)Ljava/lang/Object;
+
+  static {};
+    descriptor: ()V
+}
+*/
+fun callback(samples: FloatArray): Int {
   println("callback got called with ${samples.size} samples");
+
+  // 1 means to continue
+  // 0 means to stop
+  return 1
 }

mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.cpp

@@ -57,7 +57,7 @@ static bool g_started = false;
 static bool g_stopped = false;
 static bool g_killed = false;
 
-static void AudioGeneratedCallback(const float *s, int32_t n) {
+static int32_t AudioGeneratedCallback(const float *s, int32_t n) {
   if (n > 0) {
     Samples samples;
     samples.data = std::vector<float>{s, s + n};
@@ -66,6 +66,10 @@ static void AudioGeneratedCallback(const float *s, int32_t n) {
     g_buffer.samples.push(std::move(samples));
     g_started = true;
   }
+  if (g_killed) {
+    return 0;
+  }
+  return 1;
 }
 
 static int PlayCallback(const void * /*in*/, void *out,
@@ -324,6 +328,7 @@ BEGIN_MESSAGE_MAP(CNonStreamingTextToSpeechDlg, CDialogEx)
 	ON_WM_PAINT()
 	ON_WM_QUERYDRAGICON()
         ON_BN_CLICKED(IDOK, &CNonStreamingTextToSpeechDlg::OnBnClickedOk)
+        ON_BN_CLICKED(IDC_STOP, &CNonStreamingTextToSpeechDlg::OnBnClickedStop)
         END_MESSAGE_MAP()
 
 
@@ -492,11 +497,18 @@ void CNonStreamingTextToSpeechDlg::Init() {
   if (tts_) {
     SherpaOnnxDestroyOfflineTts(tts_);
   }
+  if (generate_thread_ && generate_thread_->joinable()) {
+    generate_thread_->join();
+  }
+
+  if (play_thread_ && play_thread_->joinable()) {
+    play_thread_->join();
+  }
  }
 
 
  static std::string ToString(const CString &s) {
-    CT2CA pszConvertedAnsiString( s);
+    CT2CA pszConvertedAnsiString(s);
     return std::string(pszConvertedAnsiString);
  }
 
@@ -510,7 +522,7 @@ void CNonStreamingTextToSpeechDlg::OnBnClickedOk() {
   }
 
   speed_.GetWindowText(s);
-  float speed = static_cast<float>(_ttof(s)); 
+  float speed = static_cast<float>(_ttof(s));
   if (speed < 0) {
     AfxMessageBox(Utf8ToUtf16("Please input a valid speed").c_str(), MB_OK);
     return;
@@ -541,28 +553,40 @@ void CNonStreamingTextToSpeechDlg::OnBnClickedOk() {
   // for simplicity
   play_thread_ = std::make_unique<std::thread>(StartPlayback, SherpaOnnxOfflineTtsSampleRate(tts_));
 
-  generate_btn_.EnableWindow(FALSE);
-
-  const SherpaOnnxGeneratedAudio *audio =
-      SherpaOnnxOfflineTtsGenerateWithCallback(tts_, ss.c_str(), speaker_id, speed, &AudioGeneratedCallback);
-
-  generate_btn_.EnableWindow(TRUE);
+  if (generate_thread_ && generate_thread_->joinable()) {
+    generate_thread_->join();
+  }
 
   output_filename_.GetWindowText(s);
   std::string filename = ToString(s);
 
-  int ok = SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate,
-                    filename.c_str());
+  generate_thread_ = std::make_unique<std::thread>([ss, this,filename, speaker_id, speed]() {
+      std::string text = ss;
 
-  SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio);
+      // generate_btn_.EnableWindow(FALSE);
 
-  if (ok) {
-    // AfxMessageBox(Utf8ToUtf16(std::string("Saved to ") + filename + " successfully").c_str(), MB_OK);
-    AppendLineToMultilineEditCtrl(my_hint_, std::string("Saved to ") + filename + " successfully");
-  } else {
-    // AfxMessageBox(Utf8ToUtf16(std::string("Failed to save to ") + filename).c_str(), MB_OK);
-    AppendLineToMultilineEditCtrl(my_hint_, std::string("Failed to saved to ") + filename);
-  }
+	  const SherpaOnnxGeneratedAudio *audio =
+		  SherpaOnnxOfflineTtsGenerateWithCallback(tts_, text.c_str(), speaker_id, speed, &AudioGeneratedCallback);
+      // generate_btn_.EnableWindow(TRUE);
+       g_stopped = true;
+
+	  int ok = SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate,
+						filename.c_str());
+
+	  SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio);
+
+	  if (ok) {
+		// AfxMessageBox(Utf8ToUtf16(std::string("Saved to ") + filename + " successfully").c_str(), MB_OK);
+
+		// AppendLineToMultilineEditCtrl(my_hint_, std::string("Saved to ") + filename + " successfully");
+	  } else {
+		// AfxMessageBox(Utf8ToUtf16(std::string("Failed to save to ") + filename).c_str(), MB_OK);
+
+		// AppendLineToMultilineEditCtrl(my_hint_, std::string("Failed to saved to ") + filename);
+	  }
+  });
 
   //CDialogEx::OnOK();
 }
+
+void CNonStreamingTextToSpeechDlg::OnBnClickedStop() { g_killed = true; }

mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.h

@@ -60,5 +60,8 @@ class CNonStreamingTextToSpeechDlg : public CDialogEx
 private:
     Microphone mic_;
 	std::unique_ptr<std::thread> play_thread_;
+	std::unique_ptr<std::thread> generate_thread_;
 
+   public:
+    afx_msg void OnBnClickedStop();
 };