audio: add another stream for sound effects output

src/audio.rs

@@ -39,10 +39,17 @@ pub struct TtsOutStream {
     pub sample_rate: u32
 }
 
+#[derive(Debug)]
+pub struct SfxOutStream {
+    pub sink: mpsc::Sender<Vec<f32>>,
+    pub sample_rate: u32
+}
+
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 struct AudioConfig {
     mic_in_connections: Vec<String>,
-    tts_out_connections: Vec<String>
+    tts_out_connections: Vec<String>,
+    sfx_out_connections: Vec<String>,
 }
 
 impl AudioConfig {
@@ -60,6 +67,7 @@ struct Notify {
     config: AudioConfig,
     mic_port: jack::Port<jack::Unowned>,
     tts_port: jack::Port<jack::Unowned>,
+    sfx_port: jack::Port<jack::Unowned>,
 }
 
 impl NotificationHandler for Notify {
@@ -77,6 +85,8 @@ impl NotificationHandler for Notify {
             ("Microphone input", port_a, &mut self.config.mic_in_connections)
         } else if port_a == self.tts_port {
             ("TTS output", port_b, &mut self.config.tts_out_connections)
+        } else if port_a == self.sfx_port {
+            ("SFX output", port_b, &mut self.config.sfx_out_connections)
         } else {
             return;
         };
@@ -96,7 +106,7 @@ impl NotificationHandler for Notify {
     }
 }
 
-pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream) {
+pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream, SfxOutStream) {
 
     let (exit_tx, exit_rx) = oneshot::channel();
 
@@ -104,11 +114,13 @@ pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream)
 
     let (mic_audio_sink, mic_audio_src) = mpsc::channel(32);
     let (tts_audio_sink, mut tts_audio_src) = mpsc::channel(32);
+    let (sfx_audio_sink, mut sfx_audio_src) = mpsc::channel(32);
     let (volume_sink, volume_src) = watch::channel(0.);
 
     let (client, _status) = jack::Client::new("Eva-Cohost", ClientOptions::default() | ClientOptions::SESSION_ID).unwrap();
     let mic_port = client.register_port("microphone-in", AudioIn::default()).unwrap();
     let mut tts_port = client.register_port("tts-out", AudioOut::default()).unwrap();
+    let mut sfx_port = client.register_port("sfx-out", AudioOut::default()).unwrap();
     let rate = client.sample_rate();
 
     let mic_name = mic_port.name().unwrap();
@@ -134,11 +146,14 @@ pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream)
         config,
         mic_port: mic_port.clone_unowned(),
         tts_port: tts_port.clone_unowned(),
+        sfx_port: sfx_port.clone_unowned()
     };
 
     let mut meter = VuMeter::new(rate.into(), 1, None);
     let mut tts_output_buf = vec![];
+    let mut sfx_output_buf = vec![];
     tts_output_buf.reserve(1024);
+    sfx_output_buf.reserve(1024);
 
     let handler = jack::contrib::ClosureProcessHandler::new(move |_client, scope| {
         if mic_port.connected_count().unwrap() > 0 {
@@ -158,9 +173,31 @@ pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream)
             });
         }
 
-        if let Ok(mut next_outbuf) = tts_audio_src.try_recv() {
+        for (src, output, port) in [
+            (&mut tts_audio_src, &mut tts_output_buf, &mut tts_port),
+            (&mut sfx_audio_src, &mut sfx_output_buf, &mut sfx_port)
+        ] {
+            if let Ok(mut next_outbuf) = src.try_recv() {
+                output.append(&mut next_outbuf);
+            }
+
+            if port.connected_count().unwrap() > 0 && !output.is_empty() {
+                let outbuf = port.as_mut_slice(scope);
+                let mut next_segment: Vec<f32> = output.drain(0..(outbuf.len()).min(output.len())).collect();
+                let underrun = outbuf.len() - next_segment.len();
+                if underrun > 0 {
+                    for _ in 0..underrun {
+                        next_segment.push(0.);
+                    }
+                }
+
+                outbuf.copy_from_slice(&next_segment);
+            }
+
+        }
+        /*if let Ok(mut next_outbuf) = tts_audio_src.try_recv() {
             tts_output_buf.append(&mut next_outbuf);
-        }   
+        }
 
         if tts_port.connected_count().unwrap() > 0 && !tts_output_buf.is_empty() {
             let outbuf = tts_port.as_mut_slice(scope);
@@ -173,7 +210,7 @@ pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream)
             }
 
             outbuf.copy_from_slice(&next_segment);
-        }
+        }*/
 
         jack::Control::Continue
     });
@@ -195,5 +232,8 @@ pub async fn start_audio_input() -> (AudioInputControl, MicStream, TtsOutStream)
     }, TtsOutStream {
         sample_rate: rate,
         sink: tts_audio_sink
+    }, SfxOutStream {
+        sample_rate: rate,
+        sink: sfx_audio_sink
     })
 }

src/main.rs

@@ -126,7 +126,7 @@ async fn main() {
     };
 
     let prediction_ctrl = prediction::start_prediction(saved_session, sys_message_src).await;
-    let (audio_ctrl, mic_stream, tts_output) = start_audio_input().await;
+    let (audio_ctrl, mic_stream, tts_output, sfx_output) = start_audio_input().await;
     let tts_ctrl = start_tts(tts_output).await;
     let transcription_ctrl = transcription::start_transcription(mic_stream).await;